tange/parallel
1
0
Fork 0
mirror of https://git.savannah.gnu.org/git/parallel.git synced 2024-11-22 05:57:54 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

parallel: Prepare for open3_setpgrp_internal. Implemented --termseq. Passes testsuite incl. mem.

Browse source
This commit is contained in:
Ole Tange 2015-06-22 01:44:53 +02:00
parent 6d3892ff84
commit 2029853dd9
3 changed files with 56 additions and 36 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
doc/release_new_version
View file

@ -209,9 +209,9 @@ cc:Tim Cuthbertson <tim3d.junk@gmail.com>,
Ryoichiro Suzuki <ryoichiro.suzuki@gmail.com>,
Jesse Alama <jesse.alama@gmail.com>
Subject: GNU Parallel 20150622 ('<<>>') released
Subject: GNU Parallel 20150622 ('Løkke') released
GNU Parallel 20150622 ('<<>>') has been released. It is available for download at: http://ftp.gnu.org/gnu/parallel/
GNU Parallel 20150622 ('Løkke') has been released. It is available for download at: http://ftp.gnu.org/gnu/parallel/
Haiku of the month:

66
src/parallel
View file

@ -931,7 +931,7 @@ sub parse_options {
if(not defined $opt::blocksize) { $opt::blocksize = "1M"; }
$opt::blocksize = multiply_binary_prefix($opt::blocksize);
if($opt::blocksize > 2**31-1) {
warning("--blocksize >= 2G causes problems. Using 2G-1\n");
warning("--blocksize >= 2G causes problems. Using 2G-1.");
$opt::blocksize = 2**31-1;
}
$opt::memfree = multiply_binary_prefix($opt::memfree);
@ -1078,7 +1078,7 @@ sub parse_options {
sub init_globals {
# Defaults:
$Global::version = 20150607;
$Global::version = 20150620;
$Global::progname = 'parallel';
$Global::infinity = 2**31;
$Global::debug = 0;
@ -6969,6 +6969,7 @@ sub start {
# job-object or undef if job not to run
sub open3_setpgrp_internal {
# Run open3+setpgrp followed by the command
# Input:
# $stdin_fh = Filehandle to use as STDIN
# $stdout_fh = Filehandle to use as STDOUT
@ -6993,29 +6994,48 @@ sub start {
return $pid;
}
sub open3_setpgrp {
my ($stdin_fh,$stdout_fh,$stderr_fh,$command) = @_;
local (*IN,*OUT,*ERR);
if(fileno $stdin_fh) {
# $stdin_fh is a file handle, not a gensym
open IN, '<&', $stdin_fh or ::die_bug("Can't dup STDIN: $!");
$stdin_fh = *IN;
sub open3_setpgrp_external {
# Run open3 on $command wrapped with a perl script doing setpgrp
# Works on systems that do not support open3(,,,"-")
# Input:
# $stdin_fh = Filehandle to use as STDIN
# $stdout_fh = Filehandle to use as STDOUT
# $stderr_fh = Filehandle to use as STDERR
# $command = Command to run
# Returns:
# $pid = Process group of job started
my ($stdin_fh,$stdout_fh,$stderr_fh,$command) = @_;
local (*OUT,*ERR);
open OUT, '>&', $stdout_fh or ::die_bug("Can't dup STDOUT: $!");
open ERR, '>&', $stderr_fh or ::die_bug("Can't dup STDERR: $!");
my $pid;
my @setpgrp_wrap = ('perl','-e',"setpgrp\;exec '$Global::shell', '-c', \@ARGV");
# The eval is needed to catch exception from open3
eval {
$pid = ::open3($stdin_fh, ">&OUT", ">&ERR", @setpgrp_wrap, $command)
|| ::die_bug("open3-$stdin_fh");
1;
};
return $pid;
}
open OUT, '>&', $stdout_fh or ::die_bug("Can't dup STDOUT: $!");
open ERR, '>&', $stderr_fh or ::die_bug("Can't dup STDERR: $!");
my $pid;
my @setpgrp_wrap = ('perl','-e',"setpgrp\;exec '$Global::shell', '-c', \@ARGV");
# The eval is needed to catch exception from open3
eval {
$pid = ::open3($stdin_fh, ">&OUT", ">&ERR", @setpgrp_wrap, $command)
|| ::die_bug("open3-$stdin_fh");
1;
};
return $pid;
}
sub open3_setpgrp {
# If the OS supports open3(x,x,x,"-") use that
# eval { if(not $pid=::open3($i,$o,$e,"-")) { exit } }
# if $!: external
# TODO build a selector that works with out side effects
no warnings 'redefine';
if(1) {
# Does not support open3(x,x,x,"-")
*open3_setpgrp = \&open3_setpgrp_external;
} else {
# Supports open3(x,x,x,"-")
*open3_setpgrp = \&open3_setpgrp_internal;
}
# The sub is now redefined. Call it
return open3_setpgrp(@_);
}
my $job = shift;
# Get the shell command to be executed (possibly with ssh infront).

22
testsuite/wanted-results/parallel-local-mem
View file

@ -16,12 +16,12 @@ Eats 2.5 GB
(yes "`seq 3000`" | head -c 2300M; echo ged) | PERL5LIB=input-files/perl-v5.14.2/lib input-files/perl-v5.14.2/perl `which parallel` -k --block 2G --pipe --recend ged md5sum
7be046296f6b0cde6049a11c116908ea -
68b329da9893e34099c7d8ad5cb9c940 -
parallel: Warning: --blocksize >= 2G causes problems. Using 2G-1
parallel: Warning: --blocksize >= 2G causes problems. Using 2G-1.
echo 'Eats 2.5 GB'
Eats 2.5 GB
(yes "`seq 3000`" | head -c 2300M; echo ged) | PERL5LIB=input-files/perl-v5.14.2/lib input-files/perl-v5.14.2/perl `which parallel` -k --block 2G --pipe --recend ged cat | wc -c
2411724804
parallel: Warning: --blocksize >= 2G causes problems. Using 2G-1
parallel: Warning: --blocksize >= 2G causes problems. Using 2G-1.
echo '**'
**
echo '### bug #44358: 2 GB records cause problems for -N'
@ -31,25 +31,25 @@ echo '5 GB version: Eats 12.5 GB'
(yes "`seq 3000`" | head -c 5000000000; echo FOO; yes "`seq 3000`" | head -c 3000000000; echo FOO; yes "`seq 3000`" | head -c 1000000000;) | PERL5LIB=input-files/perl-v5.14.2/lib input-files/perl-v5.14.2/perl `which parallel` --pipe --recend FOO -N2 --block 1g -k LANG=c wc -c
8000000007
1000000001
parallel: Warning: A record was longer than 1000000000. Increasing to --blocksize 1300000001
parallel: Warning: A record was longer than 1300000001. Increasing to --blocksize 1690000003
parallel: Warning: A record was longer than 1690000003. Increasing to --blocksize 2147483647
parallel: Warning: A record was longer than 1000000000. Increasing to --blocksize 1300000001.
parallel: Warning: A record was longer than 1300000001. Increasing to --blocksize 1690000003.
parallel: Warning: A record was longer than 1690000003. Increasing to --blocksize 2147483647.
echo '2 GB version: eats 10 GB'
2 GB version: eats 10 GB
(yes "`seq 3000`" | head -c 2300M; echo FOO; yes "`seq 3000`" | head -c 2300M; echo FOO; yes "`seq 3000`" | head -c 1000M;) | PERL5LIB=input-files/perl-v5.14.2/lib input-files/perl-v5.14.2/perl `which parallel` --pipe --recend FOO -N2 --block 1g -k LANG=c wc -c
4823449607
1048576001
parallel: Warning: A record was longer than 1000000000. Increasing to --blocksize 1300000001
parallel: Warning: A record was longer than 1300000001. Increasing to --blocksize 1690000003
parallel: Warning: A record was longer than 1690000003. Increasing to --blocksize 2147483647
parallel: Warning: A record was longer than 1000000000. Increasing to --blocksize 1300000001.
parallel: Warning: A record was longer than 1300000001. Increasing to --blocksize 1690000003.
parallel: Warning: A record was longer than 1690000003. Increasing to --blocksize 2147483647.
echo '### -L >4GB'
### -L >4GB
(head -c 5000000000 /dev/zero; echo FOO; head -c 3000000000 /dev/zero; echo FOO; head -c 1000000000 /dev/zero;) | parallel --pipe -L2 --block 1g -k LANG=c wc -c
8000000008
1000000000
parallel: Warning: A record was longer than 1000000000. Increasing to --blocksize 1300000001
parallel: Warning: A record was longer than 1300000001. Increasing to --blocksize 1690000003
parallel: Warning: A record was longer than 1690000003. Increasing to --blocksize 2147483647
parallel: Warning: A record was longer than 1000000000. Increasing to --blocksize 1300000001.
parallel: Warning: A record was longer than 1300000001. Increasing to --blocksize 1690000003.
parallel: Warning: A record was longer than 1690000003. Increasing to --blocksize 2147483647.
make[1]: Entering directory `/home/tange/privat/parallel/testsuite'
# Make sure we can reach the virtual machines
sudo ifconfig wlan0:0 192.168.1.72