From 2029853dd9ed61a08d6273f23268766355b33435 Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Mon, 22 Jun 2015 01:44:53 +0200 Subject: [PATCH] parallel: Prepare for open3_setpgrp_internal. Implemented --termseq. Passes testsuite incl. mem. --- doc/release_new_version | 4 +- src/parallel | 66 ++++++++++++++------- testsuite/wanted-results/parallel-local-mem | 22 +++---- 3 files changed, 56 insertions(+), 36 deletions(-) diff --git a/doc/release_new_version b/doc/release_new_version index d151317c..cdec5419 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -209,9 +209,9 @@ cc:Tim Cuthbertson , Ryoichiro Suzuki , Jesse Alama -Subject: GNU Parallel 20150622 ('<<>>') released +Subject: GNU Parallel 20150622 ('Løkke') released -GNU Parallel 20150622 ('<<>>') has been released. It is available for download at: http://ftp.gnu.org/gnu/parallel/ +GNU Parallel 20150622 ('Løkke') has been released. It is available for download at: http://ftp.gnu.org/gnu/parallel/ Haiku of the month: diff --git a/src/parallel b/src/parallel index a6fc1dc1..eaa450cf 100755 --- a/src/parallel +++ b/src/parallel @@ -931,7 +931,7 @@ sub parse_options { if(not defined $opt::blocksize) { $opt::blocksize = "1M"; } $opt::blocksize = multiply_binary_prefix($opt::blocksize); if($opt::blocksize > 2**31-1) { - warning("--blocksize >= 2G causes problems. Using 2G-1\n"); + warning("--blocksize >= 2G causes problems. Using 2G-1."); $opt::blocksize = 2**31-1; } $opt::memfree = multiply_binary_prefix($opt::memfree); @@ -1078,7 +1078,7 @@ sub parse_options { sub init_globals { # Defaults: - $Global::version = 20150607; + $Global::version = 20150620; $Global::progname = 'parallel'; $Global::infinity = 2**31; $Global::debug = 0; @@ -6969,6 +6969,7 @@ sub start { # job-object or undef if job not to run sub open3_setpgrp_internal { + # Run open3+setpgrp followed by the command # Input: # $stdin_fh = Filehandle to use as STDIN # $stdout_fh = Filehandle to use as STDOUT @@ -6993,29 +6994,48 @@ sub start { return $pid; } -sub open3_setpgrp { - my ($stdin_fh,$stdout_fh,$stderr_fh,$command) = @_; - local (*IN,*OUT,*ERR); - if(fileno $stdin_fh) { - # $stdin_fh is a file handle, not a gensym - open IN, '<&', $stdin_fh or ::die_bug("Can't dup STDIN: $!"); - $stdin_fh = *IN; + sub open3_setpgrp_external { + # Run open3 on $command wrapped with a perl script doing setpgrp + # Works on systems that do not support open3(,,,"-") + # Input: + # $stdin_fh = Filehandle to use as STDIN + # $stdout_fh = Filehandle to use as STDOUT + # $stderr_fh = Filehandle to use as STDERR + # $command = Command to run + # Returns: + # $pid = Process group of job started + my ($stdin_fh,$stdout_fh,$stderr_fh,$command) = @_; + local (*OUT,*ERR); + open OUT, '>&', $stdout_fh or ::die_bug("Can't dup STDOUT: $!"); + open ERR, '>&', $stderr_fh or ::die_bug("Can't dup STDERR: $!"); + + my $pid; + my @setpgrp_wrap = ('perl','-e',"setpgrp\;exec '$Global::shell', '-c', \@ARGV"); + # The eval is needed to catch exception from open3 + eval { + $pid = ::open3($stdin_fh, ">&OUT", ">&ERR", @setpgrp_wrap, $command) + || ::die_bug("open3-$stdin_fh"); + 1; + }; + return $pid; } - open OUT, '>&', $stdout_fh or ::die_bug("Can't dup STDOUT: $!"); - open ERR, '>&', $stderr_fh or ::die_bug("Can't dup STDERR: $!"); - - my $pid; - my @setpgrp_wrap = ('perl','-e',"setpgrp\;exec '$Global::shell', '-c', \@ARGV"); - # The eval is needed to catch exception from open3 - eval { - $pid = ::open3($stdin_fh, ">&OUT", ">&ERR", @setpgrp_wrap, $command) - || ::die_bug("open3-$stdin_fh"); - 1; - }; - return $pid; -} - + sub open3_setpgrp { + # If the OS supports open3(x,x,x,"-") use that + # eval { if(not $pid=::open3($i,$o,$e,"-")) { exit } } + # if $!: external + # TODO build a selector that works with out side effects + no warnings 'redefine'; + if(1) { + # Does not support open3(x,x,x,"-") + *open3_setpgrp = \&open3_setpgrp_external; + } else { + # Supports open3(x,x,x,"-") + *open3_setpgrp = \&open3_setpgrp_internal; + } + # The sub is now redefined. Call it + return open3_setpgrp(@_); + } my $job = shift; # Get the shell command to be executed (possibly with ssh infront). diff --git a/testsuite/wanted-results/parallel-local-mem b/testsuite/wanted-results/parallel-local-mem index b7e63152..8e48e8e5 100644 --- a/testsuite/wanted-results/parallel-local-mem +++ b/testsuite/wanted-results/parallel-local-mem @@ -16,12 +16,12 @@ Eats 2.5 GB (yes "`seq 3000`" | head -c 2300M; echo ged) | PERL5LIB=input-files/perl-v5.14.2/lib input-files/perl-v5.14.2/perl `which parallel` -k --block 2G --pipe --recend ged md5sum 7be046296f6b0cde6049a11c116908ea - 68b329da9893e34099c7d8ad5cb9c940 - -parallel: Warning: --blocksize >= 2G causes problems. Using 2G-1 +parallel: Warning: --blocksize >= 2G causes problems. Using 2G-1. echo 'Eats 2.5 GB' Eats 2.5 GB (yes "`seq 3000`" | head -c 2300M; echo ged) | PERL5LIB=input-files/perl-v5.14.2/lib input-files/perl-v5.14.2/perl `which parallel` -k --block 2G --pipe --recend ged cat | wc -c 2411724804 -parallel: Warning: --blocksize >= 2G causes problems. Using 2G-1 +parallel: Warning: --blocksize >= 2G causes problems. Using 2G-1. echo '**' ** echo '### bug #44358: 2 GB records cause problems for -N' @@ -31,25 +31,25 @@ echo '5 GB version: Eats 12.5 GB' (yes "`seq 3000`" | head -c 5000000000; echo FOO; yes "`seq 3000`" | head -c 3000000000; echo FOO; yes "`seq 3000`" | head -c 1000000000;) | PERL5LIB=input-files/perl-v5.14.2/lib input-files/perl-v5.14.2/perl `which parallel` --pipe --recend FOO -N2 --block 1g -k LANG=c wc -c 8000000007 1000000001 -parallel: Warning: A record was longer than 1000000000. Increasing to --blocksize 1300000001 -parallel: Warning: A record was longer than 1300000001. Increasing to --blocksize 1690000003 -parallel: Warning: A record was longer than 1690000003. Increasing to --blocksize 2147483647 +parallel: Warning: A record was longer than 1000000000. Increasing to --blocksize 1300000001. +parallel: Warning: A record was longer than 1300000001. Increasing to --blocksize 1690000003. +parallel: Warning: A record was longer than 1690000003. Increasing to --blocksize 2147483647. echo '2 GB version: eats 10 GB' 2 GB version: eats 10 GB (yes "`seq 3000`" | head -c 2300M; echo FOO; yes "`seq 3000`" | head -c 2300M; echo FOO; yes "`seq 3000`" | head -c 1000M;) | PERL5LIB=input-files/perl-v5.14.2/lib input-files/perl-v5.14.2/perl `which parallel` --pipe --recend FOO -N2 --block 1g -k LANG=c wc -c 4823449607 1048576001 -parallel: Warning: A record was longer than 1000000000. Increasing to --blocksize 1300000001 -parallel: Warning: A record was longer than 1300000001. Increasing to --blocksize 1690000003 -parallel: Warning: A record was longer than 1690000003. Increasing to --blocksize 2147483647 +parallel: Warning: A record was longer than 1000000000. Increasing to --blocksize 1300000001. +parallel: Warning: A record was longer than 1300000001. Increasing to --blocksize 1690000003. +parallel: Warning: A record was longer than 1690000003. Increasing to --blocksize 2147483647. echo '### -L >4GB' ### -L >4GB (head -c 5000000000 /dev/zero; echo FOO; head -c 3000000000 /dev/zero; echo FOO; head -c 1000000000 /dev/zero;) | parallel --pipe -L2 --block 1g -k LANG=c wc -c 8000000008 1000000000 -parallel: Warning: A record was longer than 1000000000. Increasing to --blocksize 1300000001 -parallel: Warning: A record was longer than 1300000001. Increasing to --blocksize 1690000003 -parallel: Warning: A record was longer than 1690000003. Increasing to --blocksize 2147483647 +parallel: Warning: A record was longer than 1000000000. Increasing to --blocksize 1300000001. +parallel: Warning: A record was longer than 1300000001. Increasing to --blocksize 1690000003. +parallel: Warning: A record was longer than 1690000003. Increasing to --blocksize 2147483647. make[1]: Entering directory `/home/tange/privat/parallel/testsuite' # Make sure we can reach the virtual machines sudo ifconfig wlan0:0 192.168.1.72