From d3a9292b2a4e98ceaf76cbd48aad0048cd96aa5e Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Sun, 14 Jul 2013 13:22:56 +0200 Subject: [PATCH 1/7] parallel: Treat STDOUT and STDERR as fd{1} and fd{2}. Find all shell opened file descriptors. --- doc/release_new_version | 2 + src/parallel | 127 ++++++++++++++++++++-------------------- 2 files changed, 66 insertions(+), 63 deletions(-) diff --git a/doc/release_new_version b/doc/release_new_version index 738b7f04..759e8d11 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -217,6 +217,8 @@ New in this release: * http://www.brunokim.com.br/blog/?p=18 +* http://timotheepoisot.fr/2013/07/08/parallel/ + * http://www.open-open.com/news/view/371301 * Bug fixes and man page updates. diff --git a/src/parallel b/src/parallel index 8c1026db..5375d566 100755 --- a/src/parallel +++ b/src/parallel @@ -33,7 +33,6 @@ use Getopt::Long; # Used to ensure code quality use strict; -$SIG{TERM} ||= sub { exit 0; }; # $SIG{TERM} is not set on Mac OS X if(not $ENV{SHELL}) { # $ENV{SHELL} is sometimes not set on Mac OS X and Windows ::warning("\$SHELL not set. Using /bin/sh.\n"); @@ -45,9 +44,10 @@ if(not $ENV{HOME}) { $ENV{HOME} = "/tmp"; } -save_sig_stdin_stdout_stderr(); - +save_stdin_stdout_stderr(); +save_original_signal_handler(); parse_options(); +::debug("Open file descriptors: ".join(" ",keys %Global::fd)."\n"); my $number_of_args; if($Global::max_number_of_args) { $number_of_args=$Global::max_number_of_args; @@ -481,12 +481,12 @@ sub write_record_to_pipe { } $job->write($header_ref); $job->write($record_ref); - my $fh = $job->stdin(); - close $fh; + my $stdin_fh = $job->fd(0); + close $stdin_fh; exit(0); } - my $fh = $job->stdin(); - close $fh; + my $stdin_fh = $job->fd(0); + close $stdin_fh; return 1; } @@ -1007,7 +1007,7 @@ sub open_joblog { } else { if($opt::joblog eq "-") { # Use STDOUT as joblog - $Global::joblog = $Global::original_stdout + $Global::joblog = $Global::fd{1}; } elsif(not open($Global::joblog, ">", $opt::joblog)) { # Overwrite the joblog ::error("Cannot write to --joblog $opt::joblog.\n"); @@ -1254,6 +1254,27 @@ sub shell_unquote { sub __FILEHANDLES__ {} + +sub save_stdin_stdout_stderr { + # Remember the original STDIN, STDOUT and STDERR + # and file descriptors opened by the shell (e.g. 3>/tmp/foo) + # Returns: N/A + + # Find file descriptors that are already opened (by the shell) + for (1..60) { + # /dev/fd/62 and above are used by bash for <(cmd) + my $fh; + if(open($fh,">&=$_")) { + $Global::fd{$_}=$fh; + } + } + open $Global::original_stderr, ">&", "STDERR" or + ::die_bug("Can't dup STDERR: $!"); + open $Global::original_stdin, "<&", "STDIN" or + ::die_bug("Can't dup STDIN: $!"); + $Global::fd{0} = $Global::original_stdin; +} + sub enough_file_handles { # check that we have enough filehandles available for starting # another job @@ -1308,19 +1329,6 @@ sub __RUNNING_THE_JOBS_AND_PRINTING_PROGRESS__ {} # $Global::total_running = total number of running jobs # $Global::total_started = total jobs started -sub save_sig_stdin_stdout_stderr { - # Remember the original signal handler, STDIN, STDOUT and STDERR - # Returns: N/A - %Global::original_sig = %SIG; - $SIG{TERM} = sub {}; # Dummy until jobs really start - open $Global::original_stdout, ">&", "STDOUT" or - ::die_bug("Can't dup STDOUT: $!"); - open $Global::original_stderr, ">&", "STDERR" or - ::die_bug("Can't dup STDERR: $!"); - open $Global::original_stdin, "<&", "STDIN" or - ::die_bug("Can't dup STDIN: $!"); -} - sub init_run_jobs { $Global::total_running = 0; $Global::total_started = 0; @@ -1468,8 +1476,8 @@ sub drain_job_queue { if($opt::pipe) { # When using --pipe sometimes file handles are not closed properly for my $job (values %Global::running) { - my $fh = $job->stdin(); - close $fh; + my $stdin_fh = $job->fd(0); + close $stdin_fh; } } if($opt::progress) { @@ -1865,6 +1873,14 @@ sub cleanup_basefile { sub __SIGNAL_HANDLING__ {} +sub save_original_signal_handler { + # Remember the original signal handler + # Returns: N/A + $SIG{TERM} ||= sub { exit 0; }; # $SIG{TERM} is not set on Mac OS X + %Global::original_sig = %SIG; + $SIG{TERM} = sub {}; # Dummy until jobs really start +} + sub list_running_jobs { # Returns: N/A for my $v (values %Global::running) { @@ -2256,8 +2272,10 @@ sub debug { # Returns: N/A $Global::debug or return; @_ = grep { defined $_ ? $_ : "" } @_; - if($Global::original_stdout) { - print $Global::original_stdout @_; + if($Global::fd{1}) { + # Original stdout was saved + my $stdout = $Global::fd{1}; + print $stdout @_; } else { print @_; } @@ -3587,18 +3605,22 @@ sub openresultsfile { } open OUT, '>&', $outfh or ::die_bug("Can't redirect STDOUT: $!"); open ERR, '>&', $errfh or ::die_bug("Can't dup STDOUT: $!"); - $self->set_stdout($outfh); - $self->set_stderr($errfh); + $self->set_fd(1,$outfh); + $self->set_fd(2,$errfh); } -sub set_stdout { +sub set_fd { + # Set file descriptor my $self = shift; - $self->{'stdout'} = shift; + my $fd_no = shift; + $self->{'fd'}{$fd_no} = shift; } -sub stdout { +sub fd { + # Get file descriptor my $self = shift; - return $self->{'stdout'}; + my $fd_no = shift; + return $self->{'fd'}{$fd_no}; } sub set_stdoutfilename { @@ -3611,32 +3633,11 @@ sub stdoutfilename { return $self->{'stdoutfilename'}; } -sub stderr { - my $self = shift; - return $self->{'stderr'}; -} - -sub set_stderr { - my $self = shift; - $self->{'stderr'} = shift; -} - -sub stdin { - my $self = shift; - return $self->{'stdin'}; -} - -sub set_stdin { - my $self = shift; - my $stdin = shift; - $self->{'stdin'} = $stdin; -} - sub write { my $self = shift; my $remaining_ref = shift; - my $in = $self->{'stdin'}; - syswrite($in,$$remaining_ref); + my $stdin_fh = $self->fd(0); + syswrite($stdin_fh,$$remaining_ref); } sub virgin { @@ -3676,7 +3677,7 @@ sub runtime { # Returns: # Run time in seconds my $self = shift; - return int(($self->endtime() - $self->starttime())*1000)/1000; + return sprintf("%.3f",int(($self->endtime() - $self->starttime())*1000)/1000); } sub endtime { @@ -4159,8 +4160,8 @@ sub start { open OUT, '>&', $outfh or ::die_bug("Can't redirect STDOUT: $!"); open ERR, '>&', $errfh or ::die_bug("Can't dup STDOUT: $!"); - $job->set_stdout($outfh); - $job->set_stderr($errfh); + $job->set_fd(1,$outfh); + $job->set_fd(2,$errfh); } else { (*OUT,*ERR)=(*STDOUT,*STDERR); } @@ -4181,7 +4182,7 @@ sub start { ::debug("$Global::total_running processes. Starting (" . $job->seq() . "): $command\n"); if($opt::pipe) { - my ($in); + my ($stdin_fh); # Wrap command with end-of-file detector, # so we do not spawn a program if there is no input. # Exit value: @@ -4198,11 +4199,11 @@ sub start { "($command);"; # The eval is needed to catch exception from open3 eval { - $pid = ::open3($in, ">&OUT", ">&ERR", $ENV{SHELL}, "-c", $command) || + $pid = ::open3($stdin_fh, ">&OUT", ">&ERR", $ENV{SHELL}, "-c", $command) || ::die_bug("open3-pipe"); 1; }; - $job->set_stdin($in); + $job->set_fd(0,$stdin_fh); } elsif(@opt::a and not $Global::stdin_in_opt_a and $job->seq() == 1 and $job->sshlogin()->string() eq ":") { # Give STDIN to the first job if using -a (but only if running @@ -4215,7 +4216,7 @@ sub start { 1; }; # Re-open to avoid complaining - open STDIN, "<&", $Global::original_stdin + open(STDIN, "<&", $Global::original_stdin) or ::die_bug("dup-\$Global::original_stdin: $!"); } elsif ($opt::tty and not $Global::tty_taken and -c "/dev/tty" and open(my $devtty_fh, "<", "/dev/tty")) { @@ -4316,8 +4317,8 @@ sub print { } # Only relevant for grouping $Global::grouped or return; - my $out = $self->stdout(); - my $err = $self->stderr(); + my $out = $self->fd(1); + my $err = $self->fd(2); my $command = $self->sshlogin_wrap(); if($Global::joblog) { From 7ff2e711dbe5713a3bb0a5ee8547d35177c8d9e1 Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Fri, 19 Jul 2013 17:42:22 +0200 Subject: [PATCH 2/7] 10seconds_install: Some Polarhomes do not support '-o'. --- 10seconds_install | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/10seconds_install b/10seconds_install index 02c54ec7..e67badb1 100644 --- a/10seconds_install +++ b/10seconds_install @@ -46,12 +46,12 @@ else echo echo "Continue anyway? (y/n)" read YN - if test $YN = "y" -o $YN = "Y"; then - # Continue - true - else + if test "$YN" = "n"; then # Stop exit 2 + else + # Continue + true fi fi From dd1a63782260be3d70d37a704a8b456900051ca6 Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Fri, 19 Jul 2013 19:13:00 +0200 Subject: [PATCH 3/7] parallel: Use file descriptors instead of STDOUT/STDERR. Fixed up Polarhome tests. Passes testsuite. --- src/parallel | 150 ++++----- testsuite/tests-to-run/parallel-local11.sh | 30 +- testsuite/tests-to-run/parallel-local18.sh | 14 +- testsuite/tests-to-run/parallel-polarhome.sh | 34 +- testsuite/wanted-results/parallel-local11 | 36 ++- testsuite/wanted-results/parallel-local18 | 32 +- testsuite/wanted-results/parallel-local9 | 8 +- testsuite/wanted-results/parallel-polarhome | 312 +++---------------- testsuite/wanted-results/test13 | 4 +- 9 files changed, 224 insertions(+), 396 deletions(-) diff --git a/src/parallel b/src/parallel index 2474eb26..2d2d8dbb 100755 --- a/src/parallel +++ b/src/parallel @@ -1310,18 +1310,17 @@ sub save_stdin_stdout_stderr { # Returns: N/A # Find file descriptors that are already opened (by the shell) - for (1..60) { + for my $fdno (1..61) { # /dev/fd/62 and above are used by bash for <(cmd) my $fh; - if(open($fh,">&=$_")) { - $Global::fd{$_}=$fh; + if(open($fh,">&=",$fdno)) { + $Global::fd{$fdno}=$fh; } } open $Global::original_stderr, ">&", "STDERR" or ::die_bug("Can't dup STDERR: $!"); open $Global::original_stdin, "<&", "STDIN" or ::die_bug("Can't dup STDIN: $!"); - $Global::fd{0} = $Global::original_stdin; } sub enough_file_handles { @@ -1334,16 +1333,18 @@ sub enough_file_handles { if($Global::grouped) { my %fh; my $enough_filehandles = 1; - # We need a filehandle for STDOUT and STDERR - # perl uses 7 filehandles for something? + # perl uses 7 filehandles for something? # open3 uses 2 extra filehandles temporarily - for my $i (1..8) { + # We need a filehandle for each redirected file descriptor + # (normally just STDOUT and STDERR) + for my $i (1..(7+2+keys %Global::fd)) { $enough_filehandles &&= open($fh{$i}, "<", "/dev/null"); } for (values %fh) { close $_; } return $enough_filehandles; } else { - return 1; + # Ungrouped does not need extra file handles + return 1; } } @@ -1490,13 +1491,15 @@ sub start_another_job { ::usleep(rand()*300); ::warning("No more processes: ", "Decreasing number of running jobs to $max. ", - "Raising ulimit -u may help.\n"); + "Raising ulimit -u or /etc/security/limits.conf may help.\n"); return 0; } } } else { # No more file handles - debug("Not starting: no more file handles\n"); + $Global::no_more_file_handles_warned++ or + ::warning("No more file handles. ". + "Raising ulimit -n or /etc/security/limits.conf may help.\n"); return 0; } } @@ -2882,16 +2885,16 @@ sub processes_available_by_system_limit { if($system_limit < $wanted_processes) { # The system_limit is less than the wanted_processes if($system_limit < 1 and not $Global::JobQueue->empty()) { - ::warning("Cannot spawn any jobs. Raising ulimit -u may help.\n"); + ::warning("Cannot spawn any jobs. Raising ulimit -u or /etc/security/limits.conf may help.\n"); ::wait_and_exit(255); } if(not $more_filehandles) { - ::warning("Only enough filehandles to run ", $system_limit, - " jobs in parallel. Raising ulimit -n may help.\n"); + ::warning("Only enough file handles to run ", $system_limit, " jobs in parallel.\n", + "Raising ulimit -n or /etc/security/limits.conf may help.\n"); } if($max_system_proc_reached) { ::warning("Only enough available processes to run ", $system_limit, - " jobs in parallel. Raising ulimit -u may help.\n"); + " jobs in parallel. Raising ulimit -u or /etc/security/limits.conf may help.\n"); } } if($] == 5.008008 and $system_limit > 1000) { @@ -3599,10 +3602,7 @@ sub new { 'commandline' => $commandline, # The commandline with no args 'workdir' => undef, # --workdir 'stdin' => undef, # filehandle for stdin (used for --pipe) - 'stdout' => undef, # filehandle for stdout (used for --group) # filename for writing stdout to (used for --files) - 'stdoutfilename' => undef, - 'stderr' => undef, # filehandle for stderr (used for --group) 'remaining' => "", # remaining data not sent to stdin (used for --pipe) 'datawritten' => 0, # amount of data sent via stdin (used for --pipe) 'transfersize' => 0, # size of files using --transfer @@ -3645,13 +3645,14 @@ sub openresultsfile { ::error("Cannot write to `$name'.\n"); ::wait_and_exit(255); } - $self->set_stdoutfilename($name); + $self->set_fd_file_name(1,$name); # prefix/name1/val1/name2/val2/stderr $name = "$dir/stderr"; if(not open($errfh,"+>",$name)) { ::error("Cannot write to `$name'.\n"); ::wait_and_exit(255); } + $self->set_fd_file_name(2,$name); open OUT, '>&', $outfh or ::die_bug("Can't redirect STDOUT: $!"); open ERR, '>&', $errfh or ::die_bug("Can't dup STDOUT: $!"); $self->set_fd(1,$outfh); @@ -3672,14 +3673,18 @@ sub fd { return $self->{'fd'}{$fd_no}; } -sub set_stdoutfilename { +sub set_fd_file_name { + # Set file name for a file descriptor my $self = shift; - $self->{'stdoutfilename'} = shift; + my $fd_no = shift; + $self->{'fd_file_name',$fd_no} = shift; } -sub stdoutfilename { +sub fd_file_name { + # Get file name for a file descriptor my $self = shift; - return $self->{'stdoutfilename'}; + my $fd_no = shift; + return $self->{'fd_file_name',$fd_no}; } sub write { @@ -4229,7 +4234,7 @@ sub start { # To group we create temporary files for STDOUT and STDERR # To avoid the cleanup unlink the files immediately (but keep them open) ($outfh, $name) = ::tempfile(SUFFIX => ".par"); - $job->set_stdoutfilename($name); + $job->set_fd_file_name(1,$name); $opt::files or unlink $name; ($errfh, $name) = ::tempfile(SUFFIX => ".par"); unlink $name; @@ -4383,18 +4388,16 @@ sub print { if($opt::pipe and $self->virgin()) { # Nothing was printed to this job: # cleanup tmp files if --files was set - unlink $self->{'stdoutfilename'}; + unlink $self->fd_file_name(1); return; } if($opt::dryrun) { # Nothing was printed to this job: # cleanup tmp files if --files was set - unlink $self->{'stdoutfilename'}; + unlink $self->fd_file_name(1); } # Only relevant for grouping $Global::grouped or return; - my $out = $self->fd(1); - my $err = $self->fd(2); my $command = $self->sshlogin_wrap(); if($Global::joblog) { @@ -4427,60 +4430,61 @@ sub print { # so flush to avoid STDOUT being buffered flush STDOUT; } - seek $err, 0, 0; - if($Global::debug) { - print STDERR "ERR:\n"; - } - if($opt::tag or defined $opt::tagstring) { - my $tag = $self->tag(); - # OpenSSH_3.6.1p2 gives 'tcgetattr: Invalid argument' with -tt - # This is a crappy way of ignoring it. - while(<$err>) { - if($_ ne "tcgetattr: Invalid argument\n") { - print STDERR $tag,$_; - } - # At most run the loop once - last; + for my $fdno (sort { $a <=> $b } keys %Global::fd) { + # Sort by file descriptor numerically: 1,2,3,..,9,10,11 + $fdno == 0 and next; + my $out_fd = $Global::fd{$fdno}; + my $in_fd = $self->fd($fdno); + if(not $in_fd) { +# ::warning("File descriptor $fdno not defined\n"); + next; } - while(<$err>) { - print STDERR $tag,$_; - } - } else { - my $buf; - sysread($err,$buf,1_000_000); - # OpenSSH_3.6.1p2 gives 'tcgetattr: Invalid argument' with -tt - $buf =~ s/^tcgetattr: Invalid argument\n//; - print STDERR $buf; - while(sysread($err,$buf,1_000_000)) { - print STDERR $buf; - } - } - flush STDERR; - - if($opt::files) { - print STDOUT $self->{'stdoutfilename'},"\n"; - } else { - my $buf; - seek $out, 0, 0; + seek $in_fd, 0, 0; if($Global::debug) { - print STDOUT "OUT:\n"; + print STDERR "File descriptor $fdno:\n"; } - if($opt::tag or defined $opt::tagstring) { - my $tag = $self->tag(); - while(<$out>) { - print STDOUT $tag,$_; - } + if($opt::files) { + $self->fd_file_name($fdno) and print $out_fd $self->fd_file_name($fdno),"\n"; } else { my $buf; - while(sysread($out,$buf,1_000_000)) { - print STDOUT $buf; + seek $in_fd, 0, 0; + if($Global::debug) { + print STDOUT "OUT:\n"; + } + if($opt::tag or defined $opt::tagstring) { + my $tag = $self->tag(); + if($fdno == 2) { + # OpenSSH_3.6.1p2 gives 'tcgetattr: Invalid argument' with -tt + # This is a crappy way of ignoring it. + while(<$in_fd>) { + if($_ ne "tcgetattr: Invalid argument\n") { + print $out_fd $tag,$_; + } + # At most run the loop once + last; + } + } + while(<$in_fd>) { + print $out_fd $tag,$_; + } + } else { + my $buf; + if($fdno == 2) { + # OpenSSH_3.6.1p2 gives 'tcgetattr: Invalid argument' with -tt + # This is a crappy way of ignoring it. + sysread($in_fd,$buf,1_000_000); + $buf =~ s/^tcgetattr: Invalid argument\n//; + print $out_fd $buf; + } + while(sysread($in_fd,$buf,1_000_000)) { + print $out_fd $buf; + } } } - flush STDOUT; + flush $out_fd; ::debug("< 4 GB' -# echo | niceload --io 10 parallel -q perl -e '"\$a=\"x\"x1000000;for(0..4300){print \$a}"' | md5sum +## echo | niceload --io 10 parallel -q perl -e '"\$a=\"x\"x1000000;for(0..4300){print \$a}"' | md5sum echo | parallel -q perl -e '$a="x"x1000000;for(0..4300){print $a}' | md5sum +echo '**' + echo "### Test Force outside the file handle limit, 2009-02-17 Gave fork error" (echo echo Start; seq 1 20000 | perl -pe 's/^/true /'; echo echo end) | stdout parallel -uj 0 | egrep -v 'processes took|adjusting' +echo '**' + echo '### Test of --retries on unreachable host' seq 2 | stdout parallel -k --retries 2 -v -S 4.3.2.1,: echo +echo '**' + echo '### Test race condition on 8 CPU (my laptop)'; seq 1 5000000 > /tmp/parallel_test; seq 1 10 | parallel -k "cat /tmp/parallel_test | parallel --pipe --recend '' -k gzip >/dev/null; echo {}" +echo '**' + echo '### Test exit val - true'; echo true | parallel; echo $? +echo '**' + echo '### Test exit val - false'; echo false | parallel; echo $? +echo '**' + echo '### Test --halt-on-error 0'; (echo "sleep 1;true"; echo "sleep 2;false";echo "sleep 3;true") | parallel -j10 --halt-on-error 0; echo $?; (echo "sleep 1;true"; echo "sleep 2;false";echo "sleep 3;true";echo "sleep 4; non_exist") | parallel -j10 --halt 0; echo $? +echo '**' + echo '### Test --halt-on-error 1'; (echo "sleep 1;true"; echo "sleep 2;false";echo "sleep 3;true") | parallel -j10 --halt-on-error 1; echo $?; (echo "sleep 1;true"; echo "sleep 2;false";echo "sleep 3;true";echo "sleep 4; non_exist") | parallel -j10 --halt 1; echo $? +echo '**' + echo '### Test --halt-on-error 2'; (echo "sleep 1;true"; echo "sleep 2;false";echo "sleep 3;true") | parallel -j10 --halt-on-error 2; echo $?; (echo "sleep 1;true"; echo "sleep 2;false";echo "sleep 3;true";echo "sleep 4; non_exist") | parallel -j10 --halt 2; echo $? -echo '### Test last dying print --halt-on-error'; +echo '**' + +echo '### Test last dying print --halt-on-error 1'; (seq 0 8;echo 0; echo 9) | parallel -j10 -kq --halt 1 perl -e 'sleep $ARGV[0];print STDERR @ARGV,"\n"; exit shift'; - echo $?; + echo exit code $? + +echo '### Test last dying print --halt-on-error 2'; (seq 0 8;echo 0; echo 9) | parallel -j10 -kq --halt 2 perl -e 'sleep $ARGV[0];print STDERR @ARGV,"\n"; exit shift'; - echo $? + echo exit code $? + +echo '**' echo '### Test slow arguments generation - https://savannah.gnu.org/bugs/?32834'; seq 1 3 | parallel -j1 "sleep 2; echo {}" | parallel -kj2 echo diff --git a/testsuite/tests-to-run/parallel-local18.sh b/testsuite/tests-to-run/parallel-local18.sh index 1fbe3ed2..08a66ca1 100644 --- a/testsuite/tests-to-run/parallel-local18.sh +++ b/testsuite/tests-to-run/parallel-local18.sh @@ -35,11 +35,17 @@ parallel -S localhost --env SPC echo 'a"$SPC"b' ::: 5 parallel -S csh@localhost --env SPC echo 'a"$SPC"b' ::: 5 parallel -S tcsh@localhost --env SPC echo 'a"$SPC"b' ::: 5 -echo '### Test --env for \n and \\ - single and double - no output is good' -perl -e 'for(10,92) { printf "%c%c %c%d\0",$_,$_,$_,$_ }' | stdout parallel --nice 19 -j4 -k -I // --arg-sep _ -0 V=// V2=V2=// parallel -k -j1 -S :,1/lo,1/tcsh@lo,1/csh@lo --env V,V2 echo \''"{}$V$V2"'\' ::: {#} {#} {#} {#} | sort | uniq -c | grep -v ' 4 '|grep -v xauth |grep -v X11 +echo '### Test --env for \n and \\ - single and double (bash only) - no output is good' +perl -e 'for(10,92) { printf "%c%c %c%d\0",$_,$_,$_,$_ }' | stdout parallel --nice 19 -j4 -k -I // --arg-sep _ -0 V=// V2=V2=// parallel -k -j1 -S 2/:,2/lo --env V,V2 echo \''"{}$V$V2"'\' ::: {#} {#} {#} {#} | sort | uniq -c | grep -v ' 4 '|grep -v xauth |grep -v X11 -echo '### Test --env for \n and \\ - single and double --onall - no output is good' -perl -e 'for(10,92) { printf "%c%c %c%d\0",$_,$_,$_,$_ }' | stdout parallel --nice 19 -j4 -k -I // --arg-sep _ -0 V=// V2=V2=// parallel -k -j1 -S :,1/lo,1/tcsh@lo,1/csh@lo --onall --env V,V2 echo \''"{}$V$V2"'\' ::: {#} | sort | uniq -c | grep -v ' 4 '|grep -v xauth |grep -v X11 +echo '### Test --env for \n and \\ - single and double (*csh only) - no output is good' +perl -e 'for(10,92) { printf "%c%c %c%d\0",$_,$_,$_,$_ }' | stdout parallel --nice 19 -j4 -k -I // --arg-sep _ -0 V=// V2=V2=// parallel -k -j1 -S 2/tcsh@lo,2/csh@lo --env V,V2 echo \''"{}$V$V2"'\' ::: {#} {#} {#} {#} | sort | uniq -c | grep -v ' 4 '|grep -v xauth |grep -v X11 + +echo '### Test --env for \n and \\ - single and double --onall (bash only) - no output is good' +perl -e 'for(10,92) { printf "%c%c %c%d\0",$_,$_,$_,$_ }' | stdout parallel --nice 19 -j4 -k -I // --arg-sep _ -0 V=// V2=V2=// parallel -k -j1 -S :,lo --onall --env V,V2 echo \''"{}$V$V2"'\' ::: {#} | sort | uniq -c | grep -v ' 4 '|grep -v xauth |grep -v X11 + +echo '### Test --env for \n and \\ - single and double --onall (*csh only) - no output is good' +perl -e 'for(10,92) { printf "%c%c %c%d\0",$_,$_,$_,$_ }' | stdout parallel --nice 19 -j4 -k -I // --arg-sep _ -0 V=// V2=V2=// parallel -k -j1 -S 1/tcsh@lo,1/csh@lo --onall --env V,V2 echo \''"{}$V$V2"'\' ::: {#} | sort | uniq -c | grep -v ' 2 '|grep -v xauth |grep -v X11 echo '### Test --env for \160 - which kills csh - single and double - no output is good' perl -e 'for(160) { printf "%c%c %c%d\0",$_,$_,$_,$_ }' | stdout parallel --nice 19 -j4 -k -I // --arg-sep _ -0 V=// V2=V2=// parallel -k -j1 -S :,1/lo,1/tcsh@lo --env V,V2 echo \''"{}$V$V2"'\' ::: {#} {#} {#} | uniq -c | grep -v ' 3 '|grep -v xauth |grep -v X11 diff --git a/testsuite/tests-to-run/parallel-polarhome.sh b/testsuite/tests-to-run/parallel-polarhome.sh index f1af6a18..d4a0a55e 100644 --- a/testsuite/tests-to-run/parallel-polarhome.sh +++ b/testsuite/tests-to-run/parallel-polarhome.sh @@ -1,19 +1,29 @@ #!/bin/bash -P="scosysv centos dragonfly netbsd freebsd solaris openbsd debian aix hpux qnx irix suse minix openindiana mandriva unixware miros raspberrypi hurd ultrix ubuntu redhat" -P="scosysv centos dragonfly netbsd freebsd solaris openbsd debian aix hpux qnx irix suse minix openindiana mandriva unixware raspberrypi hurd ultrix ubuntu" -#P="scosysv hpux qnx irix openindiana ultrix" -POLAR=`parallel echo {}.polarhome.com ::: $P` +P_ALL="vax freebsd solaris openbsd netbsd debian alpha aix redhat hpux ultrix qnx irix tru64 openindiana suse openstep mandriva ubuntu scosysv unixware dragonfly centos miros hurd minix raspberrypi" +P_NOTWORKING="vax alpha openstep" +P_NOTWORKING_YET="ultrix tru64" + +P_WORKING="freebsd solaris openbsd netbsd debian aix redhat hpux qnx irix openindiana suse mandriva ubuntu scosysv unixware dragonfly centos miros hurd minix raspberrypi" + +P="$P_WORKING" +POLAR=`parallel -k echo {}.polarhome.com ::: $P` +# Avoid the stupid /etc/issue.net banner at Polarhome echo '### Tests on polarhome machines' echo 'Setup on polarhome machines' -stdout parallel -kj0 ssh {} mkdir -p bin ::: $POLAR >/dev/null 2>/dev/null & -# scp to each polarhome machine do not work. From redhat it works. -stdout rsync -a `which parallel` redhat.polarhome.com:bin/ -stdout ssh redhat.polarhome.com \ - chmod 755 bin/parallel\; \ - bin/parallel --tag -kj0 -v --delay 0.2 ssh {} rm -f bin/parallel\\\;scp bin/parallel {}:bin/ ::: $POLAR | sort +stdout parallel -kj0 ssh -oLogLevel=quiet {} mkdir -p bin ::: $POLAR & +# scp to each polarhome machine do not work. Use cat +copy_to_host() { + H=$1 + # Avoid the stupid /etc/issue.net banner with -oLogLevel=quiet + ssh -oLogLevel=quiet $H rm -f bin/parallel + cat `which parallel` | ssh -oLogLevel=quiet $H 'cat > bin/parallel; chmod 755 bin/parallel' +} +export -f copy_to_host +stdout parallel -j0 --timeout 20 --tag -kj0 -v copy_to_host {} ::: $POLAR # Now test -echo 'Run the test on polarhome machines' -stdout parallel --argsep // -k --tag ssh {} bin/parallel -k echo Works on ::: {} // $POLAR | sort +echo '### Run the test on polarhome machines' +stdout parallel -j0 --argsep // -k --tag ssh -oLogLevel=quiet {} bin/perl bin/parallel -k echo Works on ::: {} // $POLAR + diff --git a/testsuite/wanted-results/parallel-local11 b/testsuite/wanted-results/parallel-local11 index f91003ba..09f68c2f 100644 --- a/testsuite/wanted-results/parallel-local11 +++ b/testsuite/wanted-results/parallel-local11 @@ -1,9 +1,12 @@ ### Test if we can deal with output > 4 GB 46a318993dfc8e2afd71ff2bc6f605f1 - +** ### Test Force outside the file handle limit, 2009-02-17 Gave fork error -parallel: Warning: Only enough filehandles to run 506 jobs in parallel. Raising ulimit -n may help. +parallel: Warning: Only enough file handles to run 506 jobs in parallel. +Raising ulimit -n or /etc/security/limits.conf may help. Start end +** ### Test of --retries on unreachable host ssh: connect to host 4.3.2.1 port 22: Connection timed out parallel: Warning: Could not figure out number of cpus on 4.3.2.1 (). Using 1. @@ -11,6 +14,7 @@ echo 1 1 echo 2 2 +** ### Test race condition on 8 CPU (my laptop) 1 2 @@ -22,14 +26,21 @@ echo 2 8 9 10 +** ### Test exit val - true 0 +** ### Test exit val - false 1 -/bin/bash: non_exist: command not found +** ### Test --halt-on-error 0 1 2 +/bin/bash: non_exist: command not found +** +### Test --halt-on-error 1 +1 +127 parallel: Starting no more jobs. Waiting for 2 jobs to finish. This job failed: sleep 2;false /bin/bash: non_exist: command not found @@ -37,16 +48,17 @@ parallel: Starting no more jobs. Waiting for 3 jobs to finish. This job failed: sleep 2;false parallel: Starting no more jobs. Waiting for 1 jobs to finish. This job failed: sleep 4; non_exist -### Test --halt-on-error 1 -1 -127 -parallel: This job failed: -sleep 2;false -parallel: This job failed: -sleep 2;false +** ### Test --halt-on-error 2 1 1 +parallel: This job failed: +sleep 2;false +parallel: This job failed: +sleep 2;false +** +### Test last dying print --halt-on-error 1 +exit code 9 0 1 2 @@ -76,13 +88,13 @@ parallel: Starting no more jobs. Waiting for 2 jobs to finish. This job failed: perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 8 parallel: Starting no more jobs. Waiting for 1 jobs to finish. This job failed: perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 9 +### Test last dying print --halt-on-error 2 +exit code 1 0 1 parallel: This job failed: perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 1 -### Test last dying print --halt-on-error -9 -1 +** ### Test slow arguments generation - https://savannah.gnu.org/bugs/?32834 1 2 diff --git a/testsuite/wanted-results/parallel-local18 b/testsuite/wanted-results/parallel-local18 index cfe5101e..e62c4171 100644 --- a/testsuite/wanted-results/parallel-local18 +++ b/testsuite/wanted-results/parallel-local18 @@ -24,19 +24,25 @@ a' * ? >o o o o &' at bin/parallel line 1310. -irix.polarhome.com Welcome to irix ...member of polarhome.com realm -mandriva.polarhome.com -mandriva.polarhome.com -mandriva.polarhome.com Mandriva Linux release 2010.2 (Official) for x86_64 -mandriva.polarhome.com Welcome to Mandrake/Mandriva ...member of polarhome.com realm -mandriva.polarhome.com Works on mandriva.polarhome.com -minix.polarhome.com Works on minix.polarhome.com -netbsd.polarhome.com -netbsd.polarhome.com -netbsd.polarhome.com NetBSD 5.1 -netbsd.polarhome.com Welcome to NetBSD ...member of polarhome.com realm -netbsd.polarhome.com Works on netbsd.polarhome.com -openbsd.polarhome.com -openbsd.polarhome.com -openbsd.polarhome.com OpenBSD 4.9 -openbsd.polarhome.com Welcome to OpenBSD ...member of polarhome.com realm +solaris.polarhome.com Works on solaris.polarhome.com openbsd.polarhome.com Works on openbsd.polarhome.com -openindiana.polarhome.com -openindiana.polarhome.com -openindiana.polarhome.com OpenIndiana SunOS 5.11 -openindiana.polarhome.com Welcome to OpenIndiana ...member of polarhome.com realm +netbsd.polarhome.com Works on netbsd.polarhome.com +debian.polarhome.com Works on debian.polarhome.com +aix.polarhome.com Works on aix.polarhome.com +redhat.polarhome.com Works on redhat.polarhome.com +hpux.polarhome.com Works on hpux.polarhome.com +qnx.polarhome.com Works on qnx.polarhome.com +qnx.polarhome.com parallel: Warning: Cannot figure out number of CPU cores. Using 1. +irix.polarhome.com Unknown open() mode '>&=' at bin/parallel line 1316. openindiana.polarhome.com Works on openindiana.polarhome.com openindiana.polarhome.com parallel: Warning: Cannot figure out number of CPU cores. Using 1. -qnx.polarhome.com -qnx.polarhome.com -qnx.polarhome.com QNX 6.5.0 -qnx.polarhome.com Welcome to QNX ...member of polarhome.com realm -qnx.polarhome.com perl: No such file or directory -raspberrypi.polarhome.com -raspberrypi.polarhome.com -raspberrypi.polarhome.com CentOS release 5.6 (Final) -raspberrypi.polarhome.com Welcome to CentOS ...member of polarhome.com realm -raspberrypi.polarhome.com Works on raspberrypi.polarhome.com -scosysv.polarhome.com -scosysv.polarhome.com SCO OpenServer(TM) Release 6 -scosysv.polarhome.com Welcome to scosysv ...member of polarhome.com realm -scosysv.polarhome.com Works on scosysv.polarhome.com -solaris.polarhome.com -solaris.polarhome.com -solaris.polarhome.com SUN Ultra-5 -solaris.polarhome.com SunOS 5.10 -solaris.polarhome.com Welcome to Solaris ...member of polarhome.com realm -solaris.polarhome.com Works on solaris.polarhome.com -suse.polarhome.com -suse.polarhome.com -suse.polarhome.com Welcome to SuSE/openSUSE ...member of polarhome.com realm suse.polarhome.com Works on suse.polarhome.com -suse.polarhome.com openSUSE 11.4 "Celadon" -ubuntu.polarhome.com -ubuntu.polarhome.com -ubuntu.polarhome.com Ubuntu 10.04.2 LTS -ubuntu.polarhome.com Welcome to Ubuntu ...member of polarhome.com realm +mandriva.polarhome.com Works on mandriva.polarhome.com ubuntu.polarhome.com Works on ubuntu.polarhome.com -ultrix.polarhome.com BEGIN failed--compilation aborted at File/Temp.pm line 148. -ultrix.polarhome.com BEGIN failed--compilation aborted at bin/parallel line 28. -ultrix.polarhome.com BEGIN not safe after errors--compilation aborted at Errno.pm line 188. -ultrix.polarhome.com Global symbol "EXPORT_OK" requires explicit package name at Errno.pm line 14. -ultrix.polarhome.com Global symbol "EXPORT_TAGS" requires explicit package name at Errno.pm line 32. -ultrix.polarhome.com Global symbol "ISA" requires explicit package name at Errno.pm line 12. -ultrix.polarhome.com Global symbol "VERSION" requires explicit package name at Errno.pm line 10. -ultrix.polarhome.com Global symbol "VERSION" requires explicit package name at Errno.pm line 11. -ultrix.polarhome.com Global symbol "VERSION" requires explicit package name at Errno.pm line 11. -ultrix.polarhome.com Variable "$VERSION" is not imported at Errno.pm line 10. -ultrix.polarhome.com Variable "$VERSION" is not imported at Errno.pm line 11. -ultrix.polarhome.com Variable "$VERSION" is not imported at Errno.pm line 11. -ultrix.polarhome.com Variable "%EXPORT_TAGS" is not imported at Errno.pm line 32. -ultrix.polarhome.com Variable "@EXPORT_OK" is not imported at Errno.pm line 14. -ultrix.polarhome.com Variable "@ISA" is not imported at Errno.pm line 12. -unixware.polarhome.com -unixware.polarhome.com SCO UnixWare 7.1.4 -unixware.polarhome.com Welcome to unixware ...member of polarhome.com realm +scosysv.polarhome.com Works on scosysv.polarhome.com unixware.polarhome.com Works on unixware.polarhome.com +dragonfly.polarhome.com Works on dragonfly.polarhome.com +centos.polarhome.com Works on centos.polarhome.com +miros.polarhome.com Works on miros.polarhome.com +hurd.polarhome.com Works on hurd.polarhome.com +minix.polarhome.com Works on minix.polarhome.com diff --git a/testsuite/wanted-results/test13 b/testsuite/wanted-results/test13 index 2ae8b3db..97ababb0 100644 --- a/testsuite/wanted-results/test13 +++ b/testsuite/wanted-results/test13 @@ -1,5 +1,6 @@ ### Test -k -parallel: Warning: Only enough filehandles to run 19 jobs in parallel. Raising ulimit -n may help. +parallel: Warning: Only enough file handles to run 19 jobs in parallel. +Raising ulimit -n or /etc/security/limits.conf may help. begin 1 2 @@ -17,6 +18,7 @@ begin 14 15 16 +parallel: Warning: No more file handles. Raising ulimit -n or /etc/security/limits.conf may help. 17 18 19 From ffc14e64ea5d929327dcd5f709453bcd31cde5fd Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Sat, 20 Jul 2013 16:42:02 +0200 Subject: [PATCH 4/7] 10seconds_install: Better support for Polarhome platforms. Accept future signing keyid. --- 10seconds_install | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/10seconds_install b/10seconds_install index e67badb1..7be84658 100644 --- a/10seconds_install +++ b/10seconds_install @@ -14,7 +14,12 @@ # If that fails, it does a personal installation. # If that fails, it copies to $HOME/bin -LATEST=$(wget -qO- http://ftpmirror.gnu.org/parallel | perl -ne '/(parallel-\d{8})/ and print $1."\n"' | sort | tail -n1) +# tail on openindiana must be /usr/xpg4/bin/tail +TAIL=$(echo | tail -n 1 2>/dev/null && echo tail || (echo | /usr/xpg4/bin/tail -n 1 && echo /usr/xpg4/bin/tail)) +# grep on openindiana must be /usr/xpg4/bin/grep +GREP=$(echo | grep -vE . 2>/dev/null && echo grep || (echo | /usr/xpg4/bin/grep -vE . && echo /usr/xpg4/bin/grep)) + +LATEST=$(wget -qO- http://ftpmirror.gnu.org/parallel | perl -ne '/(parallel-\d{8})/ and print $1."\n"' | sort | $TAIL -n1) if test \! -d $LATEST/src/; then # Source dir does not exist rm -f $LATEST.tar.bz2 $LATEST.tar.bz2.sig @@ -27,8 +32,10 @@ if gpg -h 2>/dev/null >/dev/null; then # GnuPG installed # Setup .gnupg/gpg.conf if not already done echo | gpg 2>/dev/null >/dev/null - if gpg --with-fingerprint --auto-key-locate keyserver --keyserver-options auto-key-retrieve $LATEST.tar.bz2.sig 2>&1 | grep -E '^Primary key fingerprint: BE9C B493 81DE 3166 A3BC 66C1 2C62 29E2 FFFF FFF1' ; then - # Source code signed by Ole Tange KeyID FFFFFFF1 + gpg --keyserver keys.gnupg.net --recv-key FFFFFFF1 + gpg --keyserver keys.gnupg.net --recv-key 88888888 + if gpg --with-fingerprint $LATEST.tar.bz2.sig 2>&1 | $GREP -E '^Primary key fingerprint: BE9C B493 81DE 3166 A3BC 66C1 2C62 29E2 FFFF FFF1|^Primary key fingerprint: CDA0 1A42 08C4 F745 0610 7E7B D1AB 4516 8888 8888' ; then + # Source code signed by Ole Tange KeyID FFFFFFF1/88888888 true else # GnuPG signature failed From 2deef988c77e3198f5b28adf40dd6d10249b2354 Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Sat, 20 Jul 2013 17:33:26 +0200 Subject: [PATCH 5/7] Released as 20130720 alpha. --- README | 12 +++--- configure | 20 +++++----- configure.ac | 2 +- doc/release_new_version | 27 +++++++++---- .../obs/home:tange/parallel/parallel.spec | 2 +- src/niceload | 2 +- src/parallel | 2 +- src/parallel.pod | 28 +++++++++---- src/parallel.texi | 39 +++++++++++++------ src/sql | 2 +- 10 files changed, 89 insertions(+), 47 deletions(-) diff --git a/README b/README index 97884824..bfd8bded 100644 --- a/README +++ b/README @@ -40,9 +40,9 @@ document. Full installation of GNU Parallel is as simple as: - wget http://ftpmirror.gnu.org/parallel/parallel-20130630.tar.bz2 - bzip2 -dc parallel-20130630.tar.bz2 | tar xvf - - cd parallel-20130630 + wget http://ftpmirror.gnu.org/parallel/parallel-20130720.tar.bz2 + bzip2 -dc parallel-20130720.tar.bz2 | tar xvf - + cd parallel-20130720 ./configure && make && make install @@ -51,9 +51,9 @@ Full installation of GNU Parallel is as simple as: If you are not root you can add ~/bin to your path and install in ~/bin and ~/share: - wget http://ftpmirror.gnu.org/parallel/parallel-20130630.tar.bz2 - bzip2 -dc parallel-20130630.tar.bz2 | tar xvf - - cd parallel-20130630 + wget http://ftpmirror.gnu.org/parallel/parallel-20130720.tar.bz2 + bzip2 -dc parallel-20130720.tar.bz2 | tar xvf - + cd parallel-20130720 ./configure --prefix=$HOME && make && make install Or if your system lacks 'make' you can simply copy src/parallel diff --git a/configure b/configure index 8118a3d1..47ef7fa3 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.68 for parallel 20130630. +# Generated by GNU Autoconf 2.68 for parallel 20130720. # # Report bugs to . # @@ -559,8 +559,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='parallel' PACKAGE_TARNAME='parallel' -PACKAGE_VERSION='20130630' -PACKAGE_STRING='parallel 20130630' +PACKAGE_VERSION='20130720' +PACKAGE_STRING='parallel 20130720' PACKAGE_BUGREPORT='bug-parallel@gnu.org' PACKAGE_URL='' @@ -1176,7 +1176,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures parallel 20130630 to adapt to many kinds of systems. +\`configure' configures parallel 20130720 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1242,7 +1242,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of parallel 20130630:";; + short | recursive ) echo "Configuration of parallel 20130720:";; esac cat <<\_ACEOF @@ -1309,7 +1309,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -parallel configure 20130630 +parallel configure 20130720 generated by GNU Autoconf 2.68 Copyright (C) 2010 Free Software Foundation, Inc. @@ -1326,7 +1326,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by parallel $as_me 20130630, which was +It was created by parallel $as_me 20130720, which was generated by GNU Autoconf 2.68. Invocation command line was $ $0 $@ @@ -2141,7 +2141,7 @@ fi # Define the identity of the package. PACKAGE='parallel' - VERSION='20130630' + VERSION='20130720' cat >>confdefs.h <<_ACEOF @@ -2704,7 +2704,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by parallel $as_me 20130630, which was +This file was extended by parallel $as_me 20130720, which was generated by GNU Autoconf 2.68. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -2766,7 +2766,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -parallel config.status 20130630 +parallel config.status 20130720 configured by $0, generated by GNU Autoconf 2.68, with options \\"\$ac_cs_config\\" diff --git a/configure.ac b/configure.ac index 14ea88e5..087ffe29 100644 --- a/configure.ac +++ b/configure.ac @@ -1,4 +1,4 @@ -AC_INIT([parallel], [20130630], [bug-parallel@gnu.org]) +AC_INIT([parallel], [20130720], [bug-parallel@gnu.org]) AM_INIT_AUTOMAKE([-Wall -Werror foreign]) AC_CONFIG_HEADERS([config.h]) AC_CONFIG_FILES([ diff --git a/doc/release_new_version b/doc/release_new_version index 759e8d11..b00494f0 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -94,6 +94,8 @@ gpg --clearsign --yes parallel-latest.tar.bz2.directive gpg --auto-key-locate keyserver --keyserver-options auto-key-retrieve parallel-latest.tar.bz2.sig gpg --auto-key-locate keyserver --keyserver-options auto-key-retrieve parallel-$YYYYMMDD.tar.bz2.sig +#../ftpsync/src/ftpsync parallel-$YYYYMMDD.tar.bz2{,.sig,*asc} ftp://ftp-upload.gnu.org/incoming/alpha/ + echo put parallel-$YYYYMMDD.tar.bz2{,.sig,*asc} | ncftp ftp://ftp-upload.gnu.org/incoming/ftp/ echo put parallel-latest.tar.bz2{,.sig,*asc} | ncftp ftp://ftp-upload.gnu.org/incoming/ftp/ #echo put parallel-$YYYYMMDD.tar.bz2{,.sig,*asc} | ncftp ftp://ftp-upload.gnu.org/incoming/alpha/ @@ -200,10 +202,17 @@ Subject: GNU Parallel 20130722 ('Engelbart') released GNU Parallel 20130722 ('Engelbart') has been released. It is available for download at: http://ftp.gnu.org/gnu/parallel/ -Very few changes so this can be considered a stable release. - New in this release: +* --round-robin with --pipe will write all blocks to already running + jobs. + +* --env can now transfer Bash function for remote execution. + +* New signing key. Due to recommendations from NIST + http://www.keylength.com/en/4/ the signing key was changed from + 1024D/4000g/ID:FFFFFFF1 to 9888R/ID:88888888. + * GNU Parallel was used (unfortunately with improper citation) in: Understanding the Impact of E-Commerce Software on the Adoption of Structured Data on the Web http://link.springer.com/chapter/10.1007/978-3-642-38366-3_9#page-1 @@ -212,16 +221,20 @@ New in this release: CWI at TREC 2012, KBA track and Session Track http://trec.nist.gov/pubs/trec21/papers/CWI.kba.session.final.pdf -* Mitigation of Adverse Effects Caused by Shock Wave Boundary Layer Interactions through Optimal Wall Shaping +* Mitigation of Adverse Effects Caused by Shock Wave Boundary Layer + Interactions through Optimal Wall Shaping. http://arc.aiaa.org/doi/abs/10.2514/6.2013-2653 -* http://www.brunokim.com.br/blog/?p=18 +* Using GNU parallel to convert images. + http://www.brunokim.com.br/blog/?p=18 -* http://timotheepoisot.fr/2013/07/08/parallel/ +* A quick way to parallelize. + http://timotheepoisot.fr/2013/07/08/parallel/ -* http://www.open-open.com/news/view/371301 +* GNU Parallel 20130522 ('Rana Plaza') 发布，并行作业执行. + http://www.open-open.com/news/view/371301 -* Bug fixes and man page updates. +* Quite a few bug fixes and man page updates. = About GNU Parallel = diff --git a/packager/obs/home:tange/parallel/parallel.spec b/packager/obs/home:tange/parallel/parallel.spec index 1a37c0a1..aa862f6d 100644 --- a/packager/obs/home:tange/parallel/parallel.spec +++ b/packager/obs/home:tange/parallel/parallel.spec @@ -1,6 +1,6 @@ Summary: Shell tool for executing jobs in parallel Name: parallel -Version: 20130622 +Version: 20130720 Release: 1 License: GPL Group: Productivity/File utilities diff --git a/src/niceload b/src/niceload index b60b567d..dbea2e6e 100755 --- a/src/niceload +++ b/src/niceload @@ -24,7 +24,7 @@ use strict; use Getopt::Long; $Global::progname="niceload"; -$Global::version = 20130630; +$Global::version = 20130720; Getopt::Long::Configure("bundling","require_order"); get_options_from_array(\@ARGV) || die_usage(); if($opt::version) { diff --git a/src/parallel b/src/parallel index 2d2d8dbb..602c1bb4 100755 --- a/src/parallel +++ b/src/parallel @@ -725,7 +725,7 @@ sub get_options_from_array { sub parse_options { # Returns: N/A # Defaults: - $Global::version = 20130630; + $Global::version = 20130720; $Global::progname = 'parallel'; $Global::infinity = 2**31; $Global::debug = 0; diff --git a/src/parallel.pod b/src/parallel.pod index 5afae17e..22cafbc4 100644 --- a/src/parallel.pod +++ b/src/parallel.pod @@ -397,7 +397,7 @@ I is a Perl Regular Expression: http://perldoc.perl.org/perlre.html -=item B<--ctrlc> (beta testing) +=item B<--ctrlc> Sends SIGINT to tasks running on remote computers thus killing them. @@ -458,13 +458,14 @@ If I is omitted, there is no end of file string. If neither B<-E> nor B<-e> is used, no end of file string is used. -=item B<--env> I +=item B<--env> I (alpha testing) Copy environment variable I. This will copy I to the environment that the command is run in. This is especially useful for -remote environments. +remote execution. -Caveat: If I contains newline ('\n') the value is messed up. +In Bash I can also be a Bash function - just remember to B the function. =item B<--eta> @@ -485,7 +486,7 @@ See also: B<--bg>, B Implies B<--semaphore>. -=item B<--filter-hosts> (alpha testing) +=item B<--filter-hosts> (beta testing) Remove down hosts. For each remote host: check that login through ssh works. If not: do not use this host. @@ -1100,6 +1101,19 @@ B<--return> is ignored when used with B<--sshlogin :> or when not used with B<--sshlogin>. +=item B<--round-robin> (alpha testing) + +=item B<--round> (alpha testing) + +Normally B<--pipe> will give a single block to each instance of the +command. With B<--round-robin> all blocks will at random be written to +commands already running. This is useful if the command takes a long +time to initialize. + +B<--keep-order> will not work with B<--round-robin> as it is +impossible to track which input block corresponds to which output. + + =item B<--max-chars>=I =item B<-s> I @@ -1370,7 +1384,7 @@ different dir for the files. Setting B<--tmpdir> is equivalent to setting $TMPDIR. -=item B<--timeout> I (beta testing) +=item B<--timeout> I Time out for command. If the command runs for longer than I seconds it will get killed with SIGTERM, followed by SIGTERM 200 ms @@ -1579,7 +1593,7 @@ See also B<-m>. Exit if the size (see the B<-s> option) is exceeded. -=item B<--xapply> (alpha testing) +=item B<--xapply> (beta testing) Read multiple input sources like B. If multiple input sources are given, one argument will be read from each of the input diff --git a/src/parallel.texi b/src/parallel.texi index 798e035a..63458aeb 100644 --- a/src/parallel.texi +++ b/src/parallel.texi @@ -420,8 +420,8 @@ separating the columns. The n'th column can be access using @emph{regexp} is a Perl Regular Expression: http://perldoc.perl.org/perlre.html -@item @strong{--ctrlc} (beta testing) -@anchor{@strong{--ctrlc} (beta testing)} +@item @strong{--ctrlc} +@anchor{@strong{--ctrlc}} Sends SIGINT to tasks running on remote computers thus killing them. @@ -485,14 +485,15 @@ because it is POSIX compliant for @strong{xargs} while this option is not. If @emph{eof-str} is omitted, there is no end of file string. If neither @strong{-E} nor @strong{-e} is used, no end of file string is used. -@item @strong{--env} @emph{var} -@anchor{@strong{--env} @emph{var}} +@item @strong{--env} @emph{var} (alpha testing) +@anchor{@strong{--env} @emph{var} (alpha testing)} Copy environment variable @emph{var}. This will copy @emph{var} to the environment that the command is run in. This is especially useful for -remote environments. +remote execution. -Caveat: If @emph{var} contains newline ('\n') the value is messed up. +In Bash @emph{var} can also be a Bash function - just remember to @strong{export +-f} the function. @item @strong{--eta} @anchor{@strong{--eta}} @@ -512,8 +513,8 @@ See also: @strong{--bg}, @strong{man sem} Implies @strong{--semaphore}. -@item @strong{--filter-hosts} (alpha testing) -@anchor{@strong{--filter-hosts} (alpha testing)} +@item @strong{--filter-hosts} (beta testing) +@anchor{@strong{--filter-hosts} (beta testing)} Remove down hosts. For each remote host: check that login through ssh works. If not: do not use this host. @@ -1180,6 +1181,20 @@ times: @strong{--return} is ignored when used with @strong{--sshlogin :} or when not used with @strong{--sshlogin}. +@item @strong{--round-robin} (alpha testing) +@anchor{@strong{--round-robin} (alpha testing)} + +@item @strong{--round} (alpha testing) +@anchor{@strong{--round} (alpha testing)} + +Normally @strong{--pipe} will give a single block to each instance of the +command. With @strong{--round-robin} all blocks will at random be written to +commands already running. This is useful if the command takes a long +time to initialize. + +@strong{--keep-order} will not work with @strong{--round-robin} as it is +impossible to track which input block corresponds to which output. + @item @strong{--max-chars}=@emph{max-chars} @anchor{@strong{--max-chars}=@emph{max-chars}} @@ -1473,8 +1488,8 @@ into temporary files in /tmp. By setting @strong{--tmpdir} you can use a different dir for the files. Setting @strong{--tmpdir} is equivalent to setting $TMPDIR. -@item @strong{--timeout} @emph{val} (beta testing) -@anchor{@strong{--timeout} @emph{val} (beta testing)} +@item @strong{--timeout} @emph{val} +@anchor{@strong{--timeout} @emph{val}} Time out for command. If the command runs for longer than @emph{val} seconds it will get killed with SIGTERM, followed by SIGTERM 200 ms @@ -1695,8 +1710,8 @@ See also @strong{-m}. Exit if the size (see the @strong{-s} option) is exceeded. -@item @strong{--xapply} (alpha testing) -@anchor{@strong{--xapply} (alpha testing)} +@item @strong{--xapply} (beta testing) +@anchor{@strong{--xapply} (beta testing)} Read multiple input sources like @strong{xapply}. If multiple input sources are given, one argument will be read from each of the input diff --git a/src/sql b/src/sql index 2bd7cef1..51ed4a98 100755 --- a/src/sql +++ b/src/sql @@ -556,7 +556,7 @@ $Global::Initfile && unlink $Global::Initfile; exit ($err); sub parse_options { - $Global::version = 20130630; + $Global::version = 20130720; $Global::progname = 'sql'; # This must be done first as this may exec myself From 7b9928c32a5e12e5d8bcbb22719cbdfccb355ea1 Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Mon, 22 Jul 2013 12:54:02 +0200 Subject: [PATCH 6/7] Released as 20130722 ('Engelbart'). --- NEWS | 32 +++++++++++++++++++ README | 12 +++---- configure | 20 ++++++------ configure.ac | 2 +- doc/release_new_version | 4 ++- .../obs/home:tange/parallel/parallel.spec | 2 +- src/niceload | 2 +- src/parallel | 2 +- src/sql | 2 +- 9 files changed, 56 insertions(+), 22 deletions(-) diff --git a/NEWS b/NEWS index 51e794e8..02ac3ff9 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,35 @@ +20130722 + +* --round-robin with --pipe will write all blocks to already running + jobs. + +* --env can now transfer Bash function for remote execution. That is + pretty cool! + +* GNU Parallel was used (unfortunately with improper citation) in: + Understanding the Impact of E-Commerce Software on the Adoption of Structured Data on the Web + http://link.springer.com/chapter/10.1007/978-3-642-38366-3_9#page-1 + +* GNU Parallel was used (unfortunately with improper citation) in: + CWI at TREC 2012, KBA track and Session Track + http://trec.nist.gov/pubs/trec21/papers/CWI.kba.session.final.pdf + +* Mitigation of Adverse Effects Caused by Shock Wave Boundary Layer + Interactions through Optimal Wall Shaping. + http://arc.aiaa.org/doi/abs/10.2514/6.2013-2653 + +* Using GNU parallel to convert images. + http://www.brunokim.com.br/blog/?p=18 + +* A quick way to parallelize. + http://timotheepoisot.fr/2013/07/08/parallel/ + +* GNU Parallel 20130522 ('Rana Plaza') 发布，并行作业执行. + http://www.open-open.com/news/view/371301 + +* Quite a few bug fixes and man page updates. + + 20130622 * --xapply now recycles arguments if an input source has more diff --git a/README b/README index bfd8bded..93839a05 100644 --- a/README +++ b/README @@ -40,9 +40,9 @@ document. Full installation of GNU Parallel is as simple as: - wget http://ftpmirror.gnu.org/parallel/parallel-20130720.tar.bz2 - bzip2 -dc parallel-20130720.tar.bz2 | tar xvf - - cd parallel-20130720 + wget http://ftpmirror.gnu.org/parallel/parallel-20130722.tar.bz2 + bzip2 -dc parallel-20130722.tar.bz2 | tar xvf - + cd parallel-20130722 ./configure && make && make install @@ -51,9 +51,9 @@ Full installation of GNU Parallel is as simple as: If you are not root you can add ~/bin to your path and install in ~/bin and ~/share: - wget http://ftpmirror.gnu.org/parallel/parallel-20130720.tar.bz2 - bzip2 -dc parallel-20130720.tar.bz2 | tar xvf - - cd parallel-20130720 + wget http://ftpmirror.gnu.org/parallel/parallel-20130722.tar.bz2 + bzip2 -dc parallel-20130722.tar.bz2 | tar xvf - + cd parallel-20130722 ./configure --prefix=$HOME && make && make install Or if your system lacks 'make' you can simply copy src/parallel diff --git a/configure b/configure index 47ef7fa3..ce44af21 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.68 for parallel 20130720. +# Generated by GNU Autoconf 2.68 for parallel 20130722. # # Report bugs to . # @@ -559,8 +559,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='parallel' PACKAGE_TARNAME='parallel' -PACKAGE_VERSION='20130720' -PACKAGE_STRING='parallel 20130720' +PACKAGE_VERSION='20130722' +PACKAGE_STRING='parallel 20130722' PACKAGE_BUGREPORT='bug-parallel@gnu.org' PACKAGE_URL='' @@ -1176,7 +1176,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures parallel 20130720 to adapt to many kinds of systems. +\`configure' configures parallel 20130722 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1242,7 +1242,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of parallel 20130720:";; + short | recursive ) echo "Configuration of parallel 20130722:";; esac cat <<\_ACEOF @@ -1309,7 +1309,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -parallel configure 20130720 +parallel configure 20130722 generated by GNU Autoconf 2.68 Copyright (C) 2010 Free Software Foundation, Inc. @@ -1326,7 +1326,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by parallel $as_me 20130720, which was +It was created by parallel $as_me 20130722, which was generated by GNU Autoconf 2.68. Invocation command line was $ $0 $@ @@ -2141,7 +2141,7 @@ fi # Define the identity of the package. PACKAGE='parallel' - VERSION='20130720' + VERSION='20130722' cat >>confdefs.h <<_ACEOF @@ -2704,7 +2704,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by parallel $as_me 20130720, which was +This file was extended by parallel $as_me 20130722, which was generated by GNU Autoconf 2.68. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -2766,7 +2766,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -parallel config.status 20130720 +parallel config.status 20130722 configured by $0, generated by GNU Autoconf 2.68, with options \\"\$ac_cs_config\\" diff --git a/configure.ac b/configure.ac index 087ffe29..6543babf 100644 --- a/configure.ac +++ b/configure.ac @@ -1,4 +1,4 @@ -AC_INIT([parallel], [20130720], [bug-parallel@gnu.org]) +AC_INIT([parallel], [20130722], [bug-parallel@gnu.org]) AM_INIT_AUTOMAKE([-Wall -Werror foreign]) AC_CONFIG_HEADERS([config.h]) AC_CONFIG_FILES([ diff --git a/doc/release_new_version b/doc/release_new_version index b00494f0..94f72ce4 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -127,6 +127,7 @@ cd parallel-$YYYYMMDD ./configure make -j && sudo make -j install pushd +sudo cp /usr/local/bin/parallel /usr/local/bin/parallel-$YYYYMMDD == Update website == @@ -207,7 +208,8 @@ New in this release: * --round-robin with --pipe will write all blocks to already running jobs. -* --env can now transfer Bash function for remote execution. +* --env can now transfer Bash function for remote execution. That is + pretty cool! * New signing key. Due to recommendations from NIST http://www.keylength.com/en/4/ the signing key was changed from diff --git a/packager/obs/home:tange/parallel/parallel.spec b/packager/obs/home:tange/parallel/parallel.spec index aa862f6d..1f533e52 100644 --- a/packager/obs/home:tange/parallel/parallel.spec +++ b/packager/obs/home:tange/parallel/parallel.spec @@ -1,6 +1,6 @@ Summary: Shell tool for executing jobs in parallel Name: parallel -Version: 20130720 +Version: 20130722 Release: 1 License: GPL Group: Productivity/File utilities diff --git a/src/niceload b/src/niceload index dbea2e6e..e3aceeaf 100755 --- a/src/niceload +++ b/src/niceload @@ -24,7 +24,7 @@ use strict; use Getopt::Long; $Global::progname="niceload"; -$Global::version = 20130720; +$Global::version = 20130722; Getopt::Long::Configure("bundling","require_order"); get_options_from_array(\@ARGV) || die_usage(); if($opt::version) { diff --git a/src/parallel b/src/parallel index 602c1bb4..e322acfa 100755 --- a/src/parallel +++ b/src/parallel @@ -725,7 +725,7 @@ sub get_options_from_array { sub parse_options { # Returns: N/A # Defaults: - $Global::version = 20130720; + $Global::version = 20130722; $Global::progname = 'parallel'; $Global::infinity = 2**31; $Global::debug = 0; diff --git a/src/sql b/src/sql index 51ed4a98..2ff22aed 100755 --- a/src/sql +++ b/src/sql @@ -556,7 +556,7 @@ $Global::Initfile && unlink $Global::Initfile; exit ($err); sub parse_options { - $Global::version = 20130720; + $Global::version = 20130722; $Global::progname = 'sql'; # This must be done first as this may exec myself From f58db1e3013d99516b2de835537e57552dd2d5a0 Mon Sep 17 00:00:00 2001 From: Tim Cuthbertson Date: Tue, 23 Jul 2013 16:36:24 +1000 Subject: [PATCH 7/7] 0install: added version 20130722 --- packager/0install/parallel.xml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/packager/0install/parallel.xml b/packager/0install/parallel.xml index 6f2b3eeb..3ae3d108 100644 --- a/packager/0install/parallel.xml +++ b/packager/0install/parallel.xml @@ -155,11 +155,15 @@ xargs or cat | sh. + + + +