From 8110572719461a6019b8f4c22d5ebf43f65cfbe6 Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Mon, 27 Apr 2015 22:56:26 +0200 Subject: [PATCH 01/14] parallel: --halt when,why,num. --- src/parallel | 145 ++++++++++++++------- testsuite/Start.sh | 23 ++-- testsuite/wanted-results/parallel-local-3s | 62 ++++++--- 3 files changed, 148 insertions(+), 82 deletions(-) diff --git a/src/parallel b/src/parallel index 3fb02e0c..9a71f93c 100755 --- a/src/parallel +++ b/src/parallel @@ -161,7 +161,7 @@ for(keys %Global::sshmaster) { kill "TERM", $_; } ::debug("init", "Halt\n"); -if($opt::halt) { +if($opt::halt and $Global::halt_when ne "never") { wait_and_exit($Global::halt_exitstatus); } else { wait_and_exit(min(undef_as_zero($Global::exitstatus),254)); @@ -920,8 +920,6 @@ sub parse_options { } $opt::memfree = multiply_binary_prefix($opt::memfree); if(defined $opt::controlmaster) { $opt::noctrlc = 1; } - if(defined $opt::halt and - $opt::halt =~ /%/) { $opt::halt /= 100; } if(defined $opt::timeout and $opt::timeout !~ /^\d+(\.\d+)?%?$/) { ::error("--timeout must be seconds or percentage\n"); wait_and_exit(255); @@ -1036,6 +1034,7 @@ sub parse_options { } citation_notice(); + parse_halt(); parse_sshlogin(); parse_env_var(); @@ -1108,6 +1107,53 @@ sub init_globals { } } +sub parse_halt { + # $opt::halt flavours + # Uses: + # $opt::halt + # $Global::halt_when + # $Global::halt_fail + # $Global::halt_success + # $Global::halt_pct + # $Global::halt_count + if(defined $opt::halt) { + my %halt_expansion = ( + "0" => "never", + "1" => "soon,fail=1", + "2" => "now,fail=1", + "-1" => "soon,success=1", + "-2" => "now,success=1", + ); + # Expand -2,-1,0,1,2 into long form + $opt::halt = $halt_expansion{$opt::halt} || $opt::halt; + # --halt 5% == --halt soon,fail=5% + $opt::halt =~ s/^(\d+)%$/soon,fail=$1%/; + # Split: soon,fail=5% + my ($when,$fail_success,$pct_count) = split /[,=]/, $opt::halt; + if(not grep { $when eq $_ } qw(never soon now)) { + ::error("--halt must have 'never', 'soon', or 'now'"); + } + $Global::halt_when = $when; + if($when ne "never") { + if($fail_success eq "fail") { + $Global::halt_fail = 1; + } elsif($fail_success eq "success") { + $Global::halt_success = 1; + } else { + ::error("--halt $when must be followed by ,success or ,fail\n"); + } + if($pct_count =~ /^(\d+)%$/) { + $Global::halt_pct = $1/100; + } elsif($pct_count =~ /^(\d+)$/) { + $Global::halt_count = $1; + } else { + ::error("--halt $when,$fail_success ", + "must be followed by ,number or ,percent%\n"); + } + } + } +} + sub parse_replacement_string_options { # Deal with --rpl # Uses: @@ -1738,6 +1784,7 @@ sub init_run_jobs { # Returns: N/A $Global::total_running = 0; $Global::total_started = 0; + $Global::total_completed = 0; $Global::tty_taken = 0; $SIG{USR1} = \&list_running_jobs; $SIG{USR2} = \&toggle_progress; @@ -2229,8 +2276,7 @@ sub progress { # $avgtime = averaged time # $eta = smoothed eta $total ||= $Global::JobQueue->total_jobs(); - my $completed = 0; - for(values %Global::host) { $completed += $_->jobs_completed() } + my $completed = $Global::total_completed; my $left = $total - $completed; if(not $completed) { return($total, $completed, $left, 0, 0, 0); @@ -3005,18 +3051,15 @@ sub reaper { # Update average runtime for timeout $Global::timeoutq->update_median_runtime($job->runtime()); } - # Force printing now if --halt forces us to exit - my $print_now = - ($opt::halt and - (($opt::halt == 2 and $job->exitstatus()) - or - ($opt::halt == -2 and not $job->exitstatus()))); - if($opt::keeporder and not $print_now) { + if($opt::keeporder) { $job->print_earlier_jobs(); } else { - $job->print(); + $job->print(); + } + if($job->should_we_halt() eq "now") { + ::killall(); + ::wait_and_exit($Global::exitstatus); } - $job->should_we_halt(); } my $sshlogin = $job->sshlogin(); $sshlogin->dec_jobs_running(); @@ -3974,6 +4017,7 @@ sub hostgroups { sub inc_jobs_completed { my $self = shift; $self->{'jobs_completed'}++; + $Global::total_completed++; } sub set_max_jobs_running { @@ -7433,56 +7477,57 @@ sub set_exitsignal { # Returns: N/A my $job = shift; if($job->exitstatus() or $job->exitsignal()) { + # Job failed $Global::exitstatus++; $Global::total_failed++; - if($opt::halt) { - if($opt::halt == 1 - or - ($opt::halt > 0 and $opt::halt < 1 and $Global::total_failed > 3 - and - $Global::total_failed / $Global::total_started > $opt::halt)) { - # If halt on error == 1 or --halt 10% - # we should gracefully exit - ::status - ("$Global::progname: Starting no more jobs. ", - "Waiting for ", scalar(keys %Global::running), - " jobs to finish. This job failed:\n", + if($Global::halt_fail) { + ::status("$Global::progname: This job failed:\n", $job->replaced(),"\n"); - $Global::start_no_new_jobs ||= 1; + if(($Global::halt_count and + $Global::halt_count <= $Global::total_failed) + or + ($Global::halt_pct and + $Global::halt_pct <= + $Global::total_failed / $Global::total_started + and $Global::total_failed > 3)) { + # More than N jobs or more than N% failed $Global::halt_exitstatus = $job->exitstatus(); - } elsif($opt::halt == 2) { - # If halt on error == 2 we should exit immediately - if(not $status_printed++) { + if($Global::halt_when eq "soon") { ::status - ("$Global::progname: This job failed:\n", - $job->replaced(),"\n"); + ("$Global::progname: Starting no more jobs. ", + "Waiting for ", scalar(keys %Global::running), + " jobs to finish.\n"); + $Global::start_no_new_jobs ||= 1; } - ::killall(); - ::wait_and_exit($job->exitstatus()); + return($Global::halt_when); } } } else { - if($opt::halt) { - if($opt::halt == -1) { - # If halt on error == -1 - # we should gracefully exit - ::status - ("$Global::progname: Starting no more jobs. ", - "Waiting for ", scalar(keys %Global::running), - " jobs to finish. This job succeeded:\n", + if($Global::halt_success) { + ::status("$Global::progname: This job succeeded:\n", $job->replaced(),"\n"); - $Global::start_no_new_jobs ||= 1; + if(($Global::halt_count and + $Global::halt_count <= + 1+$Global::total_completed-$Global::total_failed) + or + ($Global::halt_pct and + $Global::halt_pct <= + (1+$Global::total_completed-$Global::total_failed) + / $Global::total_completed + and ($Global::total_completed-$Global::total_failed) > 3)) { $Global::halt_exitstatus = $job->exitstatus(); - } elsif($opt::halt == -2) { - # If halt on error == -2 we should exit immediately - ::status - ("$Global::progname: This job succeeded:\n", - $job->replaced(),"\n"); - ::killall(); - ::wait_and_exit($job->exitstatus()); + if($Global::halt_when eq "soon") { + ::status + ("$Global::progname: Starting no more jobs. ", + "Waiting for ", scalar(keys %Global::running), + " jobs to finish.\n"); + $Global::start_no_new_jobs ||= 1; + } + return($Global::halt_when); } } } + return ""; } } diff --git a/testsuite/Start.sh b/testsuite/Start.sh index 83d3fd49..d09c2341 100644 --- a/testsuite/Start.sh +++ b/testsuite/Start.sh @@ -10,21 +10,22 @@ export TIMEOUT=$MAX_SEC_PER_TEST run_test() { script="$1" base=`basename "$script" .sh` - export TMPDIR=/tmp/$base + export TMPDIR=/tmp/"$base" + mkdir -p "$TMPDIR" if [ "$TRIES" = "3" ] ; then # Try 3 times - bash $script > actual-results/$base - diff -Naur wanted-results/$base actual-results/$base >/dev/null || - bash $script > actual-results/$base - diff -Naur wanted-results/$base actual-results/$base >/dev/null || - bash $script > actual-results/$base - diff -Naur wanted-results/$base actual-results/$base || - (touch $script && echo touch $script) + bash "$script" > actual-results/"$base" + diff -Naur wanted-results/"$base" actual-results/"$base" >/dev/null || + bash "$script" > actual-results/"$base" + diff -Naur wanted-results/"$base" actual-results/"$base" >/dev/null || + bash "$script" > actual-results/"$base" + diff -Naur wanted-results/"$base" actual-results/"$base" || + (touch "$script" && echo touch "$script") else # Run only once - bash $script > actual-results/$base - diff -Naur wanted-results/$base actual-results/$base || - (touch $script && echo touch $script) + bash "$script" > actual-results/"$base" + diff -Naur wanted-results/"$base" actual-results/"$base" || + (touch "$script" && echo touch "$script") fi # Check if it was cleaned up diff --git a/testsuite/wanted-results/parallel-local-3s b/testsuite/wanted-results/parallel-local-3s index 02242a1f..4a6ec714 100644 --- a/testsuite/wanted-results/parallel-local-3s +++ b/testsuite/wanted-results/parallel-local-3s @@ -17,13 +17,14 @@ echo '### Test --halt-on-error 1'; (echo "sleep 1;true"; echo "sleep 2;false"; ### Test --halt-on-error 1 1 127 -parallel: Starting no more jobs. Waiting for 2 jobs to finish. This job failed: +parallel: This job failed: sleep 2;false parallel: Starting no more jobs. Waiting for 3 jobs to finish. This job failed: sleep 2;false /bin/bash: non_exist: command not found -parallel: Starting no more jobs. Waiting for 1 jobs to finish. This job failed: +parallel: This job failed: sleep 4; non_exist +parallel: Starting no more jobs. Waiting for 1 jobs to finish. echo '**' ** echo '### Test --halt-on-error 2'; (echo "sleep 1;true"; echo "sleep 2;false";echo "sleep 3;true") | parallel -j10 --halt-on-error 2; echo $?; (echo "sleep 1;true"; echo "sleep 2;false";echo "sleep 3;true";echo "sleep 4; non_exist") | parallel -j10 --halt 2; echo $? @@ -40,10 +41,11 @@ echo '### Test --halt -1'; (echo "sleep 1;false"; echo "sleep 2;true";echo "sl ### Test --halt -1 0 0 -parallel: Starting no more jobs. Waiting for 2 jobs to finish. This job succeeded: +parallel: This job succeeded: sleep 2;true parallel: Starting no more jobs. Waiting for 3 jobs to finish. This job succeeded: sleep 2;true +parallel: Starting no more jobs. Waiting for 3 jobs to finish. /bin/bash: non_exist: command not found echo '**' ** @@ -62,33 +64,42 @@ echo '### Test last dying print --halt-on-error 1'; (seq 0 8;echo 0; echo 9) | exit code 9 0 1 -parallel: Starting no more jobs. Waiting for 9 jobs to finish. This job failed: +parallel: This job failed: perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 1 +parallel: Starting no more jobs. Waiting for 9 jobs to finish. 2 -parallel: Starting no more jobs. Waiting for 8 jobs to finish. This job failed: +parallel: This job failed: perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 2 +parallel: Starting no more jobs. Waiting for 8 jobs to finish. 3 -parallel: Starting no more jobs. Waiting for 7 jobs to finish. This job failed: +parallel: This job failed: perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 3 +parallel: Starting no more jobs. Waiting for 7 jobs to finish. 4 -parallel: Starting no more jobs. Waiting for 6 jobs to finish. This job failed: +parallel: This job failed: perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 4 +parallel: Starting no more jobs. Waiting for 6 jobs to finish. 5 -parallel: Starting no more jobs. Waiting for 5 jobs to finish. This job failed: +parallel: This job failed: perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 5 +parallel: Starting no more jobs. Waiting for 5 jobs to finish. 6 -parallel: Starting no more jobs. Waiting for 4 jobs to finish. This job failed: +parallel: This job failed: perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 6 +parallel: Starting no more jobs. Waiting for 4 jobs to finish. 7 -parallel: Starting no more jobs. Waiting for 3 jobs to finish. This job failed: +parallel: This job failed: perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 7 +parallel: Starting no more jobs. Waiting for 3 jobs to finish. 8 0 -parallel: Starting no more jobs. Waiting for 2 jobs to finish. This job failed: +parallel: This job failed: perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 8 +parallel: Starting no more jobs. Waiting for 2 jobs to finish. 9 -parallel: Starting no more jobs. Waiting for 1 jobs to finish. This job failed: +parallel: This job failed: perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 9 +parallel: Starting no more jobs. Waiting for 1 jobs to finish. echo '### Test last dying print --halt-on-error 2'; (seq 0 8;echo 0; echo 9) | parallel -j10 -kq --halt 2 perl -e 'sleep $ARGV[0];print STDERR @ARGV,"\n"; exit shift'; echo exit code $? ### Test last dying print --halt-on-error 2 exit code 1 @@ -101,33 +112,42 @@ echo '### Test last dying print --halt-on-error -1'; (seq 0 8;echo 0; echo 9) exit code 0 0 1 -parallel: Starting no more jobs. Waiting for 9 jobs to finish. This job succeeded: +parallel: This job succeeded: perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ not\ shift 1 +parallel: Starting no more jobs. Waiting for 9 jobs to finish. 2 -parallel: Starting no more jobs. Waiting for 8 jobs to finish. This job succeeded: +parallel: This job succeeded: perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ not\ shift 2 +parallel: Starting no more jobs. Waiting for 8 jobs to finish. 3 -parallel: Starting no more jobs. Waiting for 7 jobs to finish. This job succeeded: +parallel: This job succeeded: perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ not\ shift 3 +parallel: Starting no more jobs. Waiting for 7 jobs to finish. 4 -parallel: Starting no more jobs. Waiting for 6 jobs to finish. This job succeeded: +parallel: This job succeeded: perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ not\ shift 4 +parallel: Starting no more jobs. Waiting for 6 jobs to finish. 5 -parallel: Starting no more jobs. Waiting for 5 jobs to finish. This job succeeded: +parallel: This job succeeded: perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ not\ shift 5 +parallel: Starting no more jobs. Waiting for 5 jobs to finish. 6 -parallel: Starting no more jobs. Waiting for 4 jobs to finish. This job succeeded: +parallel: This job succeeded: perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ not\ shift 6 +parallel: Starting no more jobs. Waiting for 4 jobs to finish. 7 -parallel: Starting no more jobs. Waiting for 3 jobs to finish. This job succeeded: +parallel: This job succeeded: perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ not\ shift 7 +parallel: Starting no more jobs. Waiting for 3 jobs to finish. 8 0 -parallel: Starting no more jobs. Waiting for 2 jobs to finish. This job succeeded: +parallel: This job succeeded: perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ not\ shift 8 +parallel: Starting no more jobs. Waiting for 2 jobs to finish. 9 -parallel: Starting no more jobs. Waiting for 1 jobs to finish. This job succeeded: +parallel: This job succeeded: perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ not\ shift 9 +parallel: Starting no more jobs. Waiting for 1 jobs to finish. echo '### Test last dying print --halt-on-error -2'; (seq 0 8;echo 0; echo 9) | parallel -j10 -kq --halt -2 perl -e 'sleep $ARGV[0];print STDERR @ARGV,"\n"; exit not shift'; echo exit code $? ### Test last dying print --halt-on-error -2 exit code 0 From fc0c6cee08b10f2a331247bb912e7eb1c3613ee7 Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Sun, 3 May 2015 01:22:34 +0200 Subject: [PATCH 02/14] parallel: Fixed bug #44995: parallel echo {#} ::: 1 2 ::: 1 2. Passes testsuite. --- doc/release_new_version | 7 +- src/parallel | 82 +++++++++++---------- src/sem.pod | 13 +++- testsuite/tests-to-run/parallel-local-3s.sh | 5 ++ testsuite/wanted-results/parallel-local-3s | 44 +++++------ testsuite/wanted-results/parallel-tutorial | 44 ++++++----- 6 files changed, 114 insertions(+), 81 deletions(-) diff --git a/doc/release_new_version b/doc/release_new_version index 9e38d778..b6774557 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -226,13 +226,14 @@ New in this release: * <> CIDER: a pipeline for detecting waves of coordinated transcriptional regulation in gene expression time-course data http://biorxiv.org/content/biorxiv/early/2015/03/17/012518.full.pdf - * <> GNU Parallel was used (unfortunately without citation) in: MUGBAS: a species free gene-based programme suite for post-GWAS analysis http://www.ncbi.nlm.nih.gov/pubmed/25765345 taxator-tk http://algbio.cs.uni-duesseldorf.de/webapps/wa-download/ (check it) * << afventer svar fra Rachel >> GNU Parallel was used in: SISRS: Site Identification from Short Read Sequences https://github.com/rachelss/SISRS/ +* GNU Parallel was cited in: Toward Enhanced Metadata Quality of Large-Scale Digital Libraries: Estimating Volume Time Range https://www.ideals.illinois.edu/bitstream/handle/2142/73656/186_ready.pdf + * GNU Parallel was cited in: Sequencing the cap-snatching repertoire of H1N1 influenza provides insight into the mechanism of viral transcription initiation http://nar.oxfordjournals.org/content/early/2015/04/20/nar.gkv333.full.pdf * GNU Parallel was cited in: Genome assemblyusing Nanopore-guided long and error-free DNA reads http://www.biomedcentral.com/content/pdf/s12864-015-1519-z.pdf @@ -247,6 +248,10 @@ taxator-tk http://algbio.cs.uni-duesseldorf.de/webapps/wa-download/ (check it) * Functions and GNU parallel for effective cluster load management http://genomespot.blogspot.dk/2015/04/functions-and-gnu-parallel-for.html +* Use parallel processing to save time importing databases http://drupalsun.com/node/41854 + +* Run multiple ssh commands in parallel with GNU Parallel http://www.ameir.net/blog/archives/380-run-multiple-ssh-commands-in-parallel-with-gnu-parallel.html + * Bug fixes and man page updates. GNU Parallel - For people who live life in the parallel lane. diff --git a/src/parallel b/src/parallel index 9a71f93c..b72b1edb 100755 --- a/src/parallel +++ b/src/parallel @@ -1053,7 +1053,7 @@ sub parse_options { sub init_globals { # Defaults: - $Global::version = 20150424; + $Global::version = 20150426; $Global::progname = 'parallel'; $Global::infinity = 2**31; $Global::debug = 0; @@ -3032,8 +3032,11 @@ sub reaper { next; } my $job = $Global::running{$stiff}; + # '-a <(seq 10)' will give us a pid not in %Global::running $job or next; + delete $Global::running{$stiff}; + $Global::total_running--; $job->set_exitstatus($? >> 8); $job->set_exitsignal($? & 127); debug("run", "seq ",$job->seq()," died (", $job->exitstatus(), ")"); @@ -3042,7 +3045,9 @@ sub reaper { # The process that died had the tty => release it $Global::tty_taken = 0; } - + my $sshlogin = $job->sshlogin(); + $sshlogin->dec_jobs_running(); + $sshlogin->inc_jobs_completed(); if(not $job->should_be_retried()) { # The job is done # Free the jobslot @@ -3057,15 +3062,10 @@ sub reaper { $job->print(); } if($job->should_we_halt() eq "now") { - ::killall(); - ::wait_and_exit($Global::exitstatus); + ::killall(); + ::wait_and_exit($Global::halt_exitstatus); } } - my $sshlogin = $job->sshlogin(); - $sshlogin->dec_jobs_running(); - $sshlogin->inc_jobs_completed(); - $Global::total_running--; - delete $Global::running{$stiff}; start_more_jobs(); if($opt::progress) { my %progress = progress(); @@ -7490,9 +7490,16 @@ sub set_exitsignal { $Global::halt_pct <= $Global::total_failed / $Global::total_started and $Global::total_failed > 3)) { - # More than N jobs or more than N% failed - $Global::halt_exitstatus = $job->exitstatus(); - if($Global::halt_when eq "soon") { + # At least N jobs had failed + # or at least N% had failed and more than 3 + if($Global::halt_count and $Global::halt_count == 1) { + $Global::halt_exitstatus = $job->exitstatus(); + } else { + $Global::halt_exitstatus = $Global::total_failed; + } + ::debug("halt","Pct: ",$Global::halt_pct,"<=",$Global::total_failed / $Global::total_started," count: ",$Global::halt_count,"\n"); + if($Global::halt_when eq "soon" + and scalar(keys %Global::running) > 0) { ::status ("$Global::progname: Starting no more jobs. ", "Waiting for ", scalar(keys %Global::running), @@ -7504,19 +7511,23 @@ sub set_exitsignal { } } else { if($Global::halt_success) { + $Global::halt_exitstatus = $Global::total_failed; ::status("$Global::progname: This job succeeded:\n", $job->replaced(),"\n"); if(($Global::halt_count and $Global::halt_count <= - 1+$Global::total_completed-$Global::total_failed) + $Global::total_completed-$Global::total_failed) or ($Global::halt_pct and $Global::halt_pct <= - (1+$Global::total_completed-$Global::total_failed) + ($Global::total_completed-$Global::total_failed) / $Global::total_completed and ($Global::total_completed-$Global::total_failed) > 3)) { - $Global::halt_exitstatus = $job->exitstatus(); - if($Global::halt_when eq "soon") { + # At least N jobs had success + # or at least N% had success and more than 3 + $Global::halt_exitstatus = 0; + if($Global::halt_when eq "soon" + and scalar(keys %Global::running) > 0) { ::status ("$Global::progname: Starting no more jobs. ", "Waiting for ", scalar(keys %Global::running), @@ -8845,18 +8856,20 @@ sub new { }, ref($class) || $class; } -sub replace { - # Calculates the corresponding value for a given perl expression - # Returns: - # The calculated string (quoted if asked for) - my $self = shift; - my $perlexpr = shift; # E.g. $_=$_ or s/.gz// - my $quote = (shift) ? 1 : 0; # should the string be quoted? - # This is actually a CommandLine-object, - # but it looks nice to be able to say {= $job->slot() =} - my $job = shift; - $perlexpr =~ s/^-?\d+ //; # Positional replace treated as normal replace - if(not defined $self->{"rpl",0,$perlexpr}) { +{ + my %perleval; + + sub replace { + # Calculates the corresponding value for a given perl expression + # Returns: + # The calculated string (quoted if asked for) + my $self = shift; + my $perlexpr = shift; # E.g. $_=$_ or s/.gz// + my $quote = (shift) ? 1 : 0; # should the string be quoted? + # This is actually a CommandLine-object, + # but it looks nice to be able to say {= $job->slot() =} + my $job = shift; + $perlexpr =~ s/^-?\d+ +//; # Positional replace treated as normal replace local $_; if($Global::trim eq "n") { $_ = $self->{'orig'}; @@ -8864,10 +8877,10 @@ sub replace { $_ = trim_of($self->{'orig'}); } ::debug("replace", "eval ", $perlexpr, " ", $_, "\n"); - if(not $Global::perleval{$perlexpr}) { + if(not $perleval{$perlexpr}) { # Make an anonymous function of the $perlexpr # And more importantly: Compile it only once - if($Global::perleval{$perlexpr} = + if($perleval{$perlexpr} = eval('sub { no strict; no warnings; my $job = shift; '. $perlexpr.' }')) { # All is good @@ -8878,14 +8891,9 @@ sub replace { } } # Execute the function - $Global::perleval{$perlexpr}->($job); - $self->{"rpl",0,$perlexpr} = $_; + $perleval{$perlexpr}->($job); + return $quote ? ::shell_quote_scalar($_) : $_; } - if(not defined $self->{"rpl",$quote,$perlexpr}) { - $self->{"rpl",1,$perlexpr} = - ::shell_quote_scalar($self->{"rpl",0,$perlexpr}); - } - return $self->{"rpl",$quote,$perlexpr}; } sub orig { diff --git a/src/sem.pod b/src/sem.pod index 389a858a..742536cf 100755 --- a/src/sem.pod +++ b/src/sem.pod @@ -161,10 +161,17 @@ available. pod2html creates two files: pod2htmd.tmp and pod2htmi.tmp which it does not clean up. It uses these two files for a short time. But if you run multiple pod2html in parallel (e.g. in a Makefile with make --j) you need to protect pod2html from running twice at the same -time. B running as a mutex will do just that: +-j) there is a risk that two different instances of pod2html will +write to the files at the same time: - sem --fg --id pod2html pod2html foo.pod > foo.html + # This may fail due to shared pod2htmd.tmp/pod2htmi.tmp files + pod2html foo.pod --outfile foo.html & pod2html bar.pod --outfile bar.html + +You need to protect pod2html from running twice at the same time. +B running as a mutex will make sure only one runs: + + sem --id pod2html pod2html foo.pod --outfile foo.html + sem --id pod2html pod2html bar.pod --outfile bar.html sem --fg --id pod2html rm -f pod2htmd.tmp pod2htmi.tmp diff --git a/testsuite/tests-to-run/parallel-local-3s.sh b/testsuite/tests-to-run/parallel-local-3s.sh index e0e1013e..caa1f920 100644 --- a/testsuite/tests-to-run/parallel-local-3s.sh +++ b/testsuite/tests-to-run/parallel-local-3s.sh @@ -14,6 +14,7 @@ echo '**' echo '### Test --halt-on-error 0'; (echo "sleep 1;true"; echo "sleep 2;false";echo "sleep 3;true") | parallel -j10 --halt-on-error 0; echo $?; + (echo "sleep 1;true"; echo "sleep 2;false";echo "sleep 3;true";echo "sleep 4; non_exist") | parallel -j10 --halt 0; echo $? @@ -22,6 +23,7 @@ echo '**' echo '### Test --halt-on-error 1'; (echo "sleep 1;true"; echo "sleep 2;false";echo "sleep 3;true") | parallel -j10 --halt-on-error 1; echo $?; + (echo "sleep 1;true"; echo "sleep 2;false";echo "sleep 3;true";echo "sleep 4; non_exist") | parallel -j10 --halt 1; echo $? @@ -30,6 +32,7 @@ echo '**' echo '### Test --halt-on-error 2'; (echo "sleep 1;true"; echo "sleep 2;false";echo "sleep 3;true") | parallel -j10 --halt-on-error 2; echo $?; + (echo "sleep 1;true"; echo "sleep 2;false";echo "sleep 3;true";echo "sleep 4; non_exist") | parallel -j10 --halt 2; echo $? @@ -38,6 +41,7 @@ echo '**' echo '### Test --halt -1'; (echo "sleep 1;false"; echo "sleep 2;true";echo "sleep 3;false") | parallel -j10 --halt-on-error -1; echo $?; + (echo "sleep 1;false"; echo "sleep 2;true";echo "sleep 3;false";echo "sleep 4; non_exist") | parallel -j10 --halt -1; echo $? @@ -46,6 +50,7 @@ echo '**' echo '### Test --halt -2'; (echo "sleep 1;false"; echo "sleep 2;true";echo "sleep 3;false") | parallel -j10 --halt-on-error -2; echo $?; + (echo "sleep 1;false"; echo "sleep 2;true";echo "sleep 3;false";echo "sleep 4; non_exist") | parallel -j10 --halt -2; echo $? diff --git a/testsuite/wanted-results/parallel-local-3s b/testsuite/wanted-results/parallel-local-3s index 4a6ec714..50514ada 100644 --- a/testsuite/wanted-results/parallel-local-3s +++ b/testsuite/wanted-results/parallel-local-3s @@ -19,12 +19,13 @@ echo '### Test --halt-on-error 1'; (echo "sleep 1;true"; echo "sleep 2;false"; 127 parallel: This job failed: sleep 2;false -parallel: Starting no more jobs. Waiting for 3 jobs to finish. This job failed: +parallel: Starting no more jobs. Waiting for 1 jobs to finish. +parallel: This job failed: sleep 2;false +parallel: Starting no more jobs. Waiting for 2 jobs to finish. /bin/bash: non_exist: command not found parallel: This job failed: sleep 4; non_exist -parallel: Starting no more jobs. Waiting for 1 jobs to finish. echo '**' ** echo '### Test --halt-on-error 2'; (echo "sleep 1;true"; echo "sleep 2;false";echo "sleep 3;true") | parallel -j10 --halt-on-error 2; echo $?; (echo "sleep 1;true"; echo "sleep 2;false";echo "sleep 3;true";echo "sleep 4; non_exist") | parallel -j10 --halt 2; echo $? @@ -43,9 +44,10 @@ echo '### Test --halt -1'; (echo "sleep 1;false"; echo "sleep 2;true";echo "sl 0 parallel: This job succeeded: sleep 2;true -parallel: Starting no more jobs. Waiting for 3 jobs to finish. This job succeeded: +parallel: Starting no more jobs. Waiting for 1 jobs to finish. +parallel: This job succeeded: sleep 2;true -parallel: Starting no more jobs. Waiting for 3 jobs to finish. +parallel: Starting no more jobs. Waiting for 2 jobs to finish. /bin/bash: non_exist: command not found echo '**' ** @@ -66,40 +68,39 @@ exit code 9 1 parallel: This job failed: perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 1 -parallel: Starting no more jobs. Waiting for 9 jobs to finish. +parallel: Starting no more jobs. Waiting for 8 jobs to finish. 2 parallel: This job failed: perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 2 -parallel: Starting no more jobs. Waiting for 8 jobs to finish. +parallel: Starting no more jobs. Waiting for 7 jobs to finish. 3 parallel: This job failed: perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 3 -parallel: Starting no more jobs. Waiting for 7 jobs to finish. +parallel: Starting no more jobs. Waiting for 6 jobs to finish. 4 parallel: This job failed: perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 4 -parallel: Starting no more jobs. Waiting for 6 jobs to finish. +parallel: Starting no more jobs. Waiting for 5 jobs to finish. 5 parallel: This job failed: perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 5 -parallel: Starting no more jobs. Waiting for 5 jobs to finish. +parallel: Starting no more jobs. Waiting for 4 jobs to finish. 6 parallel: This job failed: perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 6 -parallel: Starting no more jobs. Waiting for 4 jobs to finish. +parallel: Starting no more jobs. Waiting for 3 jobs to finish. 7 parallel: This job failed: perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 7 -parallel: Starting no more jobs. Waiting for 3 jobs to finish. +parallel: Starting no more jobs. Waiting for 2 jobs to finish. 8 0 parallel: This job failed: perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 8 -parallel: Starting no more jobs. Waiting for 2 jobs to finish. +parallel: Starting no more jobs. Waiting for 1 jobs to finish. 9 parallel: This job failed: perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 9 -parallel: Starting no more jobs. Waiting for 1 jobs to finish. echo '### Test last dying print --halt-on-error 2'; (seq 0 8;echo 0; echo 9) | parallel -j10 -kq --halt 2 perl -e 'sleep $ARGV[0];print STDERR @ARGV,"\n"; exit shift'; echo exit code $? ### Test last dying print --halt-on-error 2 exit code 1 @@ -114,40 +115,39 @@ exit code 0 1 parallel: This job succeeded: perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ not\ shift 1 -parallel: Starting no more jobs. Waiting for 9 jobs to finish. +parallel: Starting no more jobs. Waiting for 8 jobs to finish. 2 parallel: This job succeeded: perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ not\ shift 2 -parallel: Starting no more jobs. Waiting for 8 jobs to finish. +parallel: Starting no more jobs. Waiting for 7 jobs to finish. 3 parallel: This job succeeded: perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ not\ shift 3 -parallel: Starting no more jobs. Waiting for 7 jobs to finish. +parallel: Starting no more jobs. Waiting for 6 jobs to finish. 4 parallel: This job succeeded: perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ not\ shift 4 -parallel: Starting no more jobs. Waiting for 6 jobs to finish. +parallel: Starting no more jobs. Waiting for 5 jobs to finish. 5 parallel: This job succeeded: perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ not\ shift 5 -parallel: Starting no more jobs. Waiting for 5 jobs to finish. +parallel: Starting no more jobs. Waiting for 4 jobs to finish. 6 parallel: This job succeeded: perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ not\ shift 6 -parallel: Starting no more jobs. Waiting for 4 jobs to finish. +parallel: Starting no more jobs. Waiting for 3 jobs to finish. 7 parallel: This job succeeded: perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ not\ shift 7 -parallel: Starting no more jobs. Waiting for 3 jobs to finish. +parallel: Starting no more jobs. Waiting for 2 jobs to finish. 8 0 parallel: This job succeeded: perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ not\ shift 8 -parallel: Starting no more jobs. Waiting for 2 jobs to finish. +parallel: Starting no more jobs. Waiting for 1 jobs to finish. 9 parallel: This job succeeded: perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ not\ shift 9 -parallel: Starting no more jobs. Waiting for 1 jobs to finish. echo '### Test last dying print --halt-on-error -2'; (seq 0 8;echo 0; echo 9) | parallel -j10 -kq --halt -2 perl -e 'sleep $ARGV[0];print STDERR @ARGV,"\n"; exit not shift'; echo exit code $? ### Test last dying print --halt-on-error -2 exit code 0 diff --git a/testsuite/wanted-results/parallel-tutorial b/testsuite/wanted-results/parallel-tutorial index d6c1c5b3..7845a371 100644 --- a/testsuite/wanted-results/parallel-tutorial +++ b/testsuite/wanted-results/parallel-tutorial @@ -478,9 +478,10 @@ Seq Host Starttime JobRuntime Send Receive Exitval Signal Command 0 1 2 -parallel: Starting no more jobs. Waiting for 2 jobs to finish. This job failed: +parallel: This job failed: echo 1; exit 1 -parallel: Starting no more jobs. Waiting for 1 jobs to finish. This job failed: +parallel: Starting no more jobs. Waiting for 1 jobs to finish. +parallel: This job failed: echo 2; exit 2 parallel -j2 --halt 2 echo {}\; exit {} ::: 0 0 1 2 3 0 @@ -496,9 +497,16 @@ echo 1; exit 1 3 4 5 -parallel: Starting no more jobs. Waiting for 2 jobs to finish. This job failed: +parallel: This job failed: +echo 1; exit 1 +parallel: This job failed: +echo 2; exit 2 +parallel: This job failed: +echo 3; exit 3 +parallel: This job failed: echo 4; exit 4 -parallel: Starting no more jobs. Waiting for 1 jobs to finish. This job failed: +parallel: Starting no more jobs. Waiting for 1 jobs to finish. +parallel: This job failed: echo 5; exit 5 parallel -k --retries 3 'echo tried {} >>/tmp/runs; echo completed {}; exit {}' ::: 1 2 0 cat /tmp/runs @@ -931,7 +939,7 @@ This helps funding further development; and it won't cost you a cent. If you pay 10000 EUR you should feel free to use GNU Parallel without citing. parallel --version -GNU parallel 20150422 +GNU parallel 20150426 Copyright (C) 2007,2008,2009,2010,2011,2012,2013,2014,2015 Ole Tange and Free Software Foundation, Inc. License GPLv3+: GNU GPL version 3 or later @@ -943,7 +951,7 @@ Web site: http://www.gnu.org/software/parallel When using programs that use GNU Parallel to process data for publication please cite as described in 'parallel --bibtex'. parallel --minversion 20130722 && echo Your version is at least 20130722. -20150422 +20150426 Your version is at least 20130722. parallel --bibtex Academic tradition requires you to cite works you base your article on. @@ -951,17 +959,17 @@ When using programs that use GNU Parallel to process data for publication please cite: @article{Tange2011a, - title = {GNU Parallel - The Command-Line Power Tool}, - author = {O. Tange}, - address = {Frederiksberg, Denmark}, - journal = {;login: The USENIX Magazine}, - month = {Feb}, - number = {1}, - volume = {36}, - url = {http://www.gnu.org/s/parallel}, - year = {2011}, - pages = {42-47} - doi = {10.5281/zenodo.16303} + title = {GNU Parallel - The Command-Line Power Tool}, + author = {O. Tange}, + address = {Frederiksberg, Denmark}, + journal = {;login: The USENIX Magazine}, + month = {Feb}, + number = {1}, + volume = {36}, + url = {http://www.gnu.org/s/parallel}, + year = {2011}, + pages = {42-47} + doi = {10.5281/zenodo.16303} } (Feel free to use \nocite{Tange2011a}) @@ -989,4 +997,4 @@ C \nice -n17 /bin/bash -c echo\ A \nice -n17 /bin/bash -c echo\ B \nice -n17 /bin/bash -c echo\ C -7 +6 From 5306f0f3966fb76c51169b1fe852353e073ca61c Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Thu, 7 May 2015 00:40:36 +0200 Subject: [PATCH 03/14] testsuite: more testing of --halt. --- Makefile.am | 2 +- Makefile.in | 2 +- doc/release_new_version | 4 + src/parallel | 6 +- src/parallel.pod | 7 +- src/parallel_tutorial.html | 16 +- src/parallel_tutorial.pod | 29 +- testsuite/tests-to-run/parallel-local-0.3s.sh | 6 + testsuite/tests-to-run/parallel-local-3s.sh | 17 +- testsuite/tests-to-run/parallel-local4.sh | 2 +- testsuite/wanted-results/parallel-local-0.3s | 9 + testsuite/wanted-results/parallel-local-3s | 564 +++++++++++++++++- testsuite/wanted-results/parallel-local4 | 2 +- testsuite/wanted-results/parallel-tutorial | 6 +- 14 files changed, 648 insertions(+), 24 deletions(-) diff --git a/Makefile.am b/Makefile.am index d7dd8c2e..8bef00a4 100644 --- a/Makefile.am +++ b/Makefile.am @@ -59,4 +59,4 @@ upload: pushd; \ sudo cp /usr/local/bin/parallel /usr/local/bin/parallel-$(YYYYMMDD) -EXTRA_DIST = CITATION +EXTRA_DIST = CITATION CREDITS diff --git a/Makefile.in b/Makefile.in index a9cc198f..a0f0e907 100644 --- a/Makefile.in +++ b/Makefile.in @@ -263,7 +263,7 @@ top_build_prefix = @top_build_prefix@ top_builddir = @top_builddir@ top_srcdir = @top_srcdir@ SUBDIRS = src -EXTRA_DIST = CITATION +EXTRA_DIST = CITATION CREDITS all: config.h $(MAKE) $(AM_MAKEFLAGS) all-recursive diff --git a/doc/release_new_version b/doc/release_new_version index b6774557..3a74fd10 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -228,8 +228,12 @@ New in this release: * <> GNU Parallel was used (unfortunately without citation) in: MUGBAS: a species free gene-based programme suite for post-GWAS analysis http://www.ncbi.nlm.nih.gov/pubmed/25765345 +http://biorxiv.org/content/biorxiv/early/2015/05/05/018085.full.pdf + taxator-tk http://algbio.cs.uni-duesseldorf.de/webapps/wa-download/ (check it) +* <> GNU Parallel was used in: Large Scale Author Name Disambiguation in Digital Libraries http://ieeexplore.ieee.org/xpl/abstractReferences.jsp?tp=&arnumber=7004487&url=http%3A%2F%2Fieeexplore.ieee.org%2Fxpls%2Fabs_all.jsp%3Farnumber%3D7004487 + * << afventer svar fra Rachel >> GNU Parallel was used in: SISRS: Site Identification from Short Read Sequences https://github.com/rachelss/SISRS/ * GNU Parallel was cited in: Toward Enhanced Metadata Quality of Large-Scale Digital Libraries: Estimating Volume Time Range https://www.ideals.illinois.edu/bitstream/handle/2142/73656/186_ready.pdf diff --git a/src/parallel b/src/parallel index b72b1edb..7a2b2ae7 100755 --- a/src/parallel +++ b/src/parallel @@ -1053,7 +1053,7 @@ sub parse_options { sub init_globals { # Defaults: - $Global::version = 20150426; + $Global::version = 20150503; $Global::progname = 'parallel'; $Global::infinity = 2**31; $Global::debug = 0; @@ -1131,7 +1131,7 @@ sub parse_halt { # Split: soon,fail=5% my ($when,$fail_success,$pct_count) = split /[,=]/, $opt::halt; if(not grep { $when eq $_ } qw(never soon now)) { - ::error("--halt must have 'never', 'soon', or 'now'"); + ::error("--halt must have 'never', 'soon', or 'now'\n"); } $Global::halt_when = $when; if($when ne "never") { @@ -7495,7 +7495,7 @@ sub set_exitsignal { if($Global::halt_count and $Global::halt_count == 1) { $Global::halt_exitstatus = $job->exitstatus(); } else { - $Global::halt_exitstatus = $Global::total_failed; + $Global::halt_exitstatus = ::min($Global::total_failed,254); } ::debug("halt","Pct: ",$Global::halt_pct,"<=",$Global::total_failed / $Global::total_started," count: ",$Global::halt_count,"\n"); if($Global::halt_when eq "soon" diff --git a/src/parallel.pod b/src/parallel.pod index 09160700..2ab37e11 100644 --- a/src/parallel.pod +++ b/src/parallel.pod @@ -809,8 +809,9 @@ See also B<--resume> B<--resume-failed>. =item B<-P> I -Number of jobslots. Run up to N jobs in parallel. 0 means as many as -possible. Default is 100% which will run one job per CPU core. +Number of jobslots on each machine. Run up to N jobs in parallel. 0 +means as many as possible. Default is 100% which will run one job per +CPU core on each machine. If B<--semaphore> is set default is 1 thus making a mutex. @@ -1498,7 +1499,7 @@ to do anything. Work as a counting semaphore. B<--semaphore> will cause GNU B to start I in the background. When the number of -simultaneous jobs is reached, GNU B will wait for one of +jobs given by B<--jobs> is reached, GNU B will wait for one of these to complete before starting another command. B<--semaphore> implies B<--bg> unless B<--fg> is specified. diff --git a/src/parallel_tutorial.html b/src/parallel_tutorial.html index a70f8698..6e357bb7 100644 --- a/src/parallel_tutorial.html +++ b/src/parallel_tutorial.html @@ -1969,7 +1969,19 @@

GNU Parallel can work as a counting semaphore. This is slower and less efficient than its normal mode.

-

An alias for 'parallel --semaphore' is 'sem'. The default is to allow only one program to run at a time (technically called a mutex). The program is started in the background. Use --wait for all 'sem's to finish:

+

A counting semaphore is like a row of toilets. People needing a toilet can use any toilet, but if there are more people than toilets, they will have to wait for one of the toilets to be available.

+ +

An alias for 'parallel --semaphore' is 'sem'.

+ +

'sem' will follow a person to the toilets, wait until a toilet is available, leave the person in the toilet and exit.

+ +

'sem --fg' will follow a person to the toilets, wait until a toilet is available, stay with the person in the toilet and exit when the person exits.

+ +

'sem --wait' will wait for all persons to leave the toilets.

+ +

'sem' does not have a queue discipline, so the next person is chosen randomly.

+ +

-j sets the number of toilets. The default is to have only one toilet (technically this is called a mutex). The program is started in the background and 'sem' exits immediately. Use --wait to wait for all 'sem's to finish:

  sem 'sleep 1; echo The first finished' &&
     echo The first is now running in the background &&
@@ -1984,7 +1996,7 @@
   The second is now running in the background
   The second finished
-

The command can be run in the foreground with --fg:

+

The command can be run in the foreground with --fg, which will only exit when the command completes:

  sem --fg 'sleep 1; echo The first finished' &&
     echo The first finished running in the foreground &&
diff --git a/src/parallel_tutorial.pod b/src/parallel_tutorial.pod
index 83796be6..02708f20 100644
--- a/src/parallel_tutorial.pod
+++ b/src/parallel_tutorial.pod
@@ -1965,10 +1965,28 @@ This technique can be used for:
 GNU Parallel can work as a counting semaphore. This is slower and less
 efficient than its normal mode.
 
-An alias for 'parallel --semaphore' is 'sem'. The default is to allow
-only one program to run at a time (technically called a mutex). The
-program is started in the background. Use --wait for all 'sem's to
-finish:
+A counting semaphore is like a row of toilets. People needing a toilet
+can use any toilet, but if there are more people than toilets, they
+will have to wait for one of the toilets to be available.
+
+An alias for 'parallel --semaphore' is 'sem'.
+
+'sem' will follow a person to the toilets, wait until a toilet is
+available, leave the person in the toilet and exit.
+
+'sem --fg' will follow a person to the toilets, wait until a toilet is
+available, stay with the person in the toilet and exit when the person
+exits.
+
+'sem --wait' will wait for all persons to leave the toilets.
+
+'sem' does not have a queue discipline, so the next person is chosen
+randomly.
+
+-j sets the number of toilets. The default is to have only one toilet
+(technically this is called a mutex). The program is started in the
+background and 'sem' exits immediately. Use --wait to wait for all
+'sem's to finish:
 
   sem 'sleep 1; echo The first finished' &&
     echo The first is now running in the background &&
@@ -1983,7 +2001,8 @@ Output:
   The second is now running in the background
   The second finished
 
-The command can be run in the foreground with --fg:
+The command can be run in the foreground with --fg, which will only
+exit when the command completes:
 
   sem --fg 'sleep 1; echo The first finished' &&
     echo The first finished running in the foreground &&
diff --git a/testsuite/tests-to-run/parallel-local-0.3s.sh b/testsuite/tests-to-run/parallel-local-0.3s.sh
index 63cf19d9..427979e3 100644
--- a/testsuite/tests-to-run/parallel-local-0.3s.sh
+++ b/testsuite/tests-to-run/parallel-local-0.3s.sh
@@ -148,6 +148,12 @@ echo '**'
 parallel --halt 2 ::: 'sleep 1' burnP6 false; killall burnP6 && echo ERROR: burnP6 should be killed
 parallel --halt -2 ::: 'sleep 1' burnP5 true; killall burnP5 && echo ERROR: burnP5 should be killed
 
+echo '**'
+
+echo '### bug #44995: parallel echo {#} ::: 1 2 ::: 1 2'
+
+parallel -k echo {#} ::: 1 2 ::: 1 2
+
 EOF
 echo '### 1 .par file from --files expected'
 find /tmp{/*,}/*.{par,tms,tmx} 2>/dev/null -mmin -10 | wc -l
diff --git a/testsuite/tests-to-run/parallel-local-3s.sh b/testsuite/tests-to-run/parallel-local-3s.sh
index caa1f920..09b236cb 100644
--- a/testsuite/tests-to-run/parallel-local-3s.sh
+++ b/testsuite/tests-to-run/parallel-local-3s.sh
@@ -74,6 +74,19 @@ echo '### Test last dying print --halt-on-error -2';
 
 echo '**'
 
+testhalt() { 
+  echo '### testhalt --halt '$1; 
+  (yes 0 | head -n 10; seq 10) | stdout parallel -kj4 --halt $1 'sleep {= $_*=0.3 =}; exit {}'; echo $?; 
+  (seq 10; yes 0 | head -n 10) | stdout parallel -kj4 --halt $1 'sleep {= $_*=0.3 =}; exit {}'; echo $?; 
+}; 
+export -f testhalt; 
+  parallel -kj0 testhalt ::: now,fail=0 now,fail=1 now,fail=2 now,fail=30%  now,fail=70% 
+    soon,fail=0 soon,fail=1 soon,fail=2 soon,fail=30% soon,fail=70% 
+    now,success=0 now,success=1 now,success=2 now,success=30% now,success=70% 
+    soon,success=0 soon,success=1 soon,success=2 soon,success=30% now,success=70%
+
+echo '**'
+
 echo '### Test slow arguments generation - https://savannah.gnu.org/bugs/?32834'; 
   seq 1 3 | parallel -j1 "sleep 2; echo {}" | parallel -kj2 echo
 
@@ -83,7 +96,7 @@ echo '### Are children killed if GNU Parallel receives TERM twice? There should
 
   parallel -q bash -c 'sleep 120 & pid=$!; wait $pid' ::: 1 & 
     T=$!; 
-    sleep 1; 
+    sleep 2; 
     pstree $$; 
     kill -TERM $T; 
     sleep 1; 
@@ -98,7 +111,7 @@ echo '### Are children killed if GNU Parallel receives INT twice? There should b
 
   parallel -q bash -c 'sleep 120 & pid=$!; wait $pid' ::: 1 & 
     T=$!; 
-    sleep 1; 
+    sleep 2; 
     pstree $$; 
     kill -INT $T; 
     sleep 1; 
diff --git a/testsuite/tests-to-run/parallel-local4.sh b/testsuite/tests-to-run/parallel-local4.sh
index b2af5054..2918a3d5 100644
--- a/testsuite/tests-to-run/parallel-local4.sh
+++ b/testsuite/tests-to-run/parallel-local4.sh
@@ -2,7 +2,7 @@
 
 cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj0 -k --joblog /tmp/jl-`basename $0` -L1
 echo '### bug #36595: silent loss of input with --pipe and --sshlogin'
-  seq 10000 | xargs | parallel --pipe -S 10/localhost cat | wc
+  seq 10000 | xargs | parallel --pipe -S 10/localhost cat 2>/dev/null | wc
 
 echo 'bug #36707: --controlmaster eats jobs'
   seq 2 | parallel -k --controlmaster --sshlogin localhost echo OK{}
diff --git a/testsuite/wanted-results/parallel-local-0.3s b/testsuite/wanted-results/parallel-local-0.3s
index d1814429..e84a1237 100644
--- a/testsuite/wanted-results/parallel-local-0.3s
+++ b/testsuite/wanted-results/parallel-local-0.3s
@@ -317,5 +317,14 @@ parallel --halt -2 ::: 'sleep 1' burnP5 true; killall burnP5 && echo ERROR: burn
 parallel: This job succeeded:
 true
 burnP5: no process found
+echo '**'
+**
+echo '### bug #44995: parallel echo {#} ::: 1 2 ::: 1 2'
+### bug #44995: parallel echo {#} ::: 1 2 ::: 1 2
+parallel -k echo {#} ::: 1 2 ::: 1 2
+1
+2
+3
+4
 ### 1 .par file from --files expected
 1
diff --git a/testsuite/wanted-results/parallel-local-3s b/testsuite/wanted-results/parallel-local-3s
index 50514ada..f2010a0a 100644
--- a/testsuite/wanted-results/parallel-local-3s
+++ b/testsuite/wanted-results/parallel-local-3s
@@ -157,6 +157,566 @@ parallel: This job succeeded:
 perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ not\ shift 1
 echo '**'
 **
+testhalt() {   echo '### testhalt --halt '$1;   (yes 0 | head -n 10; seq 10) | stdout parallel -kj4 --halt $1 'sleep {= $_*=0.3 =}; exit {}'; echo $?;   (seq 10; yes 0 | head -n 10) | stdout parallel -kj4 --halt $1 'sleep {= $_*=0.3 =}; exit {}'; echo $?; }; export -f testhalt;   parallel -kj0 testhalt ::: now,fail=0 now,fail=1 now,fail=2 now,fail=30%  now,fail=70%     soon,fail=0 soon,fail=1 soon,fail=2 soon,fail=30% soon,fail=70%     now,success=0 now,success=1 now,success=2 now,success=30% now,success=70%     soon,success=0 soon,success=1 soon,success=2 soon,success=30% now,success=70%
+### testhalt --halt now,fail=0
+parallel: This job failed:
+sleep 0.3; exit 1
+parallel: This job failed:
+sleep 0.6; exit 2
+parallel: This job failed:
+sleep 0.9; exit 3
+parallel: This job failed:
+sleep 1.2; exit 4
+parallel: This job failed:
+sleep 1.5; exit 5
+parallel: This job failed:
+sleep 1.8; exit 6
+parallel: This job failed:
+sleep 2.1; exit 7
+parallel: This job failed:
+sleep 2.4; exit 8
+parallel: This job failed:
+sleep 2.7; exit 9
+parallel: This job failed:
+sleep 3; exit 10
+0
+parallel: This job failed:
+sleep 0.3; exit 1
+parallel: This job failed:
+sleep 0.6; exit 2
+parallel: This job failed:
+sleep 0.9; exit 3
+parallel: This job failed:
+sleep 1.2; exit 4
+parallel: This job failed:
+sleep 1.5; exit 5
+parallel: This job failed:
+sleep 1.8; exit 6
+parallel: This job failed:
+sleep 2.1; exit 7
+parallel: This job failed:
+sleep 2.4; exit 8
+parallel: This job failed:
+sleep 2.7; exit 9
+parallel: This job failed:
+sleep 3; exit 10
+0
+### testhalt --halt now,fail=1
+parallel: This job failed:
+sleep 0.3; exit 1
+1
+parallel: This job failed:
+sleep 0.3; exit 1
+1
+### testhalt --halt now,fail=2
+parallel: This job failed:
+sleep 0.3; exit 1
+parallel: This job failed:
+sleep 0.6; exit 2
+2
+parallel: This job failed:
+sleep 0.3; exit 1
+parallel: This job failed:
+sleep 0.6; exit 2
+2
+### testhalt --halt now,fail=30%
+parallel: This job failed:
+sleep 0.3; exit 1
+parallel: This job failed:
+sleep 0.6; exit 2
+parallel: This job failed:
+sleep 0.9; exit 3
+parallel: This job failed:
+sleep 1.2; exit 4
+parallel: This job failed:
+sleep 1.5; exit 5
+parallel: This job failed:
+sleep 1.8; exit 6
+6
+parallel: This job failed:
+sleep 0.3; exit 1
+parallel: This job failed:
+sleep 0.6; exit 2
+parallel: This job failed:
+sleep 0.9; exit 3
+parallel: This job failed:
+sleep 1.2; exit 4
+4
+### testhalt --halt now,fail=70%
+parallel: This job failed:
+sleep 0.3; exit 1
+parallel: This job failed:
+sleep 0.6; exit 2
+parallel: This job failed:
+sleep 0.9; exit 3
+parallel: This job failed:
+sleep 1.2; exit 4
+parallel: This job failed:
+sleep 1.5; exit 5
+parallel: This job failed:
+sleep 1.8; exit 6
+parallel: This job failed:
+sleep 2.1; exit 7
+parallel: This job failed:
+sleep 2.4; exit 8
+parallel: This job failed:
+sleep 2.7; exit 9
+parallel: This job failed:
+sleep 3; exit 10
+0
+parallel: This job failed:
+sleep 0.3; exit 1
+parallel: This job failed:
+sleep 0.6; exit 2
+parallel: This job failed:
+sleep 0.9; exit 3
+parallel: This job failed:
+sleep 1.2; exit 4
+parallel: This job failed:
+sleep 1.5; exit 5
+parallel: This job failed:
+sleep 1.8; exit 6
+parallel: This job failed:
+sleep 2.1; exit 7
+7
+### testhalt --halt soon,fail=0
+parallel: This job failed:
+sleep 0.3; exit 1
+parallel: This job failed:
+sleep 0.6; exit 2
+parallel: This job failed:
+sleep 0.9; exit 3
+parallel: This job failed:
+sleep 1.2; exit 4
+parallel: This job failed:
+sleep 1.5; exit 5
+parallel: This job failed:
+sleep 1.8; exit 6
+parallel: This job failed:
+sleep 2.1; exit 7
+parallel: This job failed:
+sleep 2.4; exit 8
+parallel: This job failed:
+sleep 2.7; exit 9
+parallel: This job failed:
+sleep 3; exit 10
+0
+parallel: This job failed:
+sleep 0.3; exit 1
+parallel: This job failed:
+sleep 0.6; exit 2
+parallel: This job failed:
+sleep 0.9; exit 3
+parallel: This job failed:
+sleep 1.2; exit 4
+parallel: This job failed:
+sleep 1.5; exit 5
+parallel: This job failed:
+sleep 1.8; exit 6
+parallel: This job failed:
+sleep 2.1; exit 7
+parallel: This job failed:
+sleep 2.4; exit 8
+parallel: This job failed:
+sleep 2.7; exit 9
+parallel: This job failed:
+sleep 3; exit 10
+0
+### testhalt --halt soon,fail=1
+parallel: This job failed:
+sleep 0.3; exit 1
+parallel: Starting no more jobs. Waiting for 3 jobs to finish.
+parallel: This job failed:
+sleep 0.6; exit 2
+parallel: Starting no more jobs. Waiting for 2 jobs to finish.
+parallel: This job failed:
+sleep 0.9; exit 3
+parallel: Starting no more jobs. Waiting for 1 jobs to finish.
+parallel: This job failed:
+sleep 1.2; exit 4
+4
+parallel: This job failed:
+sleep 0.3; exit 1
+parallel: Starting no more jobs. Waiting for 3 jobs to finish.
+parallel: This job failed:
+sleep 0.6; exit 2
+parallel: Starting no more jobs. Waiting for 2 jobs to finish.
+parallel: This job failed:
+sleep 0.9; exit 3
+parallel: Starting no more jobs. Waiting for 1 jobs to finish.
+parallel: This job failed:
+sleep 1.2; exit 4
+4
+### testhalt --halt soon,fail=2
+parallel: This job failed:
+sleep 0.3; exit 1
+parallel: This job failed:
+sleep 0.6; exit 2
+parallel: Starting no more jobs. Waiting for 3 jobs to finish.
+parallel: This job failed:
+sleep 0.9; exit 3
+parallel: Starting no more jobs. Waiting for 2 jobs to finish.
+parallel: This job failed:
+sleep 1.2; exit 4
+parallel: Starting no more jobs. Waiting for 1 jobs to finish.
+parallel: This job failed:
+sleep 1.5; exit 5
+5
+parallel: This job failed:
+sleep 0.3; exit 1
+parallel: This job failed:
+sleep 0.6; exit 2
+parallel: Starting no more jobs. Waiting for 3 jobs to finish.
+parallel: This job failed:
+sleep 0.9; exit 3
+parallel: Starting no more jobs. Waiting for 2 jobs to finish.
+parallel: This job failed:
+sleep 1.2; exit 4
+parallel: Starting no more jobs. Waiting for 1 jobs to finish.
+parallel: This job failed:
+sleep 1.5; exit 5
+5
+### testhalt --halt soon,fail=30%
+parallel: This job failed:
+sleep 0.3; exit 1
+parallel: This job failed:
+sleep 0.6; exit 2
+parallel: This job failed:
+sleep 0.9; exit 3
+parallel: This job failed:
+sleep 1.2; exit 4
+parallel: This job failed:
+sleep 1.5; exit 5
+parallel: This job failed:
+sleep 1.8; exit 6
+parallel: Starting no more jobs. Waiting for 3 jobs to finish.
+parallel: This job failed:
+sleep 2.1; exit 7
+parallel: Starting no more jobs. Waiting for 2 jobs to finish.
+parallel: This job failed:
+sleep 2.4; exit 8
+parallel: Starting no more jobs. Waiting for 1 jobs to finish.
+parallel: This job failed:
+sleep 2.7; exit 9
+9
+parallel: This job failed:
+sleep 0.3; exit 1
+parallel: This job failed:
+sleep 0.6; exit 2
+parallel: This job failed:
+sleep 0.9; exit 3
+parallel: This job failed:
+sleep 1.2; exit 4
+parallel: Starting no more jobs. Waiting for 3 jobs to finish.
+parallel: This job failed:
+sleep 1.5; exit 5
+parallel: Starting no more jobs. Waiting for 2 jobs to finish.
+parallel: This job failed:
+sleep 1.8; exit 6
+parallel: Starting no more jobs. Waiting for 1 jobs to finish.
+parallel: This job failed:
+sleep 2.1; exit 7
+7
+### testhalt --halt soon,fail=70%
+parallel: This job failed:
+sleep 0.3; exit 1
+parallel: This job failed:
+sleep 0.6; exit 2
+parallel: This job failed:
+sleep 0.9; exit 3
+parallel: This job failed:
+sleep 1.2; exit 4
+parallel: This job failed:
+sleep 1.5; exit 5
+parallel: This job failed:
+sleep 1.8; exit 6
+parallel: This job failed:
+sleep 2.1; exit 7
+parallel: This job failed:
+sleep 2.4; exit 8
+parallel: This job failed:
+sleep 2.7; exit 9
+parallel: This job failed:
+sleep 3; exit 10
+0
+parallel: This job failed:
+sleep 0.3; exit 1
+parallel: This job failed:
+sleep 0.6; exit 2
+parallel: This job failed:
+sleep 0.9; exit 3
+parallel: This job failed:
+sleep 1.2; exit 4
+parallel: This job failed:
+sleep 1.5; exit 5
+parallel: This job failed:
+sleep 1.8; exit 6
+parallel: This job failed:
+sleep 2.1; exit 7
+parallel: Starting no more jobs. Waiting for 3 jobs to finish.
+parallel: This job failed:
+sleep 2.4; exit 8
+parallel: Starting no more jobs. Waiting for 2 jobs to finish.
+parallel: This job failed:
+sleep 2.7; exit 9
+parallel: Starting no more jobs. Waiting for 1 jobs to finish.
+parallel: This job failed:
+sleep 3; exit 10
+10
+### testhalt --halt now,success=0
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
+0
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
+7
+### testhalt --halt now,success=1
+parallel: This job succeeded:
+sleep 0; exit 0
+0
+parallel: This job succeeded:
+sleep 0; exit 0
+0
+### testhalt --halt now,success=2
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
+0
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
+0
+### testhalt --halt now,success=30%
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
+0
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
+0
+### testhalt --halt now,success=70%
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
+0
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
+7
+### testhalt --halt soon,success=0
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
+0
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
+7
+### testhalt --halt soon,success=1
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: Starting no more jobs. Waiting for 3 jobs to finish.
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: Starting no more jobs. Waiting for 2 jobs to finish.
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: Starting no more jobs. Waiting for 1 jobs to finish.
+parallel: This job succeeded:
+sleep 0; exit 0
+0
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: Starting no more jobs. Waiting for 3 jobs to finish.
+0
+### testhalt --halt soon,success=2
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: Starting no more jobs. Waiting for 3 jobs to finish.
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: Starting no more jobs. Waiting for 2 jobs to finish.
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: Starting no more jobs. Waiting for 1 jobs to finish.
+parallel: This job succeeded:
+sleep 0; exit 0
+0
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: Starting no more jobs. Waiting for 3 jobs to finish.
+0
+### testhalt --halt soon,success=30%
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: Starting no more jobs. Waiting for 3 jobs to finish.
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: Starting no more jobs. Waiting for 2 jobs to finish.
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: Starting no more jobs. Waiting for 1 jobs to finish.
+parallel: This job succeeded:
+sleep 0; exit 0
+0
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: Starting no more jobs. Waiting for 3 jobs to finish.
+0
+### testhalt --halt now,success=70%
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
+0
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
+7
+echo '**'
+**
 echo '### Test slow arguments generation - https://savannah.gnu.org/bugs/?32834';   seq 1 3 | parallel -j1 "sleep 2; echo {}" | parallel -kj2 echo
 ### Test slow arguments generation - https://savannah.gnu.org/bugs/?32834
 1
@@ -166,7 +726,7 @@ echo '**'
 **
 echo '### Are children killed if GNU Parallel receives TERM twice? There should be no sleep at the end'
 ### Are children killed if GNU Parallel receives TERM twice? There should be no sleep at the end
-  parallel -q bash -c 'sleep 120 & pid=$!; wait $pid' ::: 1 &     T=$!;     sleep 1;     pstree $$;     kill -TERM $T;     sleep 1;     pstree $$;     kill -TERM $T;     sleep 1;     pstree $$; echo '**'
+  parallel -q bash -c 'sleep 120 & pid=$!; wait $pid' ::: 1 &     T=$!;     sleep 2;     pstree $$;     kill -TERM $T;     sleep 1;     pstree $$;     kill -TERM $T;     sleep 1;     pstree $$; echo '**'
 bash-+-perl---bash---sleep
      `-pstree
 bash-+-perl---bash---sleep
@@ -178,7 +738,7 @@ parallel: Waiting for these 1 jobs to finish. Send SIGTERM again to stop now.
 parallel: bash -c sleep\ 120\ \&\ pid=\$\!\;\ wait\ \$pid 1
 echo '### Are children killed if GNU Parallel receives INT twice? There should be no sleep at the end'
 ### Are children killed if GNU Parallel receives INT twice? There should be no sleep at the end
-  parallel -q bash -c 'sleep 120 & pid=$!; wait $pid' ::: 1 &     T=$!;     sleep 1;     pstree $$;     kill -INT $T;     sleep 1;     pstree $$; 
+  parallel -q bash -c 'sleep 120 & pid=$!; wait $pid' ::: 1 &     T=$!;     sleep 2;     pstree $$;     kill -INT $T;     sleep 1;     pstree $$; 
 bash-+-perl---bash---sleep
      `-pstree
 bash---pstree
diff --git a/testsuite/wanted-results/parallel-local4 b/testsuite/wanted-results/parallel-local4
index ab1a6a63..94dafa04 100644
--- a/testsuite/wanted-results/parallel-local4
+++ b/testsuite/wanted-results/parallel-local4
@@ -1,6 +1,6 @@
 echo '### bug #36595: silent loss of input with --pipe and --sshlogin'
 ### bug #36595: silent loss of input with --pipe and --sshlogin
-  seq 10000 | xargs | parallel --pipe -S 10/localhost cat | wc
+  seq 10000 | xargs | parallel --pipe -S 10/localhost cat 2>/dev/null | wc
       1   10000   48894
 echo 'bug #36707: --controlmaster eats jobs'
 bug #36707: --controlmaster eats jobs
diff --git a/testsuite/wanted-results/parallel-tutorial b/testsuite/wanted-results/parallel-tutorial
index 7845a371..bb8b8998 100644
--- a/testsuite/wanted-results/parallel-tutorial
+++ b/testsuite/wanted-results/parallel-tutorial
@@ -939,7 +939,7 @@ This helps funding further development; and it won't cost you a cent.
 If you pay 10000 EUR you should feel free to use GNU Parallel without citing.
 
   parallel --version
-GNU parallel 20150426
+GNU parallel 20150502
 Copyright (C) 2007,2008,2009,2010,2011,2012,2013,2014,2015 Ole Tange
 and Free Software Foundation, Inc.
 License GPLv3+: GNU GPL version 3 or later 
@@ -951,7 +951,7 @@ Web site: http://www.gnu.org/software/parallel
 When using programs that use GNU Parallel to process data for publication
 please cite as described in 'parallel --bibtex'.
   parallel --minversion 20130722 && echo Your version is at least 20130722.
-20150426
+20150502
 Your version is at least 20130722.
   parallel --bibtex
 Academic tradition requires you to cite works you base your article on.
@@ -997,4 +997,4 @@ C
 \nice -n17 /bin/bash -c echo\ A
 \nice -n17 /bin/bash -c echo\ B
 \nice -n17 /bin/bash -c echo\ C
-6
+7

From 35cbf61ecf0439337426d7e92f74b11ad38a8f96 Mon Sep 17 00:00:00 2001
From: Ole Tange 
Date: Sun, 10 May 2015 15:02:07 +0200
Subject: [PATCH 04/14] parallel: --halt returns 0..100. Passes testsuite.

---
 doc/release_new_version                       |  21 +-
 src/parallel                                  |  77 +++--
 src/parallel.pod                              | 103 ++++--
 testsuite/tests-to-run/parallel-local-3s.sh   |  12 +-
 testsuite/tests-to-run/parallel-local-ssh2.sh |   2 +-
 testsuite/tests-to-run/parallel-tutorial.sh   |   2 +
 testsuite/wanted-results/parallel-local-3s    | 309 +++++++-----------
 testsuite/wanted-results/parallel-tutorial    |  16 +-
 8 files changed, 268 insertions(+), 274 deletions(-)

diff --git a/doc/release_new_version b/doc/release_new_version
index 3a74fd10..a7f48876 100644
--- a/doc/release_new_version
+++ b/doc/release_new_version
@@ -222,29 +222,30 @@ Haiku of the month:
 
 New in this release:
 
-* <> Comparing the CarbonTracker and TM5-4DVar data assimilation systems for CO2 surface flux inversions http://www.atmos-chem-phys-discuss.net/15/8883/2015/acpd-15-8883-2015-discussion.html
-
-* <> CIDER: a pipeline for detecting waves of coordinated transcriptional regulation in gene expression time-course data http://biorxiv.org/content/biorxiv/early/2015/03/17/012518.full.pdf
-
-* <> GNU Parallel was used (unfortunately without citation) in: MUGBAS: a species free gene-based programme suite for post-GWAS analysis http://www.ncbi.nlm.nih.gov/pubmed/25765345
-
-http://biorxiv.org/content/biorxiv/early/2015/05/05/018085.full.pdf
+* <> GNU Parallel was used (unfortunately without citation) in: MUGBAS: a species free gene-based programme suite for post-GWAS analysis http://www.ncbi.nlm.nih.gov/pubmed/25765345
 
 taxator-tk http://algbio.cs.uni-duesseldorf.de/webapps/wa-download/ (check it)
 
 * <> GNU Parallel was used in: Large Scale Author Name Disambiguation in Digital Libraries http://ieeexplore.ieee.org/xpl/abstractReferences.jsp?tp=&arnumber=7004487&url=http%3A%2F%2Fieeexplore.ieee.org%2Fxpls%2Fabs_all.jsp%3Farnumber%3D7004487
 
-* << afventer svar fra Rachel >> GNU Parallel was used in: SISRS: Site Identification from Short Read Sequences https://github.com/rachelss/SISRS/
+* << Update forventet juni >> GNU Parallel was used in: SISRS: Site Identification from Short Read Sequences https://github.com/rachelss/SISRS/
+
+* <> GNU Parallel was used (unfortunately with wrong citation) in: TADSim: Discrete Event-Based Performance Prediction for Temperature-Accelerated Dynamics http://vruehle.de/publications/2015c.pdf
+
+
+* GNU Parallel was cited in: CIDER: a pipeline for detecting waves of coordinated transcriptional regulation in gene expression time-course data http://biorxiv.org/content/biorxiv/early/2015/03/17/012518.full.pdf
+
+* GNU Parallel was cited in: Building Genomic Analysis Pipelines in a Hackathon Setting with Bioinformatician Teams: DNA-seq, Epigenomics, Metagenomics and RNA-seq http://biorxiv.org/content/biorxiv/early/2015/05/05/018085.full.pdf
 
 * GNU Parallel was cited in: Toward Enhanced Metadata Quality of Large-Scale Digital Libraries: Estimating Volume Time Range https://www.ideals.illinois.edu/bitstream/handle/2142/73656/186_ready.pdf
 
 * GNU Parallel was cited in: Sequencing the cap-snatching repertoire of H1N1 influenza provides insight into the mechanism of viral transcription initiation http://nar.oxfordjournals.org/content/early/2015/04/20/nar.gkv333.full.pdf
 
-* GNU Parallel was cited in: Genome assemblyusing Nanopore-guided long and error-free DNA reads http://www.biomedcentral.com/content/pdf/s12864-015-1519-z.pdf
+* GNU Parallel was cited in: Genome assembly using Nanopore-guided long and error-free DNA reads http://www.biomedcentral.com/content/pdf/s12864-015-1519-z.pdf
 
 * GNU Parallel was cited in: Contrasting regional architectures of schizophrenia and other complex diseases using fast variance components analysis http://biorxiv.org/content/biorxiv/early/2015/03/13/016527.full.pdf
 
-* <> GNU Parallel was used (unfortunately with wrong citation) in: TADSim: Discrete Event-Based Performance Prediction for Temperature-Accelerated Dynamics http://vruehle.de/publications/2015c.pdf
+* GNU Parallel was used (unfortunately with wrong citation) in: Comparing the CarbonTracker and TM5-4DVar data assimilation systems for CO2 surface flux inversions http://www.atmos-chem-phys-discuss.net/15/8883/2015/acpd-15-8883-2015-discussion.html
 
 * GNU Parallel was used in: Gene Set Omic Analysis (GSOA) method https://bitbucket.org/srp33/gsoa
 
diff --git a/src/parallel b/src/parallel
index 7a2b2ae7..d68e582c 100755
--- a/src/parallel
+++ b/src/parallel
@@ -110,7 +110,7 @@ if($opt::nonall or $opt::onall) {
 $Global::JobQueue = JobQueue->new(
     \@command,\@input_source_fh,$Global::ContextReplace,$number_of_args,\@Global::ret_files);
 
-if($opt::eta or $opt::bar or $opt::shuf) {
+if($opt::eta or $opt::bar or $opt::shuf or $Global::halt_pct) {
     # Count the number of jobs or shuffle all jobs
     # before starting any
     $Global::JobQueue->total_jobs();
@@ -162,9 +162,17 @@ for(keys %Global::sshmaster) {
 }
 ::debug("init", "Halt\n");
 if($opt::halt and $Global::halt_when ne "never") {
+    if(not defined $Global::halt_exitstatus) {
+	if($Global::halt_pct) {
+	    $Global::halt_exitstatus =
+		::ceil($Global::total_failed / $Global::total_started * 100);
+	} elsif($Global::halt_count) {
+	    $Global::halt_exitstatus = ::min($Global::total_failed,101);
+	}
+    }
     wait_and_exit($Global::halt_exitstatus);
 } else {
-    wait_and_exit(min(undef_as_zero($Global::exitstatus),254));
+    wait_and_exit(min(undef_as_zero($Global::exitstatus),101));
 }
 
 sub __PIPE_MODE__ {}
@@ -1093,7 +1101,6 @@ sub init_globals {
     $Global::stderr_verbose = 0;
     $Global::default_simultaneous_sshlogins = 9;
     $Global::exitstatus = 0;
-    $Global::halt_exitstatus = 0;
     $Global::arg_sep = ":::";
     $Global::arg_file_sep = "::::";
     $Global::trim = 'n';
@@ -3349,16 +3356,14 @@ sub bibtex {
 	    if(open (my $fh, ">", $ENV{'HOME'}."/.parallel/will-cite")) {
 		close $fh;
 		print "\nThank you for your support. It is much appreciated. The citation\n",
-		"notice is now silenced. You may also use '--will-cite'.\n",
-		"If you use '--will-cite' in scripts you are expected to pay\n",
-		"the 10000 EUR, because you are making it harder to see the\n",
-		"citation notice.\n\n";
+		"notice is now silenced. For other ways to silence the citation notice\n",
+		"see 'man parallel' under '--bibtex'.\n\n";
 	    } else {
 		print "\nThank you for your support. It is much appreciated. The citation\n",
 		"cannot permanently be silenced. Use '--will-cite' instead.\n",
-		"If you use '--will-cite' in scripts you are expected to pay\n",
-		"the 10000 EUR, because you are making it harder to see the\n",
-		"citation notice.\n\n";
+		"If you use '--will-cite' in scripts you are making it harder to see the\n",
+		"citation notice.  However, if you pay 10000 EUR, you should feel free\n",
+		"to use '--will-cite'.\n\n";
 		last;
 	    }
 	}
@@ -7471,11 +7476,19 @@ sub set_exitsignal {
 
 {
     my $status_printed;
+    my $total_jobs;
 
     sub should_we_halt {
 	# Should we halt? Immediately? Gracefully?
 	# Returns: N/A
 	my $job = shift;
+	#  --halt # => 1..100 (number of jobs failed, 101 means > 100)
+	#  --halt % => 1..100 (pct of jobs failed)
+	if($Global::halt_pct and not $Global::halt_count) {
+	    $total_jobs ||= $Global::JobQueue->total_jobs();
+	    # From the pct compute the number of jobs that must fail/succeed
+	    $Global::halt_count = $total_jobs * $Global::halt_pct;
+	}
 	if($job->exitstatus() or $job->exitsignal()) {
 	    # Job failed
 	    $Global::exitstatus++;
@@ -7483,21 +7496,23 @@ sub set_exitsignal {
 	    if($Global::halt_fail) {
 		::status("$Global::progname: This job failed:\n",
 			 $job->replaced(),"\n");
-		if(($Global::halt_count and
-		    $Global::halt_count <= $Global::total_failed)
-		   or
-		   ($Global::halt_pct and
-		    $Global::halt_pct <= 
-		    $Global::total_failed / $Global::total_started
-		    and $Global::total_failed > 3)) {
+		if($Global::halt_count <= $Global::total_failed) {
 		    # At least N jobs had failed
-		    # or at least N% had failed and more than 3
-		    if($Global::halt_count and $Global::halt_count == 1) {
-			$Global::halt_exitstatus = $job->exitstatus();
-		    } else {
-			$Global::halt_exitstatus = ::min($Global::total_failed,254);
+		    if(not defined $Global::halt_exitstatus) {
+			if($Global::halt_pct) {
+			    # --halt now,fail=X% or soon,fail=X%
+			    $Global::halt_exitstatus =
+				::ceil($Global::total_failed / $total_jobs * 100);
+			} elsif($Global::halt_count) {
+			    # --halt now,fail=X or soon,fail=X
+			    $Global::halt_exitstatus = ::min($Global::total_failed,101);
+			}
+			if($Global::halt_count and $Global::halt_count == 1) {
+			    # --halt now,fail=1 or soon,fail=1
+			    $Global::halt_exitstatus = $job->exitstatus();
+			}
 		    }
-		    ::debug("halt","Pct: ",$Global::halt_pct,"<=",$Global::total_failed / $Global::total_started," count: ",$Global::halt_count,"\n");
+		    ::debug("halt","Pct: ",$Global::halt_pct," count: ",$Global::halt_count,"\n");
 		    if($Global::halt_when eq "soon"
 		       and scalar(keys %Global::running) > 0) {
 			::status
@@ -7511,20 +7526,14 @@ sub set_exitsignal {
 	    }
 	} else {
 	    if($Global::halt_success) {
-		$Global::halt_exitstatus = $Global::total_failed;
+		::debug("halt","Pct: ",$Global::halt_pct,"<=",
+			" count: ",$Global::halt_count,"\n");
 		::status("$Global::progname: This job succeeded:\n",
 			 $job->replaced(),"\n");
-		if(($Global::halt_count and
-		    $Global::halt_count <=
-		    $Global::total_completed-$Global::total_failed)
-		   or
-		   ($Global::halt_pct and
-		    $Global::halt_pct <=
-		    ($Global::total_completed-$Global::total_failed)
-		    / $Global::total_completed
-		    and ($Global::total_completed-$Global::total_failed) > 3)) {
+		if($Global::halt_count <=
+		   $Global::total_completed-$Global::total_failed) {
 		    # At least N jobs had success
-		    # or at least N% had success and more than 3
+		    # or at least N% had success
 		    $Global::halt_exitstatus = 0;
 		    if($Global::halt_when eq "soon"
 		       and scalar(keys %Global::running) > 0) {
diff --git a/src/parallel.pod b/src/parallel.pod
index 2ab37e11..5dd5a282 100644
--- a/src/parallel.pod
+++ b/src/parallel.pod
@@ -422,9 +422,19 @@ Implies B<--semaphore>.
 
 =item B<--bibtex>
 
-Print the BibTeX entry for GNU B and disable citation
+Print the BibTeX entry for GNU B and silence citation
 notice.
 
+If it is impossible for you to run B<--bibtex> you can use
+B<--will-cite>.
+
+If you use B<--will-cite> in scripts to be run by others you are
+making it harder for others to see the citation notice.  The
+development of GNU B is indirectly financed through
+citations, so if users do not know they should cite then that makes it
+harder to finance development. However, if you pay 10000 EUR, you
+should feel free to use B<--will-cite>.
+
 
 =item B<--block> I
 
@@ -691,42 +701,80 @@ Print a summary of the options to GNU B and exit.
 
 =item B<--halt> I
 
-How should GNU B terminate?
+When should GNU B terminate? In some situations it makes no
+sense to run all jobs. GNU B should simply give up as soon
+as a condition is met.
+
+I defaults to B, which runs all jobs no matter what.
+
+I can also take on the form of I,I.
+
+I can be 'now' which means kill all running jobs and halt
+immediately, or it can be 'soon' which means wait for all running jobs
+to complete, but start no new jobs.
+
+I can be 'fail=X', 'fail=Y%', 'success=X', or 'success=Y%' where
+X is the number of jobs that has to fail or succeed before halting,
+and Y is the percentage of jobs that has to fail or succeed before
+halting.
+
+Example:
+
+=over 23
+
+=item Z<> --halt now,fail=1
+
+exit when the first job fails. Kill running jobs.
+
+=item Z<> --halt soon,fail=3
+
+exit when 3 jobs fail, but wait for running jobs to complete.
+
+=item Z<> --halt soon,fail=3%
+
+exit when 3% of the jobs have failed, but wait for running jobs to complete.
+
+=item Z<> --halt now,success=1
+
+exit when a job succeeds. Kill running jobs.
+
+=item Z<> --halt soon,success=3
+
+exit when 3 jobs succeeds, but wait for running jobs to complete.
+
+=item Z<> --halt now,success=3%
+
+exit when 3% of the jobs have succeeded. Kill running jobs.
+
+=back
+
+For backwards compability these also work:
 
 =over 7
 
 =item Z<>0
 
-Do not halt if a job fails. Exit status will be the number of jobs
-failed. This is the default.
+never
 
 =item Z<>1
 
-Do not start new jobs if a job fails, but complete the running jobs
-including cleanup. The exit status will be the exit status from the
-last failing job.
+soon,fail=1
 
 =item Z<>2
 
-Kill off all jobs immediately and exit without cleanup. The exit
-status will be the exit status from the failing job.
+now,fail=1
 
 =item Z<>-1
 
-Do not start new jobs if a job succeeds, but complete the running jobs
-including cleanup. The exit status will be the exit status from the
-last failing job if any.
+soon,success=1
 
 =item Z<>-2
 
-Kill off all jobs immediately and exit without cleanup. The exit
-status will be 0.
+now,success=1
 
 =item Z<>1-99%
 
-If I% of the jobs fail and minimum 3: Do not start new jobs, but
-complete the running jobs including cleanup. The exit status will be
-the exit status from the last failing job.
+soon,fail=1-99%
 
 =back
 
@@ -3482,21 +3530,25 @@ remote computers:
 
 =head1 EXIT STATUS
 
-If B<--halt-on-error> 0 or not specified:
+Exit status depends on B<--halt-on-error> if one of these are used:
+success=X, success=Y%, fail=Y%.
 
 =over 6
 
 =item Z<>0
 
-All jobs ran without error.
+All jobs ran without error. If success=X is used: X jobs ran without
+error. If success=Y% is used: Y% of the jobs ran without error.
 
-=item Z<>1-253
+=item Z<>1-100
 
-Some of the jobs failed. The exit status gives the number of failed jobs
+Some of the jobs failed. The exit status gives the number of failed
+jobs. If Y% is used the exit status is the percentage of jobs that
+failed.
 
-=item Z<>254
+=item Z<>101
 
-More than 253 jobs failed.
+More than 100 jobs failed.
 
 =item Z<>255
 
@@ -3504,7 +3556,8 @@ Other error.
 
 =back
 
-If B<--halt-on-error> 1 or 2: Exit status of the failing job.
+If fail=1 is used, the exit status will be the exit status of the
+failing job.
 
 
 =head1 DIFFERENCES BETWEEN GNU Parallel AND ALTERNATIVES
@@ -3533,7 +3586,7 @@ Manipulation of input
  M3. Arguments can be put anywhere in the execution line
  M4. Multiple arguments can be put anywhere in the execution line
  M5. Arguments can be replaced with context
- M6. Input can be treated as complete execution line
+ M6. Input can be treated as the complete command line
 
 Outputs
  O1. Grouping output so output from different jobs do not mix
diff --git a/testsuite/tests-to-run/parallel-local-3s.sh b/testsuite/tests-to-run/parallel-local-3s.sh
index 09b236cb..61167988 100644
--- a/testsuite/tests-to-run/parallel-local-3s.sh
+++ b/testsuite/tests-to-run/parallel-local-3s.sh
@@ -24,7 +24,7 @@ echo '### Test --halt-on-error 1';
   (echo "sleep 1;true"; echo "sleep 2;false";echo "sleep 3;true") | parallel -j10 --halt-on-error 1; 
   echo $?; 
 
-  (echo "sleep 1;true"; echo "sleep 2;false";echo "sleep 3;true";echo "sleep 4; non_exist") | parallel -j10 --halt 1; 
+  (echo "sleep 1;true"; echo "sleep 2; non_exist";echo "sleep 3;true";echo "sleep 4; false") | parallel -j10 --halt 1; 
   echo $?
 
 echo '**'
@@ -56,20 +56,20 @@ echo '### Test --halt -2';
 
 echo '**'
 
-echo '### Test last dying print --halt-on-error 1'; 
-  (seq 0 8;echo 0; echo 9) | parallel -j10 -kq --halt 1 perl -e 'sleep $ARGV[0];print STDERR @ARGV,"\n"; exit shift'; 
+echo '### Test first dying print --halt-on-error 1'; 
+  (echo 0; echo 3; seq 0 7;echo 0; echo 8) | parallel -j10 -kq --halt 1 perl -e 'sleep $ARGV[0];print STDERR @ARGV,"\n"; exit shift'; 
   echo exit code $?
 
 echo '### Test last dying print --halt-on-error 2'; 
-  (seq 0 8;echo 0; echo 9) | parallel -j10 -kq --halt 2 perl -e 'sleep $ARGV[0];print STDERR @ARGV,"\n"; exit shift'; 
+  (echo 0; echo 3; seq 0 7;echo 0; echo 8) | parallel -j10 -kq --halt 2 perl -e 'sleep $ARGV[0];print STDERR @ARGV,"\n"; exit shift'; 
   echo exit code $?
 
 echo '### Test last dying print --halt-on-error -1'; 
-  (seq 0 8;echo 0; echo 9) | parallel -j10 -kq --halt -1 perl -e 'sleep $ARGV[0];print STDERR @ARGV,"\n"; exit not shift'; 
+  (echo 0; echo 3; seq 0 7;echo 0; echo 8) | parallel -j10 -kq --halt -1 perl -e 'sleep $ARGV[0];print STDERR @ARGV,"\n"; exit not shift'; 
   echo exit code $?
 
 echo '### Test last dying print --halt-on-error -2'; 
-  (seq 0 8;echo 0; echo 9) | parallel -j10 -kq --halt -2 perl -e 'sleep $ARGV[0];print STDERR @ARGV,"\n"; exit not shift'; 
+  (echo 0; echo 3; seq 0 7;echo 0; echo 8) | parallel -j10 -kq --halt -2 perl -e 'sleep $ARGV[0];print STDERR @ARGV,"\n"; exit not shift'; 
   echo exit code $?
 
 echo '**'
diff --git a/testsuite/tests-to-run/parallel-local-ssh2.sh b/testsuite/tests-to-run/parallel-local-ssh2.sh
index ec173d53..7a8306da 100644
--- a/testsuite/tests-to-run/parallel-local-ssh2.sh
+++ b/testsuite/tests-to-run/parallel-local-ssh2.sh
@@ -3,7 +3,7 @@
 # /tmp/parallel-local-ssh2 will by default be owned by me and should be writable by *@localhost
 chmod 777 "$TMPDIR" 2>/dev/null
 
-cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | stdout parallel -vj5 -k --joblog /tmp/jl-`basename $0` -L1
+cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | stdout parallel -vj5 --retries 3 -k --joblog /tmp/jl-`basename $0` -L1
 echo "### bug #43518: GNU Parallel doesn't obey servers' jobs capacity when an ssh login file is reloaded"
   # Pre-20141106 Would reset the number of jobs run on all sshlogin if --slf changed
   # Thus must take at least 25 sec to run
diff --git a/testsuite/tests-to-run/parallel-tutorial.sh b/testsuite/tests-to-run/parallel-tutorial.sh
index f295b232..00c3726e 100644
--- a/testsuite/tests-to-run/parallel-tutorial.sh
+++ b/testsuite/tests-to-run/parallel-tutorial.sh
@@ -33,6 +33,8 @@ perl -ne '$/="\n\n"; /^Output/../^[^O]\S/ and next; /^  / and print;' ../../src/
             s/... ... .. ..:..:.. \D+ ..../DATE OUTPUT/;
             # Timestamp from --joblog
             s/\d{10}.\d{3}\s+..\d+/TIMESTAMP\t9.999/g;
+            # Version
+            s/201\d{5}/VERSION/g;
             # Remote script
             s/(PARALLEL_PID\D+)\d+/${1}000000/g;
             # /usr/bin/time -f %e
diff --git a/testsuite/wanted-results/parallel-local-3s b/testsuite/wanted-results/parallel-local-3s
index f2010a0a..11c7561f 100644
--- a/testsuite/wanted-results/parallel-local-3s
+++ b/testsuite/wanted-results/parallel-local-3s
@@ -13,19 +13,19 @@ echo '### Test --halt-on-error 0';   (echo "sleep 1;true"; echo "sleep 2;false";
 /bin/bash: non_exist: command not found
 echo '**'
 **
-echo '### Test --halt-on-error 1';   (echo "sleep 1;true"; echo "sleep 2;false";echo "sleep 3;true") | parallel -j10 --halt-on-error 1;   echo $?;   (echo "sleep 1;true"; echo "sleep 2;false";echo "sleep 3;true";echo "sleep 4; non_exist") | parallel -j10 --halt 1;   echo $?
+echo '### Test --halt-on-error 1';   (echo "sleep 1;true"; echo "sleep 2;false";echo "sleep 3;true") | parallel -j10 --halt-on-error 1;   echo $?;   (echo "sleep 1;true"; echo "sleep 2; non_exist";echo "sleep 3;true";echo "sleep 4; false") | parallel -j10 --halt 1;   echo $?
 ### Test --halt-on-error 1
 1
 127
 parallel: This job failed:
 sleep 2;false
 parallel: Starting no more jobs. Waiting for 1 jobs to finish.
-parallel: This job failed:
-sleep 2;false
-parallel: Starting no more jobs. Waiting for 2 jobs to finish.
 /bin/bash: non_exist: command not found
 parallel: This job failed:
-sleep 4; non_exist
+sleep 2; non_exist
+parallel: Starting no more jobs. Waiting for 2 jobs to finish.
+parallel: This job failed:
+sleep 4; false
 echo '**'
 **
 echo '### Test --halt-on-error 2';   (echo "sleep 1;true"; echo "sleep 2;false";echo "sleep 3;true") | parallel -j10 --halt-on-error 2;   echo $?;   (echo "sleep 1;true"; echo "sleep 2;false";echo "sleep 3;true";echo "sleep 4; non_exist") | parallel -j10 --halt 2;   echo $?
@@ -61,98 +61,98 @@ parallel: This job succeeded:
 sleep 2;true
 echo '**'
 **
-echo '### Test last dying print --halt-on-error 1';   (seq 0 8;echo 0; echo 9) | parallel -j10 -kq --halt 1 perl -e 'sleep $ARGV[0];print STDERR @ARGV,"\n"; exit shift';   echo exit code $?
-### Test last dying print --halt-on-error 1
-exit code 9
+echo '### Test first dying print --halt-on-error 1';   (echo 0; echo 3; seq 0 7;echo 0; echo 8) | parallel -j10 -kq --halt 1 perl -e 'sleep $ARGV[0];print STDERR @ARGV,"\n"; exit shift';   echo exit code $?
+### Test first dying print --halt-on-error 1
+exit code 1
 0
-1
 parallel: This job failed:
 perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 1
 parallel: Starting no more jobs. Waiting for 8 jobs to finish.
-2
 parallel: This job failed:
 perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 2
 parallel: Starting no more jobs. Waiting for 7 jobs to finish.
 3
+0
+1
+2
 parallel: This job failed:
 perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 3
 parallel: Starting no more jobs. Waiting for 6 jobs to finish.
+3
+parallel: This job failed:
+perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 3
+parallel: Starting no more jobs. Waiting for 5 jobs to finish.
 4
 parallel: This job failed:
 perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 4
-parallel: Starting no more jobs. Waiting for 5 jobs to finish.
+parallel: Starting no more jobs. Waiting for 4 jobs to finish.
 5
 parallel: This job failed:
 perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 5
-parallel: Starting no more jobs. Waiting for 4 jobs to finish.
+parallel: Starting no more jobs. Waiting for 3 jobs to finish.
 6
 parallel: This job failed:
 perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 6
-parallel: Starting no more jobs. Waiting for 3 jobs to finish.
-7
-parallel: This job failed:
-perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 7
 parallel: Starting no more jobs. Waiting for 2 jobs to finish.
-8
+7
 0
 parallel: This job failed:
-perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 8
+perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 7
 parallel: Starting no more jobs. Waiting for 1 jobs to finish.
-9
+8
 parallel: This job failed:
-perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 9
-echo '### Test last dying print --halt-on-error 2';   (seq 0 8;echo 0; echo 9) | parallel -j10 -kq --halt 2 perl -e 'sleep $ARGV[0];print STDERR @ARGV,"\n"; exit shift';   echo exit code $?
+perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 8
+echo '### Test last dying print --halt-on-error 2';   (echo 0; echo 3; seq 0 7;echo 0; echo 8) | parallel -j10 -kq --halt 2 perl -e 'sleep $ARGV[0];print STDERR @ARGV,"\n"; exit shift';   echo exit code $?
 ### Test last dying print --halt-on-error 2
 exit code 1
 0
-1
 parallel: This job failed:
 perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 1
-echo '### Test last dying print --halt-on-error -1';   (seq 0 8;echo 0; echo 9) | parallel -j10 -kq --halt -1 perl -e 'sleep $ARGV[0];print STDERR @ARGV,"\n"; exit not shift';   echo exit code $?
+echo '### Test last dying print --halt-on-error -1';   (echo 0; echo 3; seq 0 7;echo 0; echo 8) | parallel -j10 -kq --halt -1 perl -e 'sleep $ARGV[0];print STDERR @ARGV,"\n"; exit not shift';   echo exit code $?
 ### Test last dying print --halt-on-error -1
 exit code 0
 0
-1
 parallel: This job succeeded:
 perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ not\ shift 1
 parallel: Starting no more jobs. Waiting for 8 jobs to finish.
-2
 parallel: This job succeeded:
 perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ not\ shift 2
 parallel: Starting no more jobs. Waiting for 7 jobs to finish.
 3
+0
+1
+2
 parallel: This job succeeded:
 perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ not\ shift 3
 parallel: Starting no more jobs. Waiting for 6 jobs to finish.
+3
+parallel: This job succeeded:
+perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ not\ shift 3
+parallel: Starting no more jobs. Waiting for 5 jobs to finish.
 4
 parallel: This job succeeded:
 perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ not\ shift 4
-parallel: Starting no more jobs. Waiting for 5 jobs to finish.
+parallel: Starting no more jobs. Waiting for 4 jobs to finish.
 5
 parallel: This job succeeded:
 perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ not\ shift 5
-parallel: Starting no more jobs. Waiting for 4 jobs to finish.
+parallel: Starting no more jobs. Waiting for 3 jobs to finish.
 6
 parallel: This job succeeded:
 perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ not\ shift 6
-parallel: Starting no more jobs. Waiting for 3 jobs to finish.
-7
-parallel: This job succeeded:
-perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ not\ shift 7
 parallel: Starting no more jobs. Waiting for 2 jobs to finish.
-8
+7
 0
 parallel: This job succeeded:
-perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ not\ shift 8
+perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ not\ shift 7
 parallel: Starting no more jobs. Waiting for 1 jobs to finish.
-9
+8
 parallel: This job succeeded:
-perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ not\ shift 9
-echo '### Test last dying print --halt-on-error -2';   (seq 0 8;echo 0; echo 9) | parallel -j10 -kq --halt -2 perl -e 'sleep $ARGV[0];print STDERR @ARGV,"\n"; exit not shift';   echo exit code $?
+perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ not\ shift 8
+echo '### Test last dying print --halt-on-error -2';   (echo 0; echo 3; seq 0 7;echo 0; echo 8) | parallel -j10 -kq --halt -2 perl -e 'sleep $ARGV[0];print STDERR @ARGV,"\n"; exit not shift';   echo exit code $?
 ### Test last dying print --halt-on-error -2
 exit code 0
 0
-1
 parallel: This job succeeded:
 perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ not\ shift 1
 echo '**'
@@ -161,45 +161,9 @@ testhalt() {   echo '### testhalt --halt '$1;   (yes 0 | head -n 10; seq 10) | s
 ### testhalt --halt now,fail=0
 parallel: This job failed:
 sleep 0.3; exit 1
-parallel: This job failed:
-sleep 0.6; exit 2
-parallel: This job failed:
-sleep 0.9; exit 3
-parallel: This job failed:
-sleep 1.2; exit 4
-parallel: This job failed:
-sleep 1.5; exit 5
-parallel: This job failed:
-sleep 1.8; exit 6
-parallel: This job failed:
-sleep 2.1; exit 7
-parallel: This job failed:
-sleep 2.4; exit 8
-parallel: This job failed:
-sleep 2.7; exit 9
-parallel: This job failed:
-sleep 3; exit 10
 0
 parallel: This job failed:
 sleep 0.3; exit 1
-parallel: This job failed:
-sleep 0.6; exit 2
-parallel: This job failed:
-sleep 0.9; exit 3
-parallel: This job failed:
-sleep 1.2; exit 4
-parallel: This job failed:
-sleep 1.5; exit 5
-parallel: This job failed:
-sleep 1.8; exit 6
-parallel: This job failed:
-sleep 2.1; exit 7
-parallel: This job failed:
-sleep 2.4; exit 8
-parallel: This job failed:
-sleep 2.7; exit 9
-parallel: This job failed:
-sleep 3; exit 10
 0
 ### testhalt --halt now,fail=1
 parallel: This job failed:
@@ -232,7 +196,7 @@ parallel: This job failed:
 sleep 1.5; exit 5
 parallel: This job failed:
 sleep 1.8; exit 6
-6
+30
 parallel: This job failed:
 sleep 0.3; exit 1
 parallel: This job failed:
@@ -241,7 +205,11 @@ parallel: This job failed:
 sleep 0.9; exit 3
 parallel: This job failed:
 sleep 1.2; exit 4
-4
+parallel: This job failed:
+sleep 1.5; exit 5
+parallel: This job failed:
+sleep 1.8; exit 6
+30
 ### testhalt --halt now,fail=70%
 parallel: This job failed:
 sleep 0.3; exit 1
@@ -263,7 +231,7 @@ parallel: This job failed:
 sleep 2.7; exit 9
 parallel: This job failed:
 sleep 3; exit 10
-0
+50
 parallel: This job failed:
 sleep 0.3; exit 1
 parallel: This job failed:
@@ -278,49 +246,37 @@ parallel: This job failed:
 sleep 1.8; exit 6
 parallel: This job failed:
 sleep 2.1; exit 7
-7
+parallel: This job failed:
+sleep 2.4; exit 8
+parallel: This job failed:
+sleep 2.7; exit 9
+parallel: This job failed:
+sleep 3; exit 10
+50
 ### testhalt --halt soon,fail=0
 parallel: This job failed:
 sleep 0.3; exit 1
+parallel: Starting no more jobs. Waiting for 3 jobs to finish.
 parallel: This job failed:
 sleep 0.6; exit 2
+parallel: Starting no more jobs. Waiting for 2 jobs to finish.
 parallel: This job failed:
 sleep 0.9; exit 3
+parallel: Starting no more jobs. Waiting for 1 jobs to finish.
 parallel: This job failed:
 sleep 1.2; exit 4
-parallel: This job failed:
-sleep 1.5; exit 5
-parallel: This job failed:
-sleep 1.8; exit 6
-parallel: This job failed:
-sleep 2.1; exit 7
-parallel: This job failed:
-sleep 2.4; exit 8
-parallel: This job failed:
-sleep 2.7; exit 9
-parallel: This job failed:
-sleep 3; exit 10
 0
 parallel: This job failed:
 sleep 0.3; exit 1
+parallel: Starting no more jobs. Waiting for 3 jobs to finish.
 parallel: This job failed:
 sleep 0.6; exit 2
+parallel: Starting no more jobs. Waiting for 2 jobs to finish.
 parallel: This job failed:
 sleep 0.9; exit 3
+parallel: Starting no more jobs. Waiting for 1 jobs to finish.
 parallel: This job failed:
 sleep 1.2; exit 4
-parallel: This job failed:
-sleep 1.5; exit 5
-parallel: This job failed:
-sleep 1.8; exit 6
-parallel: This job failed:
-sleep 2.1; exit 7
-parallel: This job failed:
-sleep 2.4; exit 8
-parallel: This job failed:
-sleep 2.7; exit 9
-parallel: This job failed:
-sleep 3; exit 10
 0
 ### testhalt --halt soon,fail=1
 parallel: This job failed:
@@ -334,7 +290,7 @@ sleep 0.9; exit 3
 parallel: Starting no more jobs. Waiting for 1 jobs to finish.
 parallel: This job failed:
 sleep 1.2; exit 4
-4
+1
 parallel: This job failed:
 sleep 0.3; exit 1
 parallel: Starting no more jobs. Waiting for 3 jobs to finish.
@@ -346,7 +302,7 @@ sleep 0.9; exit 3
 parallel: Starting no more jobs. Waiting for 1 jobs to finish.
 parallel: This job failed:
 sleep 1.2; exit 4
-4
+1
 ### testhalt --halt soon,fail=2
 parallel: This job failed:
 sleep 0.3; exit 1
@@ -361,7 +317,7 @@ sleep 1.2; exit 4
 parallel: Starting no more jobs. Waiting for 1 jobs to finish.
 parallel: This job failed:
 sleep 1.5; exit 5
-5
+2
 parallel: This job failed:
 sleep 0.3; exit 1
 parallel: This job failed:
@@ -375,7 +331,7 @@ sleep 1.2; exit 4
 parallel: Starting no more jobs. Waiting for 1 jobs to finish.
 parallel: This job failed:
 sleep 1.5; exit 5
-5
+2
 ### testhalt --halt soon,fail=30%
 parallel: This job failed:
 sleep 0.3; exit 1
@@ -398,7 +354,7 @@ sleep 2.4; exit 8
 parallel: Starting no more jobs. Waiting for 1 jobs to finish.
 parallel: This job failed:
 sleep 2.7; exit 9
-9
+30
 parallel: This job failed:
 sleep 0.3; exit 1
 parallel: This job failed:
@@ -407,16 +363,20 @@ parallel: This job failed:
 sleep 0.9; exit 3
 parallel: This job failed:
 sleep 1.2; exit 4
-parallel: Starting no more jobs. Waiting for 3 jobs to finish.
 parallel: This job failed:
 sleep 1.5; exit 5
-parallel: Starting no more jobs. Waiting for 2 jobs to finish.
 parallel: This job failed:
 sleep 1.8; exit 6
-parallel: Starting no more jobs. Waiting for 1 jobs to finish.
+parallel: Starting no more jobs. Waiting for 3 jobs to finish.
 parallel: This job failed:
 sleep 2.1; exit 7
-7
+parallel: Starting no more jobs. Waiting for 2 jobs to finish.
+parallel: This job failed:
+sleep 2.4; exit 8
+parallel: Starting no more jobs. Waiting for 1 jobs to finish.
+parallel: This job failed:
+sleep 2.7; exit 9
+30
 ### testhalt --halt soon,fail=70%
 parallel: This job failed:
 sleep 0.3; exit 1
@@ -438,7 +398,7 @@ parallel: This job failed:
 sleep 2.7; exit 9
 parallel: This job failed:
 sleep 3; exit 10
-0
+50
 parallel: This job failed:
 sleep 0.3; exit 1
 parallel: This job failed:
@@ -453,59 +413,20 @@ parallel: This job failed:
 sleep 1.8; exit 6
 parallel: This job failed:
 sleep 2.1; exit 7
-parallel: Starting no more jobs. Waiting for 3 jobs to finish.
 parallel: This job failed:
 sleep 2.4; exit 8
-parallel: Starting no more jobs. Waiting for 2 jobs to finish.
 parallel: This job failed:
 sleep 2.7; exit 9
-parallel: Starting no more jobs. Waiting for 1 jobs to finish.
 parallel: This job failed:
 sleep 3; exit 10
-10
+50
 ### testhalt --halt now,success=0
 parallel: This job succeeded:
 sleep 0; exit 0
-parallel: This job succeeded:
-sleep 0; exit 0
-parallel: This job succeeded:
-sleep 0; exit 0
-parallel: This job succeeded:
-sleep 0; exit 0
-parallel: This job succeeded:
-sleep 0; exit 0
-parallel: This job succeeded:
-sleep 0; exit 0
-parallel: This job succeeded:
-sleep 0; exit 0
-parallel: This job succeeded:
-sleep 0; exit 0
-parallel: This job succeeded:
-sleep 0; exit 0
-parallel: This job succeeded:
-sleep 0; exit 0
 0
 parallel: This job succeeded:
 sleep 0; exit 0
-parallel: This job succeeded:
-sleep 0; exit 0
-parallel: This job succeeded:
-sleep 0; exit 0
-parallel: This job succeeded:
-sleep 0; exit 0
-parallel: This job succeeded:
-sleep 0; exit 0
-parallel: This job succeeded:
-sleep 0; exit 0
-parallel: This job succeeded:
-sleep 0; exit 0
-parallel: This job succeeded:
-sleep 0; exit 0
-parallel: This job succeeded:
-sleep 0; exit 0
-parallel: This job succeeded:
-sleep 0; exit 0
-7
+0
 ### testhalt --halt now,success=1
 parallel: This job succeeded:
 sleep 0; exit 0
@@ -533,6 +454,10 @@ parallel: This job succeeded:
 sleep 0; exit 0
 parallel: This job succeeded:
 sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
 0
 parallel: This job succeeded:
 sleep 0; exit 0
@@ -542,6 +467,10 @@ parallel: This job succeeded:
 sleep 0; exit 0
 parallel: This job succeeded:
 sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
 0
 ### testhalt --halt now,success=70%
 parallel: This job succeeded:
@@ -552,7 +481,19 @@ parallel: This job succeeded:
 sleep 0; exit 0
 parallel: This job succeeded:
 sleep 0; exit 0
-0
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
+50
 parallel: This job succeeded:
 sleep 0; exit 0
 parallel: This job succeeded:
@@ -573,50 +514,24 @@ parallel: This job succeeded:
 sleep 0; exit 0
 parallel: This job succeeded:
 sleep 0; exit 0
-7
+50
 ### testhalt --halt soon,success=0
 parallel: This job succeeded:
 sleep 0; exit 0
+parallel: Starting no more jobs. Waiting for 3 jobs to finish.
 parallel: This job succeeded:
 sleep 0; exit 0
+parallel: Starting no more jobs. Waiting for 2 jobs to finish.
 parallel: This job succeeded:
 sleep 0; exit 0
-parallel: This job succeeded:
-sleep 0; exit 0
-parallel: This job succeeded:
-sleep 0; exit 0
-parallel: This job succeeded:
-sleep 0; exit 0
-parallel: This job succeeded:
-sleep 0; exit 0
-parallel: This job succeeded:
-sleep 0; exit 0
-parallel: This job succeeded:
-sleep 0; exit 0
+parallel: Starting no more jobs. Waiting for 1 jobs to finish.
 parallel: This job succeeded:
 sleep 0; exit 0
 0
 parallel: This job succeeded:
 sleep 0; exit 0
-parallel: This job succeeded:
-sleep 0; exit 0
-parallel: This job succeeded:
-sleep 0; exit 0
-parallel: This job succeeded:
-sleep 0; exit 0
-parallel: This job succeeded:
-sleep 0; exit 0
-parallel: This job succeeded:
-sleep 0; exit 0
-parallel: This job succeeded:
-sleep 0; exit 0
-parallel: This job succeeded:
-sleep 0; exit 0
-parallel: This job succeeded:
-sleep 0; exit 0
-parallel: This job succeeded:
-sleep 0; exit 0
-7
+parallel: Starting no more jobs. Waiting for 3 jobs to finish.
+0
 ### testhalt --halt soon,success=1
 parallel: This job succeeded:
 sleep 0; exit 0
@@ -664,6 +579,10 @@ parallel: This job succeeded:
 sleep 0; exit 0
 parallel: This job succeeded:
 sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
 parallel: Starting no more jobs. Waiting for 3 jobs to finish.
 parallel: This job succeeded:
 sleep 0; exit 0
@@ -682,6 +601,10 @@ parallel: This job succeeded:
 sleep 0; exit 0
 parallel: This job succeeded:
 sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
 parallel: Starting no more jobs. Waiting for 3 jobs to finish.
 0
 ### testhalt --halt now,success=70%
@@ -693,7 +616,19 @@ parallel: This job succeeded:
 sleep 0; exit 0
 parallel: This job succeeded:
 sleep 0; exit 0
-0
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
+parallel: This job succeeded:
+sleep 0; exit 0
+50
 parallel: This job succeeded:
 sleep 0; exit 0
 parallel: This job succeeded:
@@ -714,7 +649,7 @@ parallel: This job succeeded:
 sleep 0; exit 0
 parallel: This job succeeded:
 sleep 0; exit 0
-7
+50
 echo '**'
 **
 echo '### Test slow arguments generation - https://savannah.gnu.org/bugs/?32834';   seq 1 3 | parallel -j1 "sleep 2; echo {}" | parallel -kj2 echo
diff --git a/testsuite/wanted-results/parallel-tutorial b/testsuite/wanted-results/parallel-tutorial
index bb8b8998..d1df6b10 100644
--- a/testsuite/wanted-results/parallel-tutorial
+++ b/testsuite/wanted-results/parallel-tutorial
@@ -495,19 +495,13 @@ echo 1; exit 1
 1
 2
 3
-4
-5
 parallel: This job failed:
 echo 1; exit 1
 parallel: This job failed:
 echo 2; exit 2
-parallel: This job failed:
-echo 3; exit 3
-parallel: This job failed:
-echo 4; exit 4
 parallel: Starting no more jobs. Waiting for 1 jobs to finish.
 parallel: This job failed:
-echo 5; exit 5
+echo 3; exit 3
   parallel -k --retries 3 'echo tried {} >>/tmp/runs; echo completed {}; exit {}' ::: 1 2 0
   cat /tmp/runs
 completed 1
@@ -939,7 +933,7 @@ This helps funding further development; and it won't cost you a cent.
 If you pay 10000 EUR you should feel free to use GNU Parallel without citing.
 
   parallel --version
-GNU parallel 20150502
+GNU parallel VERSION
 Copyright (C) 2007,2008,2009,2010,2011,2012,2013,2014,2015 Ole Tange
 and Free Software Foundation, Inc.
 License GPLv3+: GNU GPL version 3 or later 
@@ -950,9 +944,9 @@ Web site: http://www.gnu.org/software/parallel
 
 When using programs that use GNU Parallel to process data for publication
 please cite as described in 'parallel --bibtex'.
-  parallel --minversion 20130722 && echo Your version is at least 20130722.
-20150502
-Your version is at least 20130722.
+  parallel --minversion VERSION && echo Your version is at least VERSION.
+VERSION
+Your version is at least VERSION.
   parallel --bibtex
 Academic tradition requires you to cite works you base your article on.
 When using programs that use GNU Parallel to process data for publication

From 31564e97fccf2cebc008843d5063b21023f5af34 Mon Sep 17 00:00:00 2001
From: Ole Tange 
Date: Sun, 10 May 2015 21:38:57 +0200
Subject: [PATCH 05/14] sem.pod: toilet analogy added to explain semaphore.

---
 src/parallel     | 10 +++--
 src/parallel.pod |  2 +-
 src/sem.pod      | 99 ++++++++++++++++++++++++++++++++++++++----------
 3 files changed, 86 insertions(+), 25 deletions(-)

diff --git a/src/parallel b/src/parallel
index d68e582c..3cb0e3e9 100755
--- a/src/parallel
+++ b/src/parallel
@@ -1061,7 +1061,7 @@ sub parse_options {
 
 sub init_globals {
     # Defaults:
-    $Global::version = 20150503;
+    $Global::version = 20150509;
     $Global::progname = 'parallel';
     $Global::infinity = 2**31;
     $Global::debug = 0;
@@ -1243,8 +1243,12 @@ sub parse_semaphore {
     if(defined $opt::bg) { $Global::semaphore = 1; }
     if(defined $opt::wait) { $Global::semaphore = 1; }
     if($Global::semaphore) {
-        # A semaphore does not take input from neither stdin nor file
-        @opt::a = ("/dev/null");
+	if(@opt::a) {
+	    # A semaphore does not take input from neither stdin nor file
+	    ::error("A semaphore does not take input from neither stdin nor a file\n");
+	    ::wait_and_exit(255);
+	}
+	@opt::a = ("/dev/null");
         push(@Global::unget_argv, [Arg->new("")]);
         $Semaphore::timeout = $opt::semaphoretimeout || 0;
         if(defined $opt::semaphorename) {
diff --git a/src/parallel.pod b/src/parallel.pod
index 5dd5a282..cee83d99 100644
--- a/src/parallel.pod
+++ b/src/parallel.pod
@@ -861,7 +861,7 @@ Number of jobslots on each machine. Run up to N jobs in parallel.  0
 means as many as possible. Default is 100% which will run one job per
 CPU core on each machine.
 
-If B<--semaphore> is set default is 1 thus making a mutex.
+If B<--semaphore> is set, the default is 1 thus making a mutex.
 
 
 =item B<--jobs> I<+N>
diff --git a/src/sem.pod b/src/sem.pod
index 742536cf..c2a78f58 100755
--- a/src/sem.pod
+++ b/src/sem.pod
@@ -12,11 +12,14 @@ B [--fg] [--id ] [--semaphoretimeout ] [-j ] [--wait] comman
 
 GNU B is an alias for GNU B.
 
-It works as a tool for executing shell commands in parallel. GNU
-B acts as a counting semaphore. When GNU B is called with
-command it will start the command in the background. When I
-number of commands are running in the background, GNU B will wait
-for one of these to complete before starting another command.
+GNU B acts as a counting semaphore. When GNU B is called
+with command it starts the command in the background. When I
+number of commands are running in the background, GNU B waits for
+one of these to complete before starting the command.
+
+GNU B does not read any arguments to build the command (no -a,
+:::, and ::::). It simply waits for a semaphore to become available
+and then runs the command given.
 
 Before looking at the options you may want to check out the examples
 after the list of options. That will give you an idea of what GNU
@@ -28,23 +31,21 @@ B is capable of.
 
 =item I
 
-Command to execute. The command may be followed by arguments for the command.
+Command to execute. The command may be followed by arguments for the
+command.
 
 
 =item B<--bg>
 
-Run command in background thus GNU B will not wait for
+Run command in background thus GNU B will not wait for
 completion of the command before exiting. This is the default.
 
+In toilet analogy: GNU B waits for a toilet to be available,
+gives the toilet to a person, and exits immediately.
+
 See also: B<--fg>
 
 
-=item B<-j> I
-
-Run up to N commands in parallel. Default is 1 thus acting like a
-mutex.
-
-
 =item B<--jobs> I
 
 =item B<-j> I
@@ -56,6 +57,8 @@ mutex.
 Run up to N commands in parallel. Default is 1 thus acting like a
 mutex.
 
+In toilet analogy: B<-j> is the number of toilets.
+
 
 =item B<--jobs> I<+N>
 
@@ -122,29 +125,70 @@ are often a good value.
 
 The semaphore is stored in ~/.parallel/semaphores/
 
+In toilet analogy the name corresponds to different types of toilets:
+e.g. male, female, customer, staff.
+
 
 =item B<--fg>
 
 Do not put command in background.
 
+In toilet analogy: GNU B waits for a toilet to be available,
+takes a person to the toilet, waits for the person to finish, and
+exits.
+
 
 =item B<--semaphoretimeout> I (alpha testing)
 
 =item B<--st> I (alpha testing)
 
-If I > 0: If the semaphore is not released within I seconds, take it anyway.
+If I > 0: If the semaphore is not released within I
+seconds, take it anyway.
 
-If I < 0: If the semaphore is not released within I seconds, exit.
+If I < 0: If the semaphore is not released within I
+seconds, exit.
+
+In toilet analogy: I > 0: If no toilet becomes available within
+I seconds, pee on the floor. I < 0: If no toilet becomes
+available within I seconds, exit without doing anything.
 
 
 =item B<--wait>
 
-=item B<-w>
-
 Wait for all commands to complete.
 
+In toilet analogy: Wait until all toilets are empty, then exit.
+
+
 =back
 
+=head1 UNDERSTANDING A SEMAPHORE
+
+Try the following example:
+
+  sem -j 2 'sleep 1;echo 1 finished';   echo sem 1 exited
+  sem -j 2 'sleep 2;echo 2 finished';   echo sem 2 exited
+  sem -j 2 'sleep 3;echo 3 finished';   echo sem 3 exited
+  sem -j 2 'sleep 4;echo 4 finished';   echo sem 4 exited
+  sem --wait; echo sem --wait done
+
+In toilet analogy this uses 2 toilets (B<-j 2>). GNU B takes '1'
+to a toilet, and exits immediately. While '1' is sleeping, another GNU
+B takes '2' to a toilet, and exits immediately.
+
+While '1' and '2' are sleeping, another GNU B waits for a free
+toilet. When '1' finishes, a toilet becomes available, and this GNU
+B stops waiting, and takes '3' to a toilet, and exits
+immediately.
+
+While '2' and '3' are sleeping, another GNU B waits for a free
+toilet.  When '2' finishes, a toilet becomes available, and this GNU
+B stops waiting, and takes '4' to a toilet, and exits
+immediately.
+
+Finally another GNU B waits for all toilets to become free.
+
+
 =head1 EXAMPLE: Gzipping *.log
 
 Run one gzip process per CPU core. Block until a CPU core becomes
@@ -165,15 +209,28 @@ you run multiple pod2html in parallel (e.g. in a Makefile with make
 write to the files at the same time:
 
   # This may fail due to shared pod2htmd.tmp/pod2htmi.tmp files
-  pod2html foo.pod --outfile foo.html & pod2html bar.pod --outfile bar.html
+  foo.html:
+          pod2html foo.pod --outfile foo.html
+
+  bar.html:
+          pod2html bar.pod --outfile bar.html
+
+  $ make -j foo.html bar.html
 
 You need to protect pod2html from running twice at the same time.
 B running as a mutex will make sure only one runs:
 
-  sem --id pod2html pod2html foo.pod --outfile foo.html
-  sem --id pod2html pod2html bar.pod --outfile bar.html
-  sem --fg --id pod2html rm -f pod2htmd.tmp pod2htmi.tmp
+  foo.html:
+          sem --id pod2html pod2html foo.pod --outfile foo.html
 
+  bar.html:
+          sem --id pod2html pod2html bar.pod --outfile bar.html
+
+  clean: foo.html bar.html
+          sem --id pod2html --wait
+          rm -f pod2htmd.tmp pod2htmi.tmp
+
+  $ make -j foo.html bar.html clean
 
 =head1 BUGS
 

From 2549b9ba1a9863ef475bbc3affa25685ad4a5968 Mon Sep 17 00:00:00 2001
From: Ole Tange 
Date: Sun, 10 May 2015 22:23:13 +0200
Subject: [PATCH 06/14] parallel: added $PARALLEL_SSH.

---
 src/parallel                                  | 22 +++++++++++++------
 src/parallel.pod                              | 13 ++++++++---
 testsuite/tests-to-run/parallel-local-ssh1.sh |  5 +++++
 testsuite/wanted-results/parallel-local-ssh1  |  5 +++++
 4 files changed, 35 insertions(+), 10 deletions(-)

diff --git a/src/parallel b/src/parallel
index 3cb0e3e9..35030f8c 100755
--- a/src/parallel
+++ b/src/parallel
@@ -5387,6 +5387,8 @@ sub sshcommand_of_sshlogin {
     #   login@host
     my $self = shift;
     my ($sshcmd, $serverlogin);
+    # If $SSH is unset, use 'ssh'
+    $ENV{'PARALLEL_SSH'} ||= "ssh";
     if($self->{'string'} =~ /(.+) (\S+)$/) {
         # Own ssh command
         $sshcmd = $1; $serverlogin = $2;
@@ -5395,7 +5397,7 @@ sub sshcommand_of_sshlogin {
         if($opt::controlmaster) {
             # Use control_path to make ssh faster
             my $control_path = $self->control_path_dir()."/ssh-%r@%h:%p";
-            $sshcmd = "ssh -S ".$control_path;
+            $sshcmd = $ENV{'PARALLEL_SSH'}." -S ".$control_path;
             $serverlogin = $self->{'string'};
             if(not $self->{'control_path'}{$control_path}++) {
                 # Master is not running for this control_path
@@ -5407,18 +5409,24 @@ sub sshcommand_of_sshlogin {
 		    $SIG{'TERM'} = undef;
                     # Ignore the 'foo' being printed
                     open(STDOUT,">","/dev/null");
-                    # OpenSSH_3.6.1p2 gives 'tcgetattr: Invalid argument' with -tt
-                    # STDERR >/dev/null to ignore "process_mux_new_session: tcgetattr: Invalid argument"
+                    # With -tt OpenSSH_3.6.1p2 gives:
+		    # 'tcgetattr: Invalid argument'
+                    # STDERR >/dev/null to ignore
+		    # "process_mux_new_session: tcgetattr: Invalid argument"
                     open(STDERR,">","/dev/null");
                     open(STDIN,"<","/dev/null");
-                    # Run a sleep that outputs data, so it will discover if the ssh connection closes.
-                    my $sleep = ::shell_quote_scalar('$|=1;while(1){sleep 1;print "foo\n"}');
-                    my @master = ("ssh", "-tt", "-MTS", $control_path, $serverlogin, "perl", "-e", $sleep);
+                    # Run a sleep that outputs data, so it will discover
+		    # if the ssh connection closes.
+                    my $sleep = ::shell_quote_scalar
+			('$|=1;while(1){sleep 1;print "foo\n"}');
+                    my @master = ($ENV{'PARALLEL_SSH'}, "-tt", "-MTS",
+				  $control_path, $serverlogin, "perl", "-e",
+				  $sleep);
                     exec(@master);
                 }
             }
         } else {
-            $sshcmd = "ssh"; $serverlogin = $self->{'string'};
+            $sshcmd = $ENV{'PARALLEL_SSH'}; $serverlogin = $self->{'string'};
         }
     }
     $self->{'sshcommand'} = $sshcmd;
diff --git a/src/parallel.pod b/src/parallel.pod
index cee83d99..1cb648a6 100644
--- a/src/parallel.pod
+++ b/src/parallel.pod
@@ -697,9 +697,9 @@ See also: B<--line-buffer> B<--ungroup>
 Print a summary of the options to GNU B and exit.
 
 
-=item B<--halt-on-error> I
+=item B<--halt-on-error> I (alpha testing)
 
-=item B<--halt> I
+=item B<--halt> I (alpha testing)
 
 When should GNU B terminate? In some situations it makes no
 sense to run all jobs. GNU B should simply give up as soon
@@ -1665,7 +1665,7 @@ Does not run the command but quotes it. Useful for making quoted
 composed commands for GNU B.
 
 
-=item B<--shuf> (beta testing)
+=item B<--shuf> (alpha testing)
 
 Shuffle jobs. When having multiple input sources it is hard to
 randomize jobs. --shuf will generate all jobs, and shuffle them before
@@ -3439,6 +3439,13 @@ $SHELL. If undefined use:
 =back
 
 
+=item $PARALLEL_SSH (alpha testing)
+
+GNU B defaults to using B for remote access. This can
+be overridden with $PARALLEL_SSH. It can also be set on a per server
+basis (see B<--sshlogin>).
+
+
 =item $PARALLEL_SEQ
 
 $PARALLEL_SEQ will be set to the sequence number of the job
diff --git a/testsuite/tests-to-run/parallel-local-ssh1.sh b/testsuite/tests-to-run/parallel-local-ssh1.sh
index b25a7bcb..e35418b4 100644
--- a/testsuite/tests-to-run/parallel-local-ssh1.sh
+++ b/testsuite/tests-to-run/parallel-local-ssh1.sh
@@ -61,6 +61,11 @@ echo '### bug #40001: --joblog and --nonall seem not to work together:'
 echo '### bug #40132: FreeBSD: --workdir . gives warning if . == $HOME'
   cd && parallel --workdir . -S lo pwd ::: ""
 
+echo '### use function as $PARALLEL_SSH'
+  foossh() { echo "FOOSSH" >&2; ssh "$@"; }; 
+  export -f foossh; 
+  PARALLEL_SSH=foossh parallel -S 1/lo echo ::: 'Run through FOOSSH?'
+
 echo '### test filename :'
   echo content-of-: > :; 
   echo : | parallel -j1 --trc {}.{.} -S parallel@lo '(echo remote-{}.{.};cat {}) > {}.{.}'; 
diff --git a/testsuite/wanted-results/parallel-local-ssh1 b/testsuite/wanted-results/parallel-local-ssh1
index a6e668fe..35218e9b 100644
--- a/testsuite/wanted-results/parallel-local-ssh1
+++ b/testsuite/wanted-results/parallel-local-ssh1
@@ -84,6 +84,11 @@ echo '### bug #40132: FreeBSD: --workdir . gives warning if . == $HOME'
 ### bug #40132: FreeBSD: --workdir . gives warning if . == $HOME
   cd && parallel --workdir . -S lo pwd ::: ""
 /home/tange
+echo '### use function as $PARALLEL_SSH'
+### use function as $PARALLEL_SSH
+  foossh() { echo "FOOSSH" >&2; ssh "$@"; };   export -f foossh;   PARALLEL_SSH=foossh parallel -S 1/lo echo ::: 'Run through FOOSSH?'
+Run through FOOSSH?
+FOOSSH
 echo '### test filename :'
 ### test filename :
   echo content-of-: > :;   echo : | parallel -j1 --trc {}.{.} -S parallel@lo '(echo remote-{}.{.};cat {}) > {}.{.}';   cat :.:; rm : :.:

From d4c347c22b19c41aed03270b303712a13935eead Mon Sep 17 00:00:00 2001
From: Ole Tange 
Date: Sun, 10 May 2015 22:52:47 +0200
Subject: [PATCH 07/14] parallel: Implemented --ssh.

---
 src/parallel                                  | 11 ++++++-----
 src/parallel.pod                              | 12 ++++++++++--
 testsuite/tests-to-run/parallel-local-ssh1.sh |  5 +++++
 testsuite/wanted-results/parallel-local-ssh1  |  5 +++++
 4 files changed, 26 insertions(+), 7 deletions(-)

diff --git a/src/parallel b/src/parallel
index 35030f8c..225d1cb6 100755
--- a/src/parallel
+++ b/src/parallel
@@ -741,6 +741,7 @@ sub options_hash {
 	 "sshlogin|S=s" => \@opt::sshlogin,
 	 "sshloginfile|slf=s" => \@opt::sshloginfile,
 	 "controlmaster|M" => \$opt::controlmaster,
+	 "ssh=s" => \$opt::ssh,
 	 "return=s" => \@opt::return,
 	 "trc=s" => \@opt::trc,
 	 "transfer" => \$opt::transfer,
@@ -5387,8 +5388,8 @@ sub sshcommand_of_sshlogin {
     #   login@host
     my $self = shift;
     my ($sshcmd, $serverlogin);
-    # If $SSH is unset, use 'ssh'
-    $ENV{'PARALLEL_SSH'} ||= "ssh";
+    # If $opt::ssh is unset, use $PARALLEL_SSH or 'ssh'
+    $opt::ssh ||= $ENV{'PARALLEL_SSH'} || "ssh";
     if($self->{'string'} =~ /(.+) (\S+)$/) {
         # Own ssh command
         $sshcmd = $1; $serverlogin = $2;
@@ -5397,7 +5398,7 @@ sub sshcommand_of_sshlogin {
         if($opt::controlmaster) {
             # Use control_path to make ssh faster
             my $control_path = $self->control_path_dir()."/ssh-%r@%h:%p";
-            $sshcmd = $ENV{'PARALLEL_SSH'}." -S ".$control_path;
+            $sshcmd = $opt::ssh." -S ".$control_path;
             $serverlogin = $self->{'string'};
             if(not $self->{'control_path'}{$control_path}++) {
                 # Master is not running for this control_path
@@ -5419,14 +5420,14 @@ sub sshcommand_of_sshlogin {
 		    # if the ssh connection closes.
                     my $sleep = ::shell_quote_scalar
 			('$|=1;while(1){sleep 1;print "foo\n"}');
-                    my @master = ($ENV{'PARALLEL_SSH'}, "-tt", "-MTS",
+                    my @master = ($opt::ssh, "-tt", "-MTS",
 				  $control_path, $serverlogin, "perl", "-e",
 				  $sleep);
                     exec(@master);
                 }
             }
         } else {
-            $sshcmd = $ENV{'PARALLEL_SSH'}; $serverlogin = $self->{'string'};
+            $sshcmd = $opt::ssh; $serverlogin = $self->{'string'};
         }
     }
     $self->{'sshcommand'} = $sshcmd;
diff --git a/src/parallel.pod b/src/parallel.pod
index 1cb648a6..f6ec81d3 100644
--- a/src/parallel.pod
+++ b/src/parallel.pod
@@ -1679,6 +1679,13 @@ Do not use the first line of input (used by GNU B itself
 when called with B<--shebang>).
 
 
+=item B<--ssh> I (alpha testing)
+
+GNU B defaults to using B for remote access. This can
+be overridden with B<--ssh>. It can also be set on a per server
+basis (see B<--sshlogin>).
+
+
 =item B<--sshdelay> I
 
 Delay starting next ssh by I seconds. GNU B will pause
@@ -3442,8 +3449,9 @@ $SHELL. If undefined use:
 =item $PARALLEL_SSH (alpha testing)
 
 GNU B defaults to using B for remote access. This can
-be overridden with $PARALLEL_SSH. It can also be set on a per server
-basis (see B<--sshlogin>).
+be overridden with $PARALLEL_SSH, which again can be overridden with
+B<--ssh>. It can also be set on a per server basis (see
+B<--sshlogin>).
 
 
 =item $PARALLEL_SEQ
diff --git a/testsuite/tests-to-run/parallel-local-ssh1.sh b/testsuite/tests-to-run/parallel-local-ssh1.sh
index e35418b4..d39377b1 100644
--- a/testsuite/tests-to-run/parallel-local-ssh1.sh
+++ b/testsuite/tests-to-run/parallel-local-ssh1.sh
@@ -66,6 +66,11 @@ echo '### use function as $PARALLEL_SSH'
   export -f foossh; 
   PARALLEL_SSH=foossh parallel -S 1/lo echo ::: 'Run through FOOSSH?'
 
+echo '### use --ssh'
+  barssh() { echo "BARSSH" >&2; ssh "$@"; }; 
+  export -f barssh; 
+  parallel --ssh barssh -S 1/lo echo ::: 'Run through BARSSH?'
+
 echo '### test filename :'
   echo content-of-: > :; 
   echo : | parallel -j1 --trc {}.{.} -S parallel@lo '(echo remote-{}.{.};cat {}) > {}.{.}'; 
diff --git a/testsuite/wanted-results/parallel-local-ssh1 b/testsuite/wanted-results/parallel-local-ssh1
index 35218e9b..5f90675b 100644
--- a/testsuite/wanted-results/parallel-local-ssh1
+++ b/testsuite/wanted-results/parallel-local-ssh1
@@ -89,6 +89,11 @@ echo '### use function as $PARALLEL_SSH'
   foossh() { echo "FOOSSH" >&2; ssh "$@"; };   export -f foossh;   PARALLEL_SSH=foossh parallel -S 1/lo echo ::: 'Run through FOOSSH?'
 Run through FOOSSH?
 FOOSSH
+echo '### use --ssh'
+### use --ssh
+  barssh() { echo "BARSSH" >&2; ssh "$@"; };   export -f barssh;   parallel --ssh barssh -S 1/lo echo ::: 'Run through BARSSH?'
+Run through BARSSH?
+BARSSH
 echo '### test filename :'
 ### test filename :
   echo content-of-: > :;   echo : | parallel -j1 --trc {}.{.} -S parallel@lo '(echo remote-{}.{.};cat {}) > {}.{.}';   cat :.:; rm : :.:

From 09de088df9562d4c6a6aef39f407e1d604c1f43d Mon Sep 17 00:00:00 2001
From: Ole Tange 
Date: Thu, 14 May 2015 16:14:42 +0200
Subject: [PATCH 08/14] parallel: Cleaned up error() and warning().

---
 src/parallel | 227 ++++++++++++++++++++++++++-------------------------
 1 file changed, 117 insertions(+), 110 deletions(-)

diff --git a/src/parallel b/src/parallel
index 225d1cb6..90348671 100755
--- a/src/parallel
+++ b/src/parallel
@@ -393,7 +393,7 @@ sub spreadstdin {
 	      my $old_blocksize = $blocksize;
 	      $blocksize = ::min(ceil($blocksize * 1.3 + 1), $two_gb);
 	      ::warning("A record was longer than $old_blocksize. " .
-			"Increasing to --blocksize $blocksize\n");
+			"Increasing to --blocksize $blocksize.");
 	  }
       }
     }
@@ -930,7 +930,7 @@ sub parse_options {
     $opt::memfree = multiply_binary_prefix($opt::memfree);
     if(defined $opt::controlmaster) { $opt::noctrlc = 1; }
     if(defined $opt::timeout and $opt::timeout !~ /^\d+(\.\d+)?%?$/) {
-	::error("--timeout must be seconds or percentage\n");
+	::error("--timeout must be seconds or percentage.");
 	wait_and_exit(255);
     }
     if(defined $opt::minversion) {
@@ -1014,8 +1014,8 @@ sub parse_options {
     if(defined $opt::pipepart and
        (defined $opt::L or defined $opt::max_lines
 	or defined $opt::max_replace_args)) {
-	::error("--pipepart is incompatible with --max-replace-args, ",
-		"--max-lines, and -L.\n");
+	::error("--pipepart is incompatible with --max-replace-args, ".
+		"--max-lines, and -L.");
 	wait_and_exit(255);
     }
     if(grep /^$Global::arg_sep$|^$Global::arg_file_sep$/o, @ARGV) {
@@ -1031,14 +1031,14 @@ sub parse_options {
         $opt::progress = $opt::bar;
     }
     if(defined $opt::retired) {
-	    ::error("-g has been retired. Use --group.\n");
-	    ::error("-B has been retired. Use --bf.\n");
-	    ::error("-T has been retired. Use --tty.\n");
-	    ::error("-U has been retired. Use --er.\n");
-	    ::error("-W has been retired. Use --wd.\n");
-	    ::error("-Y has been retired. Use --shebang.\n");
-	    ::error("-H has been retired. Use --halt.\n");
-	    ::error("--tollef has been retired. Use -u -q --arg-sep -- and --load for -l.\n");
+	    ::error("-g has been retired. Use --group.",
+		    "-B has been retired. Use --bf.",
+		    "-T has been retired. Use --tty.",
+		    "-U has been retired. Use --er.",
+		    "-W has been retired. Use --wd.",
+		    "-Y has been retired. Use --shebang.",
+		    "-H has been retired. Use --halt.",
+		    "--tollef has been retired. Use -u -q --arg-sep -- and --load for -l.");
             ::wait_and_exit(255);
     }
     citation_notice();
@@ -1051,7 +1051,7 @@ sub parse_options {
         # As we do not know the max line length on the remote machine
         # long commands generated by xargs may fail
         # If $opt::max_replace_args is set, it is probably safe
-        ::warning("Using -X or -m with --sshlogin may fail.\n");
+        ::warning("Using -X or -m with --sshlogin may fail.");
     }
 
     if(not defined $opt::jobs) {
@@ -1110,7 +1110,7 @@ sub init_globals {
     $ENV{'TMPDIR'} ||= "/tmp";
     if(not $ENV{HOME}) {
 	# $ENV{HOME} is sometimes not set if called from PHP
-	::warning("\$HOME not set. Using /tmp\n");
+	::warning("\$HOME not set. Using /tmp.");
 	$ENV{HOME} = "/tmp";
     }
 }
@@ -1266,7 +1266,7 @@ sub parse_semaphore {
         }
 	if($Global::interactive and $opt::bg) {
 	    ::error("Jobs running in the ".
-		    "background cannot be interactive.\n");
+		    "background cannot be interactive.");
             ::wait_and_exit(255);
 	}
     }
@@ -1279,7 +1279,7 @@ sub record_env {
     if(open(my $vars_fh, ">", $ignore_filename)) {
 	print $vars_fh map { $_,"\n" } keys %ENV;
     } else {
-	::error("Cannot write to $ignore_filename.\n");
+	::error("Cannot write to $ignore_filename.");
 	::wait_and_exit(255);
     }
 }
@@ -1321,7 +1321,7 @@ sub open_joblog {
     if(($opt::resume or $opt::resume_failed)
        and
        not ($opt::joblog or $opt::results)) {
-        ::error("--resume and --resume-failed require --joblog or --results.\n");
+        ::error("--resume and --resume-failed require --joblog or --results.");
 	::wait_and_exit(255);
     }
     if($opt::joblog) {
@@ -1343,6 +1343,7 @@ sub open_joblog {
 			# This is 30% faster than set_job_already_run($1);
 			vec($Global::job_already_run,($1||0),1) = 1;
 		    } elsif(not /\d+\s+[^\s]+\s+([-0-9.]+\s+){6}/) {
+			chomp;
 			::error("Format of '$opt::joblog' is wrong: $_");
 			::wait_and_exit(255);
 		    }
@@ -1353,7 +1354,7 @@ sub open_joblog {
 	if($append) {
 	    # Append to joblog
 	    if(not open($Global::joblog, ">>", $opt::joblog)) {
-		::error("Cannot append to --joblog $opt::joblog.\n");
+		::error("Cannot append to --joblog $opt::joblog.");
 		::wait_and_exit(255);
 	    }
 	} else {
@@ -1362,7 +1363,7 @@ sub open_joblog {
 		$Global::joblog = $Global::fd{1};
 	    } elsif(not open($Global::joblog, ">", $opt::joblog)) {
 		# Overwrite the joblog
-		::error("Cannot write to --joblog $opt::joblog.\n");
+		::error("Cannot write to --joblog $opt::joblog.");
 		::wait_and_exit(255);
 	    }
 	    print $Global::joblog
@@ -1484,7 +1485,7 @@ sub read_options {
 		if(grep /^$profile$/, @config_profiles) {
 		    # config file is not required to exist
 		} else {
-		    ::error("$profile not readable.\n");
+		    ::error("$profile not readable.");
 		    wait_and_exit(255);
 		}
 	    }
@@ -1758,7 +1759,7 @@ sub open_or_exit {
     }
     my $fh = gensym;
     if(not open($fh, "<", $file)) {
-        ::error("Cannot open input file `$file': No such file or directory.\n");
+        ::error("Cannot open input file `$file': No such file or directory.");
         wait_and_exit(255);
     }
     return $fh;
@@ -2011,15 +2012,15 @@ sub init_run_jobs {
 		    # Count down the number of jobs to run for this SSHLogin.
 		    my $max = $sshlogin->max_jobs_running();
 		    if($max > 1) { $max--; } else {
-			::error("No more processes: cannot run a single job. Something is wrong.\n");
+			::error("No more processes: cannot run a single job. Something is wrong.");
 			::wait_and_exit(255);
 		    }
 		    $sshlogin->set_max_jobs_running($max);
 		    # Sleep up to 300 ms to give other processes time to die
 		    ::usleep(rand()*300);
-		    ::warning("No more processes: ",
-			      "Decreasing number of running jobs to $max. ",
-			      "Raising ulimit -u or /etc/security/limits.conf may help.\n");
+		    ::warning("No more processes: ".
+			      "Decreasing number of running jobs to $max.",
+			      "Raising ulimit -u or /etc/security/limits.conf may help.");
 		    return 0;
 		}
 	    }
@@ -2027,7 +2028,7 @@ sub init_run_jobs {
 	    # No more file handles
 	    $no_more_file_handles_warned++ or
 		::warning("No more file handles. ",
-			  "Raising ulimit -n or /etc/security/limits.conf may help.\n");
+			  "Raising ulimit -n or /etc/security/limits.conf may help.");
 	    return 0;
 	}
     }
@@ -2096,7 +2097,7 @@ sub drain_job_queue {
 	    # These jobs may not be started:
 	    # * because there the --filter-hosts has removed all
 	    if(not %Global::host) {
-		::error("There are no hosts left to run on.\n");
+		::error("There are no hosts left to run on.");
 		::wait_and_exit(255);
 	    }
 	    # * because of loadavg
@@ -2104,7 +2105,7 @@ sub drain_job_queue {
             start_more_jobs();
 	    $sleep = ::reap_usleep($sleep);
 	    if($Global::max_jobs_running == 0) {
-		::warning("There are no job slots available. Increase --jobs.\n");
+		::warning("There are no job slots available. Increase --jobs.");
 	    }
         }
     } while ($Global::total_running > 0
@@ -2473,7 +2474,7 @@ sub expand_slf_shorthand {
     } elsif(not -r $file) {
 	if(not -r $ENV{'HOME'}."/.parallel/".$file) {
 		# Try prepending ~/.parallel
-		::error("Cannot open $file.\n");
+		::error("Cannot open $file.");
 		::wait_and_exit(255);
 	} else {
 	    $file = $ENV{'HOME'}."/.parallel/".$file;
@@ -2499,7 +2500,7 @@ sub read_sshloginfile {
     } else {
 	if(not open($in_fh, "<", $file)) {
 	    # Try the filename
-	    ::error("Cannot open $file.\n");
+	    ::error("Cannot open $file.");
 	    ::wait_and_exit(255);
 	}
     }
@@ -2587,15 +2588,15 @@ sub parse_sshlogin {
         if(not remote_hosts()) {
             # There are no remote hosts
             if(@opt::trc) {
-		::warning("--trc ignored as there are no remote --sshlogin.\n");
+		::warning("--trc ignored as there are no remote --sshlogin.");
             } elsif (defined $opt::transfer) {
-		::warning("--transfer ignored as there are no remote --sshlogin.\n");
+		::warning("--transfer ignored as there are no remote --sshlogin.");
             } elsif (@opt::return) {
-                ::warning("--return ignored as there are no remote --sshlogin.\n");
+                ::warning("--return ignored as there are no remote --sshlogin.");
             } elsif (defined $opt::cleanup) {
-		::warning("--cleanup ignored as there are no remote --sshlogin.\n");
+		::warning("--cleanup ignored as there are no remote --sshlogin.");
             } elsif (@opt::basefile) {
-                ::warning("--basefile ignored as there are no remote --sshlogin.\n");
+                ::warning("--basefile ignored as there are no remote --sshlogin.");
             }
         }
     }
@@ -2624,7 +2625,7 @@ sub setup_basefile {
       if($sshlogin->string() eq ":") { next }
       for my $file (@opt::basefile) {
 	if($file !~ m:^/: and $opt::workdir eq "...") {
-	  ::error("Work dir '...' will not work with relative basefiles.\n");
+	  ::error("Work dir '...' will not work with relative basefiles.");
 	  ::wait_and_exit(255);
 	}
 	$workdir ||= Job->new("")->workdir();
@@ -2669,7 +2670,7 @@ sub filter_hosts {
 	    parse_host_filtering(parallelized_host_filtering());
 
     delete @Global::host{@$down_hosts_ref};
-    @$down_hosts_ref and ::warning("Removed @$down_hosts_ref\n");
+    @$down_hosts_ref and ::warning("Removed @$down_hosts_ref.");
 
     $Global::minimal_command_line_length = 8_000_000;
     while (my ($sshlogin, $obj) = each %Global::host) {
@@ -2734,7 +2735,7 @@ sub parse_host_filtering {
 		# signal == 127: parallel not installed remote
 		# Set ncpus and ncores = 1
 		::warning("Could not figure out ",
-			  "number of cpus on $host. Using 1.\n");
+			  "number of cpus on $host. Using 1.");
 		$ncores{$host} = 1;
 		$ncpus{$host} = 1;
 		$maxlen{$host} = Limits::Command::max_length();
@@ -3282,16 +3283,14 @@ sub status {
 
 sub warning {
     my @w = @_;
-    my $fh = $Global::status_fd || *STDERR;
     my $prog = $Global::progname || "parallel";
-    print $fh $prog, ": Warning: ", @w;
+    status(map { ($prog, ": Warning: ", $_, "\n"); } @w);
 }
 
 sub error {
     my @w = @_;
-    my $fh = $Global::status_fd || *STDERR;
     my $prog = $Global::progname || "parallel";
-    print $fh $prog, ": Error: ", @w;
+    status(map { ($prog, ": Error: ", $_, "\n"); } @w);
 }
 
 sub die_bug {
@@ -3401,6 +3400,14 @@ sub tmpname {
     # if you ssh to localhost (or a shared file system) under a different name
     my $name = shift;
     my($tmpname);
+    if(not -w $ENV{'TMPDIR'}) {
+	if(not -e $ENV{'TMPDIR'}) {
+	    ::error("Tmpdir '$ENV{'TMPDIR'}' does not exist.","Try 'mkdir $ENV{'TMPDIR'}'");
+	} else {
+	    ::error("Tmpdir '$ENV{'TMPDIR'}' is not writable.","Try 'chmod +w $ENV{'TMPDIR'}'");
+	}
+	::wait_and_exit(255);
+    }
     do {
 	$tmpname = $ENV{'TMPDIR'}."/".$name.
 	    join"", map { (0..9,"a".."z","A".."Z")[rand(62)] } (1..5);
@@ -3555,8 +3562,8 @@ sub multiply_binary_prefix {
 	   tell $disk_full_fh != 8193) {
 	    # On raspbian the disk can be full except for 10 chars.
 	    if(not $error_printed) {
-		::error("Output is incomplete. Cannot append to buffer file in $ENV{'TMPDIR'}. Is the disk full?\n");
-		::error("Change \$TMPDIR with --tmpdir or use --compress.\n");
+		::error("Output is incomplete. Cannot append to buffer file in $ENV{'TMPDIR'}. Is the disk full?",
+			"Change \$TMPDIR with --tmpdir or use --compress.");
 		$error_printed = 1;
 	    }
 	    ::wait_and_exit(255);
@@ -4524,11 +4531,11 @@ sub compute_max_loadavg {
                 close $in_fh;
                 $load = $self->compute_max_loadavg($opt_load_file);
             } else {
-                ::error("Cannot open $loadspec.\n");
+                ::error("Cannot open $loadspec.");
                 ::wait_and_exit(255);
             }
         } else {
-            ::error("Parsing of --load failed.\n");
+            ::error("Parsing of --load failed.");
             ::die_usage();
         }
         if($load < 0.01) {
@@ -4730,8 +4737,8 @@ sub compute_number_of_processes {
 		# It took more than 0.01 second to fork a processes on avg.
 		# Give the user a warning. He can press Ctrl-C if this
 		# sucks.
-		::warning("Starting $system_limit processes took > $forktime sec.\n",
-			  "Consider adjusting -j. Press CTRL-C to stop.\n");
+		::warning("Starting $system_limit processes took > $forktime sec.",
+			  "Consider adjusting -j. Press CTRL-C to stop.");
 		$slow_spawining_warning_printed = 1;
 	    }
 	}
@@ -4740,19 +4747,20 @@ sub compute_number_of_processes {
 	if($system_limit < $wanted_processes) {
 	    # The system_limit is less than the wanted_processes
 	    if($system_limit < 1 and not $Global::JobQueue->empty()) {
-		::warning("Cannot spawn any jobs. Raising ulimit -u or /etc/security/limits.conf\n",
-			  "or /proc/sys/kernel/pid_max may help.\n");
+		::warning("Cannot spawn any jobs. Raising ulimit -u or /etc/security/limits.conf",
+			  "or /proc/sys/kernel/pid_max may help.");
 		::wait_and_exit(255);
 	    }
 	    if(not $more_filehandles) {
-		::warning("Only enough file handles to run ", $system_limit, " jobs in parallel.\n",
-			  "Running 'parallel -j0 -N", $system_limit, " --pipe parallel -j0' or ",
-			  "raising ulimit -n or /etc/security/limits.conf may help.\n");
+		::warning("Only enough file handles to run ", $system_limit, " jobs in parallel.",
+			  "Running 'parallel -j0 -N $system_limit --pipe parallel -j0' or ".
+			  "raising ulimit -n or /etc/security/limits.conf may help.");
 	    }
 	    if($max_system_proc_reached) {
-		::warning("Only enough available processes to run ", $system_limit,
-			  " jobs in parallel. Raising ulimit -u or /etc/security/limits.conf\n",
-			  "or /proc/sys/kernel/pid_max may help.\n");
+		::warning("Only enough available processes to run ". $system_limit.
+			  " jobs in parallel.",
+			  "Raising ulimit -u or /etc/security/limits.conf ",
+			  "or /proc/sys/kernel/pid_max may help.");
 	    }
 	}
 	if($] == 5.008008 and $system_limit > 1000) {
@@ -4788,12 +4796,12 @@ sub simultaneous_sshlogin_limit {
 	      $self->simultaneous_sshlogin($wanted_processes));
     if($ssh_limit < $wanted_processes) {
         my $serverlogin = $self->serverlogin();
-        ::warning("ssh to $serverlogin only allows ",
-		  "for $ssh_limit simultaneous logins.\n",
-		  "You may raise this by changing ",
-		  "/etc/ssh/sshd_config:MaxStartups and MaxSessions on $serverlogin.\n",
-		  "Using only ",$ssh_limit-1," connections ",
-		  "to avoid race conditions.\n");
+        ::warning("ssh to $serverlogin only allows ".
+		  "for $ssh_limit simultaneous logins.",
+		  "You may raise this by changing ".
+		  "/etc/ssh/sshd_config:MaxStartups and MaxSessions on $serverlogin.",
+		  "Using only ".($ssh_limit-1)." connections ".
+		  "to avoid race conditions.");
     }
     # Race condition can cause problem if using all sshs.
     if($ssh_limit > 1) { $ssh_limit -= 1; }
@@ -4863,11 +4871,11 @@ sub user_requested_processes {
                 close $in_fh;
                 $processes = $self->user_requested_processes($opt_P_file);
             } else {
-                ::error("Cannot open $opt_P.\n");
+                ::error("Cannot open $opt_P.");
                 ::wait_and_exit(255);
             }
         } else {
-            ::error("Parsing of --jobs/-j/--max-procs/-P failed.\n");
+            ::error("Parsing of --jobs/-j/--max-procs/-P failed.");
             ::die_usage();
         }
 	$processes = ::ceil($processes);
@@ -4899,8 +4907,8 @@ sub ncpus {
             if($ncpu =~ /^\s*[0-9]+\s*$/s) {
                 $self->{'ncpus'} = $ncpu;
             } else {
-                ::warning("Could not figure out ",
-			  "number of cpus on $serverlogin ($ncpu). Using 1.\n");
+                ::warning("Could not figure out ".
+			  "number of cpus on $serverlogin ($ncpu). Using 1.");
                 $self->{'ncpus'} = 1;
             }
         }
@@ -4961,7 +4969,7 @@ sub no_of_cpus {
 	chomp $no_of_cpus;
         return $no_of_cpus;
     } else {
-        ::warning("Cannot figure out number of cpus. Using 1.\n");
+        ::warning("Cannot figure out number of cpus. Using 1.");
         return 1;
     }
 }
@@ -5018,7 +5026,7 @@ sub no_of_cores {
 	chomp $no_of_cores;
         return $no_of_cores;
     } else {
-        ::warning("Cannot figure out number of CPU cores. Using 1.\n");
+        ::warning("Cannot figure out number of CPU cores. Using 1.");
         return 1;
     }
 }
@@ -5460,7 +5468,7 @@ sub rsync_transfer_cmd {
     my $file = shift;
     my $workdir = shift;
     if(not -r $file) {
-	::warning($file, " is not readable and will not be transferred.\n");
+	::warning($file. " is not readable and will not be transferred.");
 	return "true";
     }
     my $rsync_destdir;
@@ -5599,10 +5607,10 @@ sub total_jobs {
 	my $start = time;
         while($job = $self->get()) {
 	    if(time - $start > 10) {
-		::warning("Reading ".scalar(@queue)." arguments took longer than 10 seconds.\n");
-		$opt::eta && ::warning("Consider removing --eta.\n");
-		$opt::bar && ::warning("Consider removing --bar.\n");
-		$opt::shuf && ::warning("Consider removing --shuf.\n");
+		::warning("Reading ".scalar(@queue)." arguments took longer than 10 seconds.");
+		$opt::eta && ::warning("Consider removing --eta.");
+		$opt::bar && ::warning("Consider removing --bar.");
+		$opt::shuf && ::warning("Consider removing --shuf.");
 		last;
 	    }
             push @queue, $job;
@@ -5792,13 +5800,13 @@ sub openoutputfiles {
 	# prefix/name1/val1/name2/val2/stdout
 	$outname = "$dir/stdout";
 	if(not open($outfhw, "+>", $outname)) {
-	    ::error("Cannot write to `$outname'.\n");
+	    ::error("Cannot write to `$outname'.");
 	    ::wait_and_exit(255);
 	}
 	# prefix/name1/val1/name2/val2/stderr
 	$errname = "$dir/stderr";
 	if(not open($errfhw, "+>", $errname)) {
-	    ::error("Cannot write to `$errname'.\n");
+	    ::error("Cannot write to `$errname'.");
 	    ::wait_and_exit(255);
 	}
 	$self->set_fh(1,"unlink","");
@@ -6517,7 +6525,7 @@ sub sshlogin_wrap {
 		push @vars, grep { not defined $ignore{$_} } keys %ENV;
 		@vars = grep { not /^_$/ } @vars;
 	    } else {
-		::error("Run '$Global::progname --record-env' in a clean environment first.\n");
+		::error("Run '$Global::progname --record-env' in a clean environment first.");
 		::wait_and_exit(255);
 	    }
 	}
@@ -6548,7 +6556,7 @@ sub sshlogin_wrap {
 	if(@bashfunc) {
 	    # Functions are not supported for all shells
 	    if($Global::shell !~ m:/(bash|rbash|zsh|rzsh|dash|ksh):) {
-		::warning("Shell functions may not be supported in $Global::shell\n");
+		::warning("Shell functions may not be supported in $Global::shell.");
 	    }
 	    $bashfuncset =
 		'@bash_functions=qw('."@bash_functions".");".
@@ -7006,13 +7014,13 @@ sub print_dryrun_and_verbose {
 #	# FIFO for communicating exit val
 #	my $tmpfifo = ::tmpfifo();
 	if(length($tmpfifo) >=100) {
-	    ::error("tmux does not support sockets with path > 100\n");
+	    ::error("tmux does not support sockets with path > 100.");
 	    ::wait_and_exit(255);
 	}
 	my $visual_command = $self->replaced();
 	my $title = $visual_command;
 	if($visual_command =~ /\0/) {
-	    ::error("Command line contains NUL. tmux is confused by NUL.\n");
+	    ::error("Command line contains NUL. tmux is confused by NUL.");
 	    ::wait_and_exit(255);
 	}
 	# ; causes problems
@@ -7229,7 +7237,7 @@ sub files_print {
     # If the job is dead: close printing fh. Needed for --compress
     close $self->fh($fdno,"w");
     if($? and $opt::compress) {
-	::error($opt::compress_program." failed.\n");
+	::error($opt::compress_program." failed.");
 	$self->set_exitstatus(255);
     }
     if($opt::compress) {
@@ -7260,7 +7268,7 @@ sub linebuffer_print {
 	# If the job is dead: close printing fh. Needed for --compress
 	close $self->fh($fdno,"w");
 	if($? and $opt::compress) {
-	    ::error($opt::compress_program." failed.\n");
+	    ::error($opt::compress_program." failed.");
 	    $self->set_exitstatus(255);
 	}
 	if($opt::compress) {
@@ -7330,7 +7338,7 @@ sub linebuffer_print {
 	    # decompress done: close fh
 	    close $in_fh;
 	    if($? and $opt::compress) {
-		::error($opt::decompress_program." failed.\n");
+		::error($opt::decompress_program." failed.");
 		$self->set_exitstatus(255);
 	    }
 	}
@@ -7343,7 +7351,7 @@ sub tag_print {
     my $buf;
     close $self->fh($fdno,"w");
     if($? and $opt::compress) {
-	::error($opt::compress_program." failed.\n");
+	::error($opt::compress_program." failed.");
 	$self->set_exitstatus(255);
     }
     seek $in_fh, 0, 0;
@@ -7373,7 +7381,7 @@ sub tag_print {
     }
     close $in_fh;
     if($? and $opt::compress) {
-	::error($opt::decompress_program." failed.\n");
+	::error($opt::decompress_program." failed.");
 	$self->set_exitstatus(255);
     }
 }
@@ -7384,7 +7392,7 @@ sub normal_print {
     my $buf;
     close $self->fh($fdno,"w");
     if($? and $opt::compress) {
-	::error($opt::compress_program." failed.\n");
+	::error($opt::compress_program." failed.");
 	$self->set_exitstatus(255);
     }
     seek $in_fh, 0, 0;
@@ -7407,7 +7415,7 @@ sub normal_print {
     }
     close $in_fh;
     if($? and $opt::compress) {
-	::error($opt::decompress_program." failed.\n");
+	::error($opt::decompress_program." failed.");
 	$self->set_exitstatus(255);
     }
 }
@@ -7672,15 +7680,15 @@ sub populate {
 		last;
 	    } else {
 		my $args = join(" ", map { $_->orig() } @$next_arg);
-		::error("Command line too long (",
-			$self->len(), " >= ",
-			$max_len,
-			") at input ",
-			$self->{'arg_queue'}->arg_number(),
+		::error("Command line too long (".
+			$self->len(). " >= ".
+			$max_len.
+			") at input ".
+			$self->{'arg_queue'}->arg_number().
 			": ".
 			((length $args > 50) ?
-			 (substr($args,0,50))."...\n" :
-			 $args."\n"));
+			 (substr($args,0,50))."..." :
+			 $args));
 		$self->{'arg_queue'}->unget($self->pop());
 		::wait_and_exit(255);
 	    }
@@ -8126,7 +8134,7 @@ sub new {
 	# Is this really a command in $PATH starting with '-'?
 	my $cmd = $1;
 	if(not ::which($cmd)) {
-	    ::error("Command ($cmd) starts with '-'. Is this a wrong option?\n");
+	    ::error("Command ($cmd) starts with '-'. Is this a wrong option?");
 	    ::wait_and_exit(255);
 	}
     }
@@ -8140,7 +8148,7 @@ sub new {
 	(defined $opt::tagstring ? $opt::tagstring : $dummy)) {
 	# Disallow \257 to avoid nested {= {= =} =}
 	if(/\257/) {
-	    ::error("Command cannot contain the character \257. Use a function for that.\n");
+	    ::error("Command cannot contain the character \257. Use a function for that.");
 	    ::wait_and_exit(255);
 	}
 	# Needs to match rightmost left parens (Perl defaults to leftmost)
@@ -8337,7 +8345,7 @@ sub get {
 	if($opt::pipe or $opt::pipepart) {
 	    if($cmd_line->replaced() eq "") {
 		# Empty command - pipe requires a command
-		::error("--pipe must have a command to pipe into (e.g. 'cat').\n");
+		::error("--pipe must have a command to pipe into (e.g. 'cat').");
 		::wait_and_exit(255);
 	    }
 	} else {
@@ -8412,8 +8420,7 @@ sub max_length {
 	    if($opt::max_chars <= $cached_limit) {
 		$Limits::Command::line_max_len = $opt::max_chars;
 	    } else {
-		::warning("Value for -s option ",
-			  "should be < $cached_limit.\n");
+		::warning("Value for -s option should be < $cached_limit.");
 	    }
 	}
     }
@@ -8477,7 +8484,7 @@ sub tmux_length {
     if($opt::tmux) {
 	$ENV{'TMUX'} ||= "tmux";
 	if(not ::which($ENV{'TMUX'})) {
-	    ::error($ENV{'TMUX'}." not found in \$PATH.\n");
+	    ::error($ENV{'TMUX'}." not found in \$PATH.");
 	    ::wait_and_exit(255);
 	}
 	my @out;
@@ -8630,9 +8637,9 @@ sub new {
     my $fhs = shift;
     for my $fh (@$fhs) {
 	if(-t $fh) {
-	    ::warning("Input is read from the terminal.\n");
-	    ::warning("Only experts do this on purpose. ".
-		      "Press CTRL-D to exit.\n");
+	    ::warning("Input is read from the terminal.",
+		      "Only experts do this on purpose. ".
+		      "Press CTRL-D to exit.");
 	}
     }
     return bless {
@@ -8865,7 +8872,7 @@ sub new {
 	    # We found hostgroups on the arg
 	    @hostgroups = split(/\+/, $1);
 	    if(not grep { defined $Global::hostgroups{$_} } @hostgroups) {
-		::warning("No such hostgroup (@hostgroups)\n");
+		::warning("No such hostgroup (@hostgroups).");
 		@hostgroups = (keys %Global::hostgroups);
 	    }
         } else {
@@ -8908,7 +8915,7 @@ sub new {
 		# All is good
 	    } else {
 		# The eval failed. Maybe $perlexpr is invalid perl?
-		::error("Cannot use $perlexpr: $@\n");
+		::error("Cannot use $perlexpr: $@");
 		::wait_and_exit(255);
 	    }
 	}
@@ -8942,7 +8949,7 @@ sub trim_of {
     } elsif($Global::trim eq "rl" or $Global::trim eq "lr") {
 	for my $arg (@strings) { $arg =~ s/^\s+//; $arg =~ s/\s+$//; }
     } else {
-	::error("--trim must be one of: r l rl lr.\n");
+	::error("--trim must be one of: r l rl lr.");
 	::wait_and_exit(255);
     }
     return wantarray ? @strings : "@strings";
@@ -9114,7 +9121,7 @@ sub acquire {
 	       and
 	       time - $start_time > $opt::semaphoretimeout) {
 		# Timeout: Take the semaphore anyway
-		::warning("Semaphore timed out. Stealing the semaphore.\n");
+		::warning("Semaphore timed out. Stealing the semaphore.");
 		if(not -e $self->{'idfile'}) {
 		    open (my $fh, ">", $self->{'idfile'}) or
 			::die_bug("timeout_write_idfile: $self->{'idfile'}");
@@ -9127,7 +9134,7 @@ sub acquire {
 	       and
 	       time - $start_time > -$opt::semaphoretimeout) {
 		# Timeout: Exit
-		::warning("Semaphore timed out. Exiting.\n");
+		::warning("Semaphore timed out. Exiting.");
 		exit(1);
 		last;
 	    }
@@ -9216,8 +9223,8 @@ sub lock {
 		last;
 	    } else {
 		if ($! =~ m/Function not implemented/) {
-		    ::warning("flock: $!");
-		    ::warning("Will wait for a random while\n");
+		    ::warning("flock: $!",
+			      "Will wait for a random while.");
 		    ::usleep(rand(5000));
 		    # File cannot be locked: No need to retry
 		    $locked = 2;
@@ -9276,7 +9283,7 @@ sub mkdir_or_die {
 	mkdir $ddir;
     }
     if(not -w $dir) {
-	::error("Cannot write to $dir: $!\n");
+	::error("Cannot write to $dir: $!");
 	::wait_and_exit(255);
     }
 }

From f290cf9b053c782efac9358d2b458382fed4988d Mon Sep 17 00:00:00 2001
From: Ole Tange 
Date: Fri, 15 May 2015 01:59:57 +0200
Subject: [PATCH 09/14] testsuite: Additional --halt testing.

---
 doc/release_new_version                       | 6 ++++++
 testsuite/tests-to-run/parallel-local-0.3s.sh | 4 ++++
 testsuite/tests-to-run/parallel-local22.sh    | 2 +-
 testsuite/wanted-results/parallel-local-0.3s  | 6 ++++++
 testsuite/wanted-results/parallel-local22     | 3 +--
 5 files changed, 18 insertions(+), 3 deletions(-)

diff --git a/doc/release_new_version b/doc/release_new_version
index a7f48876..d55b0869 100644
--- a/doc/release_new_version
+++ b/doc/release_new_version
@@ -222,6 +222,10 @@ Haiku of the month:
 
 New in this release:
 
+* Security: The security issue for --sshlogin + --fifo/--cat has been fixed. Thereby all issues with http://lists.gnu.org/archive/html/parallel/2015-04/msg00045.html have been fixed.
+
+* Security: After further security analysis the issue fixed in 20150422 also fixed the problem for --tmux.
+
 * <> GNU Parallel was used (unfortunately without citation) in: MUGBAS: a species free gene-based programme suite for post-GWAS analysis http://www.ncbi.nlm.nih.gov/pubmed/25765345
 
 taxator-tk http://algbio.cs.uni-duesseldorf.de/webapps/wa-download/ (check it)
@@ -257,6 +261,8 @@ taxator-tk http://algbio.cs.uni-duesseldorf.de/webapps/wa-download/ (check it)
 
 * Run multiple ssh commands in parallel with GNU Parallel http://www.ameir.net/blog/archives/380-run-multiple-ssh-commands-in-parallel-with-gnu-parallel.html
 
+* Parallel? Gnu parallel! https://debian.pro/1834
+
 * Bug fixes and man page updates.
 
 GNU Parallel - For people who live life in the parallel lane.
diff --git a/testsuite/tests-to-run/parallel-local-0.3s.sh b/testsuite/tests-to-run/parallel-local-0.3s.sh
index 427979e3..b6887359 100644
--- a/testsuite/tests-to-run/parallel-local-0.3s.sh
+++ b/testsuite/tests-to-run/parallel-local-0.3s.sh
@@ -148,6 +148,10 @@ echo '**'
 parallel --halt 2 ::: 'sleep 1' burnP6 false; killall burnP6 && echo ERROR: burnP6 should be killed
 parallel --halt -2 ::: 'sleep 1' burnP5 true; killall burnP5 && echo ERROR: burnP5 should be killed
 
+parallel --halt error ::: 1
+parallel --halt soon ::: 1
+parallel --halt now ::: 1
+
 echo '**'
 
 echo '### bug #44995: parallel echo {#} ::: 1 2 ::: 1 2'
diff --git a/testsuite/tests-to-run/parallel-local22.sh b/testsuite/tests-to-run/parallel-local22.sh
index 5dc67ddd..fc2aaac2 100755
--- a/testsuite/tests-to-run/parallel-local22.sh
+++ b/testsuite/tests-to-run/parallel-local22.sh
@@ -72,7 +72,7 @@ echo '### bug #42892: parallel -a nonexiting --pipepart'
 echo '### bug #42913: Dont use $SHELL but the shell currently running'
   echo '## Unknown shell => $SHELL (bash)'
   parallel -j1 "cp \`which {}\` /tmp/SHELL; /tmp/SHELL -c 'parallel -Dinit echo ::: 1' | grep which;" 
-  ::: ash bash csh dash fdsh fish fizsh ksh ksh93 mksh pdksh posh rbash rush rzsh sash sh static-sh tcsh yash zsh; 
+  ::: ash bash csh dash fish fizsh ksh ksh93 mksh pdksh posh rbash rush rzsh sash sh static-sh tcsh yash zsh; 
   rm -f /tmp/SHELL /tmp/par*.par
 
   echo '## Known shells -c'
diff --git a/testsuite/wanted-results/parallel-local-0.3s b/testsuite/wanted-results/parallel-local-0.3s
index e84a1237..a06402ac 100644
--- a/testsuite/wanted-results/parallel-local-0.3s
+++ b/testsuite/wanted-results/parallel-local-0.3s
@@ -317,6 +317,12 @@ parallel --halt -2 ::: 'sleep 1' burnP5 true; killall burnP5 && echo ERROR: burn
 parallel: This job succeeded:
 true
 burnP5: no process found
+parallel --halt error ::: 1
+parallel: Error: --halt must have 'never', 'soon', or 'now'
+parallel --halt soon ::: 1
+parallel: Error: --halt soon must be followed by ,success or ,fail
+parallel --halt now ::: 1
+parallel: Error: --halt now must be followed by ,success or ,fail
 echo '**'
 **
 echo '### bug #44995: parallel echo {#} ::: 1 2 ::: 1 2'
diff --git a/testsuite/wanted-results/parallel-local22 b/testsuite/wanted-results/parallel-local22
index d6a5d841..0a219e6b 100644
--- a/testsuite/wanted-results/parallel-local22
+++ b/testsuite/wanted-results/parallel-local22
@@ -95,7 +95,7 @@ echo '### bug #42913: Dont use $SHELL but the shell currently running'
 ### bug #42913: Dont use $SHELL but the shell currently running
   echo '## Unknown shell => $SHELL (bash)'
 ## Unknown shell => $SHELL (bash)
-  parallel -j1 "cp \`which {}\` /tmp/SHELL; /tmp/SHELL -c 'parallel -Dinit echo ::: 1' | grep which;"   ::: ash bash csh dash fdsh fish fizsh ksh ksh93 mksh pdksh posh rbash rush rzsh sash sh static-sh tcsh yash zsh;   rm -f /tmp/SHELL /tmp/par*.par
+  parallel -j1 "cp \`which {}\` /tmp/SHELL; /tmp/SHELL -c 'parallel -Dinit echo ::: 1' | grep which;"   ::: ash bash csh dash fish fizsh ksh ksh93 mksh pdksh posh rbash rush rzsh sash sh static-sh tcsh yash zsh;   rm -f /tmp/SHELL /tmp/par*.par
 shell? /bin/bash -c cp `which ash` /tmp/SHELL; /tmp/SHELL -c 'parallel -Dinit echo ::: 1' | grep which;
 which bash => shell path /bin/bash
 shell? /bin/bash -c cp `which bash` /tmp/SHELL; /tmp/SHELL -c 'parallel -Dinit echo ::: 1' | grep which;
@@ -131,7 +131,6 @@ shell? /bin/bash -c cp `which yash` /tmp/SHELL; /tmp/SHELL -c 'parallel -Dinit e
 which bash => shell path /bin/bash
 shell? /bin/bash -c cp `which zsh` /tmp/SHELL; /tmp/SHELL -c 'parallel -Dinit echo ::: 1' | grep which;
 which bash => shell path /bin/bash
-/tmp/SHELL: -c: bad option(s)
 Local configuration error occurred.
 Contact the systems administrator for further assistance.
 SHELL: applet not found

From 847fbe125a6ad9d791330527ad2233daa9797bc2 Mon Sep 17 00:00:00 2001
From: Ole Tange 
Date: Fri, 15 May 2015 08:47:44 +0200
Subject: [PATCH 10/14] parallel: Formatting of error messages. Passes
 testsuite.

---
 CREDITS                                       |  2 ++
 src/parallel                                  | 15 +++++++++------
 testsuite/tests-to-run/parallel-local-0.3s.sh |  6 +++---
 testsuite/wanted-results/parallel-freebsd     |  2 +-
 testsuite/wanted-results/parallel-local-0.3s  | 12 ++++++------
 testsuite/wanted-results/parallel-local-100s  |  3 ++-
 testsuite/wanted-results/parallel-local-ssh1  |  2 +-
 testsuite/wanted-results/parallel-local164    |  3 ++-
 testsuite/wanted-results/parallel-local4      |  2 +-
 testsuite/wanted-results/parallel-local5      |  8 ++++----
 testsuite/wanted-results/parallel-local9      |  2 +-
 testsuite/wanted-results/parallel-tutorial    |  2 +-
 12 files changed, 33 insertions(+), 26 deletions(-)

diff --git a/CREDITS b/CREDITS
index 3bfd36d9..da9295e5 100644
--- a/CREDITS
+++ b/CREDITS
@@ -1,3 +1,5 @@
+Rasmus Villemoes: Code snips for signal processing.
+Martin d'Anjou: Code snips for signal processing.
 rici@stackoverflow.com: Documentation on exporting arrays using --env.
 Malcolm Cook: The idea to use a general perl expression as replacement strings.
 Ævar Arnfjörð Bjarmason: Reading the whole code.
diff --git a/src/parallel b/src/parallel
index 22620e60..c39285f2 100755
--- a/src/parallel
+++ b/src/parallel
@@ -1139,7 +1139,8 @@ sub parse_halt {
 	# Split: soon,fail=5%
 	my ($when,$fail_success,$pct_count) = split /[,=]/, $opt::halt;
 	if(not grep { $when eq $_ } qw(never soon now)) {
-	    ::error("--halt must have 'never', 'soon', or 'now'\n");
+	    ::error("--halt must have 'never', 'soon', or 'now'.");
+	    ::wait_and_exit(255);
 	}
 	$Global::halt_when = $when;
 	if($when ne "never") {
@@ -1148,15 +1149,17 @@ sub parse_halt {
 	    } elsif($fail_success eq "success") {
 		$Global::halt_success = 1;
 	    } else {
-		::error("--halt $when must be followed by ,success or ,fail\n");
+		::error("--halt $when must be followed by ,success or ,fail.");
+		::wait_and_exit(255);
 	    }
 	    if($pct_count =~ /^(\d+)%$/) {
 		$Global::halt_pct = $1/100;
 	    } elsif($pct_count =~ /^(\d+)$/) {
 		$Global::halt_count = $1;
 	    } else {
-		::error("--halt $when,$fail_success ",
-			"must be followed by ,number or ,percent%\n");
+		::error("--halt $when,$fail_success ".
+			"must be followed by ,number or ,percent%.");
+		::wait_and_exit(255);
 	    }
 	}
     }
@@ -4752,8 +4755,8 @@ sub compute_number_of_processes {
 		::wait_and_exit(255);
 	    }
 	    if(not $more_filehandles) {
-		::warning("Only enough file handles to run ", $system_limit, " jobs in parallel.",
-			  "Running 'parallel -j0 -N $system_limit --pipe parallel -j0' or ".
+		::warning("Only enough file handles to run ". $system_limit. " jobs in parallel.",
+			  "Running 'parallel -j0 -N $system_limit --pipe parallel -j0' or ",
 			  "raising ulimit -n or /etc/security/limits.conf may help.");
 	    }
 	    if($max_system_proc_reached) {
diff --git a/testsuite/tests-to-run/parallel-local-0.3s.sh b/testsuite/tests-to-run/parallel-local-0.3s.sh
index b6887359..d9944ca0 100644
--- a/testsuite/tests-to-run/parallel-local-0.3s.sh
+++ b/testsuite/tests-to-run/parallel-local-0.3s.sh
@@ -148,9 +148,9 @@ echo '**'
 parallel --halt 2 ::: 'sleep 1' burnP6 false; killall burnP6 && echo ERROR: burnP6 should be killed
 parallel --halt -2 ::: 'sleep 1' burnP5 true; killall burnP5 && echo ERROR: burnP5 should be killed
 
-parallel --halt error ::: 1
-parallel --halt soon ::: 1
-parallel --halt now ::: 1
+parallel --halt error echo ::: should not print
+parallel --halt soon echo ::: should not print
+parallel --halt now echo ::: should not print
 
 echo '**'
 
diff --git a/testsuite/wanted-results/parallel-freebsd b/testsuite/wanted-results/parallel-freebsd
index 6e8a5463..4f8b38fd 100644
--- a/testsuite/wanted-results/parallel-freebsd
+++ b/testsuite/wanted-results/parallel-freebsd
@@ -58,7 +58,7 @@ bash -c 'echo bug \#43358: shellshock breaks exporting functions using --env _;
 bug #43358: shellshock breaks exporting functions using --env _
 Non-shellshock-hardened to non-shellshock-hardened
 Function non-shellshock-hardened
-parallel: Warning: Shell functions may not be supported in bash
+parallel: Warning: Shell functions may not be supported in bash.
 bash -c 'echo bug \#43358: shellshock breaks exporting functions using --env _;   echo Non-shellshock-hardened to shellshock-hardened;   funky() { echo Function $1; };   export -f funky;   parallel --env funky -S parallel@192.168.1.72 funky ::: shellshock-hardened'
 bug #43358: shellshock breaks exporting functions using --env _
 Non-shellshock-hardened to shellshock-hardened
diff --git a/testsuite/wanted-results/parallel-local-0.3s b/testsuite/wanted-results/parallel-local-0.3s
index a06402ac..44f0d0a2 100644
--- a/testsuite/wanted-results/parallel-local-0.3s
+++ b/testsuite/wanted-results/parallel-local-0.3s
@@ -317,12 +317,12 @@ parallel --halt -2 ::: 'sleep 1' burnP5 true; killall burnP5 && echo ERROR: burn
 parallel: This job succeeded:
 true
 burnP5: no process found
-parallel --halt error ::: 1
-parallel: Error: --halt must have 'never', 'soon', or 'now'
-parallel --halt soon ::: 1
-parallel: Error: --halt soon must be followed by ,success or ,fail
-parallel --halt now ::: 1
-parallel: Error: --halt now must be followed by ,success or ,fail
+parallel --halt error echo ::: should not print
+parallel: Error: --halt must have 'never', 'soon', or 'now'.
+parallel --halt soon echo ::: should not print
+parallel: Error: --halt soon must be followed by ,success or ,fail.
+parallel --halt now echo ::: should not print
+parallel: Error: --halt now must be followed by ,success or ,fail.
 echo '**'
 **
 echo '### bug #44995: parallel echo {#} ::: 1 2 ::: 1 2'
diff --git a/testsuite/wanted-results/parallel-local-100s b/testsuite/wanted-results/parallel-local-100s
index 7fc32c33..25b5a8e6 100644
--- a/testsuite/wanted-results/parallel-local-100s
+++ b/testsuite/wanted-results/parallel-local-100s
@@ -9,7 +9,8 @@ echo "### Test Force outside the file handle limit, 2009-02-17 Gave fork error"
 ### Test Force outside the file handle limit, 2009-02-17 Gave fork error
   (echo echo Start; seq 1 20000 | perl -pe 's/^/true /'; echo echo end) | stdout parallel -uj 0 | egrep -v 'processes took|adjusting'
 parallel: Warning: Only enough file handles to run 252 jobs in parallel.
-Running 'parallel -j0 -N252 --pipe parallel -j0' or raising ulimit -n or /etc/security/limits.conf may help.
+parallel: Warning: Running 'parallel -j0 -N 252 --pipe parallel -j0' or 
+parallel: Warning: raising ulimit -n or /etc/security/limits.conf may help.
 Start
 end
 echo '**'
diff --git a/testsuite/wanted-results/parallel-local-ssh1 b/testsuite/wanted-results/parallel-local-ssh1
index 5f90675b..f95a0403 100644
--- a/testsuite/wanted-results/parallel-local-ssh1
+++ b/testsuite/wanted-results/parallel-local-ssh1
@@ -1,7 +1,7 @@
 echo '### Stop if all hosts are filtered and there are no hosts left to run on'
 ### Stop if all hosts are filtered and there are no hosts left to run on
   stdout parallel --filter-hosts -S no-such.host echo ::: 1
-parallel: Warning: Removed no-such.host
+parallel: Warning: Removed no-such.host.
 parallel: Error: There are no hosts left to run on.
 echo '### Can csh propagate a variable containing \n';   export A=$(seq 3); parallel -S csh@localhost --env A bash -c \''echo "$A"'\' ::: dummy
 ### Can csh propagate a variable containing \n
diff --git a/testsuite/wanted-results/parallel-local164 b/testsuite/wanted-results/parallel-local164
index dd7e8787..5bdaaa52 100644
--- a/testsuite/wanted-results/parallel-local164
+++ b/testsuite/wanted-results/parallel-local164
@@ -344,7 +344,8 @@ bug #38439: "open files" with --files --pipe blocks after a while
 19 of 20
 20 of 20
 parallel: Warning: Only enough file handles to run 1 jobs in parallel.
-Running 'parallel -j0 -N1 --pipe parallel -j0' or raising ulimit -n or /etc/security/limits.conf may help.
+parallel: Warning: Running 'parallel -j0 -N 1 --pipe parallel -j0' or 
+parallel: Warning: raising ulimit -n or /etc/security/limits.conf may help.
 echo 'bug #34241: --pipe should not spawn unneeded processes - part 2'
 bug #34241: --pipe should not spawn unneeded processes - part 2
   seq 500 | parallel --tmpdir . -j10 --pipe --block 1k --files wc >/dev/null;   ls *.par | wc -l; rm *.par;   seq 500 | parallel --tmpdir . -j10 --pipe --block 1k --files --dry-run wc >/dev/null;   echo No .par should exist;   stdout ls *.par
diff --git a/testsuite/wanted-results/parallel-local4 b/testsuite/wanted-results/parallel-local4
index 94dafa04..05280180 100644
--- a/testsuite/wanted-results/parallel-local4
+++ b/testsuite/wanted-results/parallel-local4
@@ -101,7 +101,7 @@ X	:	XXXXXXXXXX.XXX	     X.XXX	X	X	X	X	true X
 echo '### How do we deal with missing $HOME'
 ### How do we deal with missing $HOME
    unset HOME; stdout perl -w $(which parallel) -k echo ::: 1 2 3
-parallel: Warning: $HOME not set. Using /tmp
+parallel: Warning: $HOME not set. Using /tmp.
 1
 2
 3
diff --git a/testsuite/wanted-results/parallel-local5 b/testsuite/wanted-results/parallel-local5
index 49d82238..c46ed9a1 100644
--- a/testsuite/wanted-results/parallel-local5
+++ b/testsuite/wanted-results/parallel-local5
@@ -1,7 +1,7 @@
 ### Test --pipe
 echo '### Test 200M records with too small block';   (    echo start;    seq 1 44 | parallel -uj1 cat /tmp/blocktest\;true;    echo end;    echo start;    seq 1 44 | parallel -uj1 cat /tmp/blocktest\;true;    echo end;    echo start;    seq 1 44 | parallel -uj1 cat /tmp/blocktest\;true;    echo end;   ) | stdout parallel -k --block 200m -j2 --pipe --recend 'end\n' wc -c |   egrep -v '^0$'
 ### Test 200M records with too small block
-parallel: Warning: A record was longer than 200000000. Increasing to --blocksize 260000001
+parallel: Warning: A record was longer than 200000000. Increasing to --blocksize 260000001.
 303111434
 303111434
 303111434
@@ -148,10 +148,10 @@ echo -n 01a02a0a0a12a34a45a6a |   parallel -k -j1 --blocksize 100 --pipe --recen
 2>0a12a34a
 3>45a6a
 echo -n 01a02a0a0a12a34a45a6a |   stdout parallel -k -j1 --blocksize 1 --pipe --recend a  -N 3  'echo -n "$PARALLEL_SEQ>"; cat; echo; sleep 0.1'
-parallel: Warning: A record was longer than 1. Increasing to --blocksize 3
-parallel: Warning: A record was longer than 3. Increasing to --blocksize 5
+parallel: Warning: A record was longer than 1. Increasing to --blocksize 3.
+parallel: Warning: A record was longer than 3. Increasing to --blocksize 5.
+parallel: Warning: A record was longer than 5. Increasing to --blocksize 8.
 1>01a02a0a
-parallel: Warning: A record was longer than 5. Increasing to --blocksize 8
 2>0a12a34a
 3>45a6a
 echo '### Test 10M records with too big block';   (    echo start;    seq 1 1 | parallel -uj1 cat /tmp/blocktest\;true;    echo end;    echo start;    seq 1 1 | parallel -uj1 cat /tmp/blocktest\;true;    echo end;    echo start;    seq 1 1 | parallel -uj1 cat /tmp/blocktest\;true;    echo end;   ) | stdout parallel -k --block 10M -j2 --pipe --recstart 'start\n' wc -c |   egrep -v '^0$'
diff --git a/testsuite/wanted-results/parallel-local9 b/testsuite/wanted-results/parallel-local9
index 781d798f..69755a1b 100644
--- a/testsuite/wanted-results/parallel-local9
+++ b/testsuite/wanted-results/parallel-local9
@@ -141,6 +141,7 @@ echo "# --recend '' --files --halt-on-error"
 350eda13a37912d755c9d733d149bdaf  -
 echo '### Test of -j filename - non-existent file';   nice stdout parallel -j no_such_file echo ::: 1
 ### Test of -j filename - non-existent file
+parallel: Error: Parsing of --jobs/-j/--max-procs/-P failed.
 Usage:
 
 parallel [options] [command [arguments]] < list_of_arguments
@@ -178,7 +179,6 @@ please cite:
 This helps funding further development; and it won't cost you a cent.
 If you pay 10000 EUR you should feel free to use GNU Parallel without citing.
 
-parallel: Error: Parsing of --jobs/-j/--max-procs/-P failed.
 echo '### Test of -j filename';   echo 3 >/tmp/jobs_to_run1;   parallel -j /tmp/jobs_to_run1 -v sleep {} ::: 10 8 6 5 4;   # Should give 6 8 10 5 4
 ### Test of -j filename
 sleep 6
diff --git a/testsuite/wanted-results/parallel-tutorial b/testsuite/wanted-results/parallel-tutorial
index 95c67cf5..2a287c2e 100644
--- a/testsuite/wanted-results/parallel-tutorial
+++ b/testsuite/wanted-results/parallel-tutorial
@@ -575,7 +575,7 @@ foo
 3
   parallel --filter-hosts -S 173.194.32.46,$SERVER1 echo ::: bar 
 bar
-parallel: Warning: Removed 173.194.32.46
+parallel: Warning: Removed 173.194.32.46.
   parallel --onall -S $SERVER1,$SERVER2 echo ::: foo bar
 foo
 bar

From fd5622b2c6ca3cb6bb88351ffc188a06295c13db Mon Sep 17 00:00:00 2001
From: Ole Tange 
Date: Fri, 15 May 2015 17:25:19 +0200
Subject: [PATCH 11/14] parallel: --fifo works on csh.

---
 src/parallel                                | 87 ++++++++++++++++-----
 src/parallel.pod                            | 10 ++-
 testsuite/tests-to-run/parallel-local-1s.sh | 10 +++
 testsuite/tests-to-run/parallel-local7.sh   |  2 +-
 testsuite/wanted-results/parallel-local-1s  | 48 ++++++++++++
 5 files changed, 133 insertions(+), 24 deletions(-)
 create mode 100644 testsuite/tests-to-run/parallel-local-1s.sh
 create mode 100644 testsuite/wanted-results/parallel-local-1s

diff --git a/src/parallel b/src/parallel
index c39285f2..20e55878 100755
--- a/src/parallel
+++ b/src/parallel
@@ -933,6 +933,14 @@ sub parse_options {
 	::error("--timeout must be seconds or percentage.");
 	wait_and_exit(255);
     }
+    if(defined $opt::fifo and $opt::cat) {
+	::error("--fifo cannot be combined with --cat.");
+	::wait_and_exit(255);
+    }
+    if((defined $opt::fifo or defined $opt::cat)
+       and not $opt::pipepart) {
+	$opt::pipe = 1;
+    }
     if(defined $opt::minversion) {
 	print $Global::version,"\n";
 	if($Global::version < $opt::minversion) {
@@ -1062,7 +1070,7 @@ sub parse_options {
 
 sub init_globals {
     # Defaults:
-    $Global::version = 20150509;
+    $Global::version = 20150515;
     $Global::progname = 'parallel';
     $Global::infinity = 2**31;
     $Global::debug = 0;
@@ -4619,7 +4627,7 @@ sub compute_number_of_processes {
 	    # This is the child
 	    # The child takes one process slot
 	    # It will be killed later
-	    $SIG{TERM} = $Global::original_sig{TERM};
+	    $SIG{'TERM'} = $Global::original_sig{'TERM'};
             sleep 10000000;
             exit(0);
 	} else {
@@ -5704,7 +5712,8 @@ sub slot {
 
     sub cattail {
 	# Returns:
-	#   $cattail = perl program for: cattail "decompress program" writerpid [file_to_decompress or stdin] [file_to_unlink]
+	#   $cattail = perl program for:
+	#     cattail "decompress program" writerpid [file_to_decompress or stdin] [file_to_unlink]
 	if(not $cattail) {
 	    $cattail = q{
 		# cat followed by tail (possibly with rm as soon at the file is opened)
@@ -6284,6 +6293,46 @@ sub total_failed {
     }
 }
 
+{
+    my $script;
+
+    sub fifo_wrap {
+	# Script to create a fifo, run a command on the fifo
+	# while copying STDIN to the fifo, and finally
+	# remove the fifo and return the exit code of the command.
+	if(not $script) {
+	    # {} == $PARALLEL_TMP for --fifo
+	    # To make it csh compatible a wrapper needs to:
+	    # * mkfifo
+	    # * spawn $command &
+	    # * cat > fifo
+	    # * waitpid to get the exit code from $command
+	    # * be less than 1000 chars long
+	    $script = "perl -e ". ::shell_quote_scalar
+		(::spacefree
+		 (0, q{
+		     ($s,$c,$f) = @ARGV;
+		     # mkfifo $PARALLEL_TMP
+		     system "mkfifo", $f;
+		     # spawn $shell -c $command &
+		     $pid = fork || exec $s, "-c", $c;
+		     open($o,">",$f) || die $!;
+		     # cat > $PARALLEL_TMP
+		     while(sysread(STDIN,$buf,32768)){
+			 syswrite $o, $buf;
+		     }
+		     close $o;
+		     # waitpid to get the exit code from $command
+		     waitpid $pid,0;
+		     # Cleanup
+		     unlink $f;
+		     exit $?/256;
+		  }));
+	}
+	return $script;
+    }
+}
+
 sub wrapped {
     # Wrap command with:
     # * --shellquote
@@ -6337,29 +6386,27 @@ sub wrapped {
 	    }
 	}
 	if($opt::cat) {
-# Append 'unlink {} without affecting $?'
+	    # In '--cat' and '--fifo' {} == $PARALLEL_TMP.
+	    # This is to make it possible to compute $PARALLEL_TMP on
+	    # the fly when running remotely.
+	    # $ENV{PARALLEL_TMP} is set in the remote wrapper before
+	    # the command is run.
+	    #
+	    # Prepend 'cat > $PARALLEL_TMP;'
+	    # Append 'unlink $PARALLEL_TMP without affecting $?'
 	    $command =
-		$self->{'commandline'}->replace_placeholders(["cat > \257<\257>; "], 0, 0).
+		'cat > $PARALLEL_TMP;'.
 		$command.";". postpone_exit_and_cleanup().
 		'$PARALLEL_TMP';
 	} elsif($opt::fifo) {
 	    # Prepend 'mkfifo {}; ('
-	    # Append ') & _PID=$!; cat > {}; wait $_PID; '
-	    # (This makes it fail in csh, but give the correct exit code in bash)
+	    # Append ') & cat > {}; wait; '
 	    # Append 'unlink {} without affecting $?'
-	    # Set $ENV{PARALLEL_TMP} when starting a job
-	    # Set $ENV{PARALLEL_TMP} in the remote wrapper
-	    # mkfifo $PARALLEL_TMP;
-	    # {} = $PARALLEL_TMP;
-	    # (...) &
-	    # cat > $PARALLEL_TMP; wait \$_PID; cleanup $PARALLEL_TMP
-	    # perl -e 'open($fifo,">",shift); while(read){print FIFO};unlink $fifo;waitpid($pid,0);exit $?' $! $PARALLEL_FIFO
-	    $command =
-		"mkfifo \$PARALLEL_TMP\n (".
-		$command.";".
-		') & _PID=$!; cat > $PARALLEL_TMP; wait $_PID; '.
-		postpone_exit_and_cleanup().
-		'$PARALLEL_TMP';
+	    $command = fifo_wrap(). " ".
+		$Global::shell. " ".
+		::shell_quote_scalar($command).
+		' $PARALLEL_TMP'.
+		';';
 	}
 	# Wrap with ssh + tranferring of files
 	$command = $self->sshlogin_wrap($command);
diff --git a/src/parallel.pod b/src/parallel.pod
index f6ec81d3..52e89b23 100644
--- a/src/parallel.pod
+++ b/src/parallel.pod
@@ -461,6 +461,8 @@ will give data to the program on stdin (standard input). With B<--cat>
 GNU B will create a temporary file with the name in {}, so
 you can do: B.
 
+Implies B<--pipe> unless B<--pipepart> is used.
+
 See also B<--fifo>.
 
 
@@ -504,9 +506,9 @@ Compress temporary files. If the output is big and very compressible
 this will take up less disk space in $TMPDIR and possibly be faster
 due to less disk I/O.
 
-GNU B will try B, B, B, B,
-B, B, B, B, B in that order, and use the
-first available.
+GNU B will try B, B, B, B,
+B, B, B, B, B, B, B in that
+order, and use the first available.
 
 
 =item B<--compress-program> I
@@ -654,6 +656,8 @@ with the name in {}, so you can do: B.
 
 Beware: If data is not read from the fifo, the job will block forever.
 
+Implies B<--pipe> unless B<--pipepart> is used.
+
 See also B<--cat>.
 
 
diff --git a/testsuite/tests-to-run/parallel-local-1s.sh b/testsuite/tests-to-run/parallel-local-1s.sh
new file mode 100644
index 00000000..a3b7bbbd
--- /dev/null
+++ b/testsuite/tests-to-run/parallel-local-1s.sh
@@ -0,0 +1,10 @@
+#!/bin/bash
+
+# Simple jobs that never fails
+# Each should be taking 1-3s and be possible to run in parallel
+# I.e.: No race conditions, no logins
+cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj0 -k --joblog /tmp/jl-`basename $0` -L1
+echo '### Test --fifo under csh'
+  csh -c "seq 3000000 | parallel -k --pipe --fifo 'sleep .{#};cat {}|wc -c ; false; echo \$status; false'"; echo exit $?
+
+EOF
diff --git a/testsuite/tests-to-run/parallel-local7.sh b/testsuite/tests-to-run/parallel-local7.sh
index 7ed68ead..ffc45f32 100755
--- a/testsuite/tests-to-run/parallel-local7.sh
+++ b/testsuite/tests-to-run/parallel-local7.sh
@@ -7,7 +7,7 @@ par_tmux_filter() {
 export -f par_tmux_filter
 
 par_tmux() {
-    (stdout parallel --timeout 3 --tmux --delay .3 echo '{}{=$_="\\"x$_=}'; echo $?) | par_tmux_filter
+    (stdout parallel --timeout 3 --tmux --delay .4 echo '{}{=$_="\\"x$_=}'; echo $?) | par_tmux_filter
 }
 export -f par_tmux
 cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj0 --timeout 60 --retries 2 -k --joblog /tmp/jl-`basename $0` -L1
diff --git a/testsuite/wanted-results/parallel-local-1s b/testsuite/wanted-results/parallel-local-1s
new file mode 100644
index 00000000..21a881be
--- /dev/null
+++ b/testsuite/wanted-results/parallel-local-1s
@@ -0,0 +1,48 @@
+echo '### Test --fifo under csh'
+### Test --fifo under csh
+  csh -c "seq 3000000 | parallel -k --pipe --fifo 'sleep .{#};cat {}|wc -c ; false; echo \$status; false'"; echo exit $?
+1048571
+1
+1048579
+1
+1048572
+1
+1048579
+1
+1048579
+1
+1048572
+1
+1048580
+1
+1048576
+1
+1048576
+1
+1048576
+1
+1048576
+1
+1048576
+1
+1048576
+1
+1048576
+1
+1048576
+1
+1048576
+1
+1048576
+1
+1048576
+1
+1048576
+1
+1048576
+1
+1048576
+1
+868800
+1
+exit 22

From 456f63d2bcd69e5825aa1676685111f51348bfd3 Mon Sep 17 00:00:00 2001
From: Ole Tange 
Date: Fri, 15 May 2015 22:05:48 +0200
Subject: [PATCH 12/14] parallel: Fail if -a file is not seekable for
 --pipepart.

---
 doc/release_new_version | 9 ++++++---
 src/parallel            | 4 ++++
 2 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/doc/release_new_version b/doc/release_new_version
index d55b0869..8b1c31cf 100644
--- a/doc/release_new_version
+++ b/doc/release_new_version
@@ -230,12 +230,11 @@ New in this release:
 
 taxator-tk http://algbio.cs.uni-duesseldorf.de/webapps/wa-download/ (check it)
 
-* <> GNU Parallel was used in: Large Scale Author Name Disambiguation in Digital Libraries http://ieeexplore.ieee.org/xpl/abstractReferences.jsp?tp=&arnumber=7004487&url=http%3A%2F%2Fieeexplore.ieee.org%2Fxpls%2Fabs_all.jsp%3Farnumber%3D7004487
+* <> GNU Parallel was used in: Large Scale Author Name Disambiguation in Digital Libraries http://ieeexplore.ieee.org/xpl/abstractReferences.jsp?tp=&arnumber=7004487&url=http%3A%2F%2Fieeexplore.ieee.org%2Fxpls%2Fabs_all.jsp%3Farnumber%3D7004487
 
 * << Update forventet juni >> GNU Parallel was used in: SISRS: Site Identification from Short Read Sequences https://github.com/rachelss/SISRS/
 
-* <> GNU Parallel was used (unfortunately with wrong citation) in: TADSim: Discrete Event-Based Performance Prediction for Temperature-Accelerated Dynamics http://vruehle.de/publications/2015c.pdf
-
+* <> GNU Parallel was used (unfortunately with wrong citation) in: TADSim: Discrete Event-Based Performance Prediction for Temperature-Accelerated Dynamics http://vruehle.de/publications/2015c.pdf
 
 * GNU Parallel was cited in: CIDER: a pipeline for detecting waves of coordinated transcriptional regulation in gene expression time-course data http://biorxiv.org/content/biorxiv/early/2015/03/17/012518.full.pdf
 
@@ -261,6 +260,10 @@ taxator-tk http://algbio.cs.uni-duesseldorf.de/webapps/wa-download/ (check it)
 
 * Run multiple ssh commands in parallel with GNU Parallel http://www.ameir.net/blog/archives/380-run-multiple-ssh-commands-in-parallel-with-gnu-parallel.html
 
+* Importing huge databases faster https://www.lullabot.com/blog/article/importing-huge-databases-faster
+
+* Run multiple ssh commands in parallel with GNU Parallel https://www.ameir.net/blog/archives/380-run-multiple-ssh-commands-in-parallel-with-gnu-parallel.html/comment-page-1
+
 * Parallel? Gnu parallel! https://debian.pro/1834
 
 * Bug fixes and man page updates.
diff --git a/src/parallel b/src/parallel
index 20e55878..1dc9a6f8 100755
--- a/src/parallel
+++ b/src/parallel
@@ -184,6 +184,10 @@ sub pipe_part_files {
     #   @commands that will cat_partial each part
     my ($file) = @_;
     my $buf = "";
+    if(not -f $file) {
+	::error("$file is not a seekable file.");
+	::wait_and_exit(255);
+    }
     my $header = find_header(\$buf,open_or_exit($file));
     # find positions
     my @pos = find_split_positions($file,$opt::blocksize,length $header);

From 87951b34d1329f3e512dba6b7ec238abfb21932e Mon Sep 17 00:00:00 2001
From: Ole Tange 
Date: Wed, 20 May 2015 21:09:33 +0200
Subject: [PATCH 13/14] parallel: --retry-failed implemented.

---
 doc/release_new_version                     |   2 +
 src/parallel                                |  68 +++++++++----
 src/parallel.pod                            |  33 ++++++-
 src/parallel_design.pod                     |  30 +++---
 src/parallel_tutorial.html                  |  98 ++++++++++++++++---
 src/parallel_tutorial.pod                   | 102 +++++++++++++++++---
 testsuite/tests-to-run/parallel-tutorial.sh |   2 +
 7 files changed, 277 insertions(+), 58 deletions(-)

diff --git a/doc/release_new_version b/doc/release_new_version
index 8b1c31cf..ec9a9b8d 100644
--- a/doc/release_new_version
+++ b/doc/release_new_version
@@ -252,6 +252,8 @@ taxator-tk http://algbio.cs.uni-duesseldorf.de/webapps/wa-download/ (check it)
 
 * GNU Parallel was used in: Gene Set Omic Analysis (GSOA) method https://bitbucket.org/srp33/gsoa
 
+* A Quick and Neat :) Orchestrator using GNU Parallel http://www.elsotanillo.net/2015/05/a-quick-and-neat-orchestrator-using-gnu-parallel/
+
 * Execute commands on multiple computers using GNU Parallel (setting up a cluster on the cheap) https://spectraldifferences.wordpress.com/2015/04/26/execute-commands-on-multiple-computers-using-gnu-parallel-setting-up-a-cluster-on-the-cheap/
 
 * Functions and GNU parallel for effective cluster load management http://genomespot.blogspot.dk/2015/04/functions-and-gnu-parallel-for.html
diff --git a/src/parallel b/src/parallel
index 1dc9a6f8..d94b45c6 100755
--- a/src/parallel
+++ b/src/parallel
@@ -704,13 +704,14 @@ sub options_hash {
 	 "results|result|res=s" => \$opt::results,
 	 "resume" => \$opt::resume,
 	 "resume-failed|resumefailed" => \$opt::resume_failed,
+	 "retry-failed|retryfailed" => \$opt::retry_failed,
 	 "silent" => \$opt::silent,
 	 "keep-order|keeporder|k" => \$opt::keeporder,
 	 "no-keep-order|nokeeporder|nok|no-k" => \$opt::nokeeporder,
 	 "group" => \$opt::group,
 	 "g" => \$opt::retired,
 	 "ungroup|u" => \$opt::ungroup,
-	 "linebuffer|linebuffered|line-buffer|line-buffered" => \$opt::linebuffer,
+	 "linebuffer|linebuffered|line-buffer|line-buffered|lb" => \$opt::linebuffer,
 	 "tmux" => \$opt::tmux,
 	 "null|0" => \$opt::0,
 	 "quote|q" => \$opt::q,
@@ -1074,7 +1075,7 @@ sub parse_options {
 
 sub init_globals {
     # Defaults:
-    $Global::version = 20150515;
+    $Global::version = 20150516;
     $Global::progname = 'parallel';
     $Global::infinity = 2**31;
     $Global::debug = 0;
@@ -1340,27 +1341,59 @@ sub open_joblog {
 	::wait_and_exit(255);
     }
     if($opt::joblog) {
-	if($opt::resume || $opt::resume_failed) {
+	if($opt::resume || $opt::resume_failed || $opt::retry_failed) {
 	    if(open(my $joblog_fh, "<", $opt::joblog)) {
 		# Read the joblog
 		$append = <$joblog_fh>; # If there is a header: Open as append later
 		my $joblog_regexp;
-		if($opt::resume_failed) {
+		if($opt::retry_failed) {
 		    # Make a regexp that only matches commands with exit+signal=0
 		    # 4 host 1360490623.067 3.445 1023 1222 0 0 command
 		    $joblog_regexp='^(\d+)(?:\t[^\t]+){5}\t0\t0\t';
-		} else {
-		    # Just match the job number
-		    $joblog_regexp='^(\d+)';
+		    my @group;
+		    while(<$joblog_fh>) {
+			if(/$joblog_regexp/o) {
+			    # This is 30% faster than set_job_already_run($1);
+			    vec($Global::job_already_run,($1||0),1) = 1;
+			    $group[$1-1] = "true";
+			} elsif(/(\d+)\s+\S+(\s+[-0-9.]+){6}\s+(.*)$/) {
+			    $group[$1-1] = $3
+			} else {
+			    chomp;
+			    ::error("Format of '$opt::joblog' is wrong: $_");
+			    ::wait_and_exit(255);
+			}
+		    }
+		    if(@group) {
+			my ($outfh,$name) = ::tmpfile(SUFFIX => ".arg");
+			unlink($name);
+			# Put args into argfile
+			print $outfh map { $_,$/ } @group;
+			seek $outfh, 0, 0;
+			exit_if_disk_full();
+			# Set filehandle to -a
+			@opt::a = ($outfh);
+		    }
+		    # Remove $command (so -a is run)
+		    @ARGV = ();
 		}
-		while(<$joblog_fh>) {
-		    if(/$joblog_regexp/o) {
-			# This is 30% faster than set_job_already_run($1);
-			vec($Global::job_already_run,($1||0),1) = 1;
-		    } elsif(not /\d+\s+[^\s]+\s+([-0-9.]+\s+){6}/) {
-			chomp;
-			::error("Format of '$opt::joblog' is wrong: $_");
-			::wait_and_exit(255);
+		if($opt::resume || $opt::resume_failed) {
+		    if($opt::resume_failed) {
+			# Make a regexp that only matches commands with exit+signal=0
+			# 4 host 1360490623.067 3.445 1023 1222 0 0 command
+			$joblog_regexp='^(\d+)(?:\t[^\t]+){5}\t0\t0\t';
+		    } else {
+			# Just match the job number
+			$joblog_regexp='^(\d+)';
+		    }
+		    while(<$joblog_fh>) {
+			if(/$joblog_regexp/o) {
+			    # This is 30% faster than set_job_already_run($1);
+			    vec($Global::job_already_run,($1||0),1) = 1;
+			} elsif(not /\d+\s+[^\s]+\s+([-0-9.]+\s+){6}/) {
+			    ::error("Format of '$opt::joblog' is wrong: $_");
+			    ::wait_and_exit(255);
+			}
 		    }
 		}
 		close $joblog_fh;
@@ -3411,8 +3444,7 @@ sub tmpfile {
 
 sub tmpname {
     # Select a name that does not exist
-    # Do not create the file as that may cause problems
-    # if you ssh to localhost (or a shared file system) under a different name
+    # Do not create the file as it may be used for creating a socket (by tmux)
     my $name = shift;
     my($tmpname);
     if(not -w $ENV{'TMPDIR'}) {
@@ -6595,7 +6627,7 @@ sub sshlogin_wrap {
 	    }
 	}
 	# Duplicate vars as BASH functions to include post-shellshock functions (v1+v2)
-	# So --env myfunc should also look for BASH_FUNC_myfunc()
+	# So --env myfunc should look for BASH_FUNC_myfunc() and BASH_FUNC_myfunc%%
 	push(@vars, "PARALLEL_PID", "PARALLEL_SEQ",
 	     map { ("BASH_FUNC_$_()", "BASH_FUNC_$_%%") } @vars);
 	# Keep only defined variables
diff --git a/src/parallel.pod b/src/parallel.pod
index 52e89b23..9c1bc121 100644
--- a/src/parallel.pod
+++ b/src/parallel.pod
@@ -84,7 +84,21 @@ If it is a Bash function you need to B the function
 first. To use aliases copy the full environment as described under
 B<--env> and use B instead of B.
 
+If it is a Ksh function you can encode the function in a variable:
+
+  foo() {
+    echo $*;
+  }
+  export fun=`typeset -f foo`; parallel 'eval "$fun";' foo ::: works
+
+To export all functions and make them available when running remote:
+
+  export fun=`typeset -f`; parallel --env fun 'eval "$fun";' foo ::: works
+
 =cut
+# ssh ksh@lo 'foo() { echo $* ; }; export fun="`typeset -f`"; parallel -S ksh@lo --env fun "eval \"\$fun\";"foo ::: works'
+# ssh zsh@lo 'foo() { echo $* ; }; export fun="`typeset -f`"; parallel -S zsh@lo --env fun "eval \"\$fun\";"foo ::: works'
+
 # If it is a zsh function you will need to use this helper function
 # B to export and to set $PARALLEL_SHELL to bash:
 # 
@@ -970,6 +984,8 @@ Implies B<-X> unless B<-m>, B<--xargs>, or B<--pipe> is set.
 
 =item B<--line-buffer>
 
+=item B<--lb>
+
 Buffer output on line basis. B<--group> will keep the output together
 for a whole job. B<--ungroup> allows output to mixup with half a line
 coming from one job and half a line coming from another
@@ -1118,6 +1134,9 @@ defaults to '\n'. To have no record separator use B<--recend "">.
 
 B<--files> is often used with B<--pipe>.
 
+B<--pipe> maxes out at around 1 GB/s input, and 100 MB/s output. If
+performance is important use B<--pipepart>.
+
 See also: B<--recstart>, B<--recend>, B<--fifo>, B<--cat>, B<--pipepart>.
 
 
@@ -1130,7 +1149,8 @@ B<--pipe>, but is much faster. It has a few limitations:
 
 =item Z<>*
 
-The file must be a physical (seekable) file and must be given using B<-a> or B<::::>.
+The file must be a physical (seekable) file (not a stream) and must be
+given using B<-a> or B<::::>.
 
 =item Z<>*
 
@@ -1437,6 +1457,17 @@ commands.
 See also B<--joblog>, B<--resume>.
 
 
+=item B<--retry-failed> (alpha testing)
+
+Retry all failed jobs in joblog. By reading B<--joblog> GNU
+B will figure out the failed jobs and run those again.
+
+B<--retry-failed> ignore the command and arguments: It only looks at
+the joblog.
+
+See also B<--joblog>, B<--resume>, B<--resume-failed>.
+
+
 =item B<--retries> I
 
 If a job fails, retry it on another computer on which it has not
diff --git a/src/parallel_design.pod b/src/parallel_design.pod
index 5b19c4be..9e48e018 100644
--- a/src/parallel_design.pod
+++ b/src/parallel_design.pod
@@ -141,19 +141,29 @@ command.
 {unlink;rmdir;} if($bash=~s/h//) {exit$bash;} exit$csh;' "$?h"
 "$status" {});
 
-{} is really just a tmpfile. The Perl script saves the exit value,
-unlinks the tmpfile, and returns the exit value - no matter if the
-shell is B (using $?) or B<*csh> (using $status).
+{} is set to $PARALLEL_TMP which is a tmpfile. The Perl script saves
+the exit value, unlinks the tmpfile, and returns the exit value - no
+matter if the shell is B (using $?) or B<*csh> (using $status).
 
 =item --fifo
 
-(mkfifo {};
- (<> {};) & _PID=$!; cat > {};  wait $_PID; perl -e '$bash=shift; $csh=shift; for(@ARGV)
-{unlink;rmdir;} if($bash=~s/h//) {exit$bash;} exit$csh;' "$?h"
-"$status" {});
+perl -e '($s,$c,$f) = @ARGV;
+system "mkfifo", $f;
+$pid = fork || exec $s, "-c", $c;
+open($o,">",$f) || die $!;
+while(sysread(STDIN,$buf,32768)){
+syswrite $o, $buf;
+}
+close $o;
+waitpid $pid,0;
+unlink $f;
+exit $?/256;' $shell <> $PARALLEL_TMP
 
-B makes sure the exit value is from that PID. This makes it 
-incompatible with B<*csh>. The Perl script is the same as from B<--cat>.
+This is an elaborate way of: mkfifo {}; run <> in the
+background using $shell; copying STDIN to {}; waiting for background
+to complete; remove {} and exit with the exit code from <>.
+
+It is made this way to be compatible with B<*csh>.
 
 =item --sshlogin I
 
@@ -249,8 +259,6 @@ For B 17000 can be lowered to 2100.
 The interesting areas are title 0..1000 with (title + whole command)
 in 996..1127 and 9331..9636.
 
-
-
 =back
 
 The ordering of the wrapping is important:
diff --git a/src/parallel_tutorial.html b/src/parallel_tutorial.html
index 6e357bb7..d506baa2 100644
--- a/src/parallel_tutorial.html
+++ b/src/parallel_tutorial.html
@@ -55,6 +55,7 @@
   
  • Control the execution
     
      
       
    • Number of simultaneous jobs
      • 
+      
    • Shuffle job order
      • 
       
    • Interactivity
      • 
       
    • A terminal for every job
      • 
       
    • Timing
      • 
@@ -93,6 +94,7 @@
   
    • Semaphore
     
   
      • 
   
    • Informational
      • 
@@ -1068,6 +1070,16 @@
 
 
        parallel --use-cpus-instead-of-cores -N0 sleep 1 :::: num8
      
 
+
      Shuffle job order
      
+
+
      If you have many jobs (e.g. by multiple combinations of input sources), it can be handy to shuffle the jobs, so you get different values run.
      
+
+
        parallel --shuf echo ::: 1 2 3 ::: a b c ::: A B C
      
+
+
      Output:
      
+
+
        All combinations but different order for each run.
      
+
 
      Interactivity
      
 
 
      GNU Parallel can ask the user if a command should be run using --interactive:
      
@@ -1098,7 +1110,7 @@
 
 
      This will tell you to run something similar to:
      
 
-
        tmux -S /tmp/paroRLCx.tms attach
      
+
        tmux -S /tmp/tmsrPrO0 attach
      
 
 
      Using normal tmux keystrokes (CTRL-b n or CTRL-b p) you can cycle between windows of the running jobs. When a job is finished it will pause for 10 seconds before closing the window.
      
 
@@ -1233,9 +1245,9 @@
 
 
      Termination
      
 
-
      For certain jobs there is no need to continue if one of the jobs fails and has an exit code != 0. GNU Parallel will stop spawning new jobs with --halt 1:
      
+
      For certain jobs there is no need to continue if one of the jobs fails and has an exit code != 0. GNU Parallel will stop spawning new jobs with --halt soon,fail=1:
      
 
-
        parallel -j2 --halt 1 echo {}\; exit {} ::: 0 0 1 2 3
      
+
        parallel -j2 --halt soon,fail=1 echo {}\; exit {} ::: 0 0 1 2 3
      
 
 
      Output:
      
 
@@ -1248,9 +1260,9 @@
   parallel: Starting no more jobs. Waiting for 1 jobs to finish. This job failed:
   echo 2; exit 2
    • -

    With --halt 2 the running jobs will be killed immediately:

    +

    With --halt now,fail=1 the running jobs will be killed immediately:

    -
      parallel -j2 --halt 2 echo {}\; exit {} ::: 0 0 1 2 3
    +
      parallel -j2 --halt now,fail=1 echo {}\; exit {} ::: 0 0 1 2 3

    Output:

    @@ -1260,23 +1272,38 @@ parallel: This job failed: echo 1; exit 1 -

    If --halt is given a percentage this percentage of the jobs must fail (though minimum 3) before GNU Parallel stops spawning more jobs:

    +

    If --halt is given a percentage this percentage of the jobs must fail before GNU Parallel stops spawning more jobs:

    -
      parallel -j2 --halt 20% echo {}\; exit {} ::: 0 0 1 2 3 4 5 6 7
    +
      parallel -j2 --halt soon,fail=20% echo {}\; exit {} ::: 0 1 2 3 4 5 6 7 8 9

    Output:

      0
-  0
   1
+  parallel: This job failed:
+  echo 1; exit 1
+  2
+  parallel: This job failed:
+  echo 2; exit 2
+  parallel: Starting no more jobs. Waiting for 1 jobs to finish.
+  3
+  parallel: This job failed:
+  echo 3; exit 3
    + +

    If you are looking for success instead of failures, you can use success:

    + +
      parallel -j2 --halt soon,success=1 echo {}\; exit {} ::: 1 2 3 0 4 5 6
    + +

    Output:

    + +
      1
   2
   3
-  4
-  parallel: Starting no more jobs. Waiting for 2 jobs to finish. This job failed:
-      echo 4; exit 4
-  5
-  parallel: Starting no more jobs. Waiting for 1 jobs to finish. This job failed:
-      echo 5; exit 5
    + 0 + parallel: This job succeeded: + echo 0; exit 0 + parallel: Starting no more jobs. Waiting for 1 jobs to finish. + 4

    GNU Parallel can retry the command with --retries. This is useful if a command fails for unknown reasons now and then.

    @@ -1382,6 +1409,12 @@

    Output: Same as above.

    +

    Or newline:

    + +
      # This gives a \n between $SERVER1 and $SERVER2
+  SERVERS="`echo $SERVER1; echo $SERVER2`"
+  parallel -S "$SERVERS" echo ::: running on more hosts
    +

    The can also be read from a file (replace user@ with the user on $SERVER2):

      echo $SERVER1 > nodefile
@@ -1401,6 +1434,24 @@
 
 
      force 4 cpus on server
    
 
+
    Servers can be put into groups by prepending '@groupname' to the server and the group can then be selected by appending '@groupname' to the argument if using '--hostgrp'.
    
+
+
      parallel --hostgrp -S @grp1/$SERVER1 -S @grp2/SERVER2 echo {} ::: run_on_grp1@grp1 run_on_grp2@grp2
    
+
+
    Output:
    
+
+
      run_on_grp1
+  run_on_grp2
    
+
+
    A host can be in multiple groups by separating groups with '+', and you can force GNU parallel to limit the groups on which the command can be run with '-S @groupname':
    
+
+
      parallel -S @grp1 -S @grp1+grp2/$SERVER1 -S @grp2/SERVER2 echo {} ::: run_on_grp1 also_grp1
    
+
+
    Output:
    
+
+
      run_on_grp1
+  also_grp1
    
+
 
    Transferring files
    
 
 
    GNU Parallel can transfer the files to be processed to the remote host. It does that using rsync.
    
@@ -2039,6 +2090,25 @@
   The third finished
   The fourth finished
    +

    Timeout

    + +

    With --semaphoretimeout you can force running the command anyway after a period (postive number) or give up (negative number):

    + +
      sem --id foo -u 'echo Slow started; sleep 5; echo Slow ended' &&
+  sem --id foo --semaphoretimeout 1 'echo Force this running after 1 sec' &&
+  sem --id foo --semaphoretimeout -2 'echo Give up after 1 sec'
+  sem --id foo --wait
    + +

    Output:

    + +
      Slow started
+  parallel: Warning: Semaphore timed out. Stealing the semaphore.
+  Force this running after 1 sec
+  Slow ended
+  parallel: Warning: Semaphore timed out. Exiting.
    + +

    Note how the 'Give up' was not run.

    +

    Informational

    GNU Parallel has some options to give short information about the configuration.

    diff --git a/src/parallel_tutorial.pod b/src/parallel_tutorial.pod index 02708f20..211709e9 100644 --- a/src/parallel_tutorial.pod +++ b/src/parallel_tutorial.pod @@ -996,6 +996,18 @@ GNU Parallel can base it on the number of CPUs: parallel --use-cpus-instead-of-cores -N0 sleep 1 :::: num8 +=head2 Shuffle job order + +If you have many jobs (e.g. by multiple combinations of input +sources), it can be handy to shuffle the jobs, so you get different +values run. + + parallel --shuf echo ::: 1 2 3 ::: a b c ::: A B C + +Output: + + All combinations but different order for each run. + =head2 Interactivity GNU Parallel can ask the user if a command should be run using --interactive: @@ -1027,7 +1039,7 @@ Using tmux GNU Parallel can start a terminal for every job run: This will tell you to run something similar to: - tmux -S /tmp/paroRLCx.tms attach + tmux -S /tmp/tmsrPrO0 attach Using normal tmux keystrokes (CTRL-b n or CTRL-b p) you can cycle between windows of the running jobs. When a job is finished it will @@ -1178,9 +1190,9 @@ Note how seq 1 2 3 have been repeated because they had exit value != 0. For certain jobs there is no need to continue if one of the jobs fails and has an exit code != 0. GNU Parallel will stop spawning new jobs -with --halt 1: +with --halt soon,fail=1: - parallel -j2 --halt 1 echo {}\; exit {} ::: 0 0 1 2 3 + parallel -j2 --halt soon,fail=1 echo {}\; exit {} ::: 0 0 1 2 3 Output: @@ -1193,9 +1205,9 @@ Output: parallel: Starting no more jobs. Waiting for 1 jobs to finish. This job failed: echo 2; exit 2 -With --halt 2 the running jobs will be killed immediately: +With --halt now,fail=1 the running jobs will be killed immediately: - parallel -j2 --halt 2 echo {}\; exit {} ::: 0 0 1 2 3 + parallel -j2 --halt now,fail=1 echo {}\; exit {} ::: 0 0 1 2 3 Output: @@ -1206,23 +1218,38 @@ Output: echo 1; exit 1 If --halt is given a percentage this percentage of the jobs must fail -(though minimum 3) before GNU Parallel stops spawning more jobs: +before GNU Parallel stops spawning more jobs: - parallel -j2 --halt 20% echo {}\; exit {} ::: 0 0 1 2 3 4 5 6 7 + parallel -j2 --halt soon,fail=20% echo {}\; exit {} ::: 0 1 2 3 4 5 6 7 8 9 Output: + + 0 + 1 + parallel: This job failed: + echo 1; exit 1 + 2 + parallel: This job failed: + echo 2; exit 2 + parallel: Starting no more jobs. Waiting for 1 jobs to finish. + 3 + parallel: This job failed: + echo 3; exit 3 - 0 - 0 +If you are looking for success instead of failures, you can use success: + + parallel -j2 --halt soon,success=1 echo {}\; exit {} ::: 1 2 3 0 4 5 6 + +Output: + 1 2 3 + 0 + parallel: This job succeeded: + echo 0; exit 0 + parallel: Starting no more jobs. Waiting for 1 jobs to finish. 4 - parallel: Starting no more jobs. Waiting for 2 jobs to finish. This job failed: - echo 4; exit 4 - 5 - parallel: Starting no more jobs. Waiting for 1 jobs to finish. This job failed: - echo 5; exit 5 GNU Parallel can retry the command with --retries. This is useful if a command fails for unknown reasons now and then. @@ -1332,6 +1359,12 @@ Or they can be separated by ,: Output: Same as above. +Or newline: + + # This gives a \n between $SERVER1 and $SERVER2 + SERVERS="`echo $SERVER1; echo $SERVER2`" + parallel -S "$SERVERS" echo ::: running on more hosts + The can also be read from a file (replace user@ with the user on $SERVER2): echo $SERVER1 > nodefile @@ -1352,6 +1385,28 @@ Output: force 4 cpus on server +Servers can be put into groups by prepending '@groupname' to the +server and the group can then be selected by appending '@groupname' to +the argument if using '--hostgrp'. + + parallel --hostgrp -S @grp1/$SERVER1 -S @grp2/SERVER2 echo {} ::: run_on_grp1@grp1 run_on_grp2@grp2 + +Output: + + run_on_grp1 + run_on_grp2 + +A host can be in multiple groups by separating groups with '+', and +you can force GNU B to limit the groups on which the command +can be run with '-S @groupname': + + parallel -S @grp1 -S @grp1+grp2/$SERVER1 -S @grp2/SERVER2 echo {} ::: run_on_grp1 also_grp1 + +Output: + + run_on_grp1 + also_grp1 + =head2 Transferring files GNU Parallel can transfer the files to be processed to the remote @@ -2052,6 +2107,25 @@ Output: The third finished The fourth finished +=head2 Timeout + +With --semaphoretimeout you can force running the command anyway after +a period (postive number) or give up (negative number): + + sem --id foo -u 'echo Slow started; sleep 5; echo Slow ended' && + sem --id foo --semaphoretimeout 1 'echo Force this running after 1 sec' && + sem --id foo --semaphoretimeout -2 'echo Give up after 1 sec' + sem --id foo --wait + +Output: + + Slow started + parallel: Warning: Semaphore timed out. Stealing the semaphore. + Force this running after 1 sec + Slow ended + parallel: Warning: Semaphore timed out. Exiting. + +Note how the 'Give up' was not run. =head1 Informational diff --git a/testsuite/tests-to-run/parallel-tutorial.sh b/testsuite/tests-to-run/parallel-tutorial.sh index 00c3726e..58fe91f9 100644 --- a/testsuite/tests-to-run/parallel-tutorial.sh +++ b/testsuite/tests-to-run/parallel-tutorial.sh @@ -35,6 +35,8 @@ perl -ne '$/="\n\n"; /^Output/../^[^O]\S/ and next; /^ / and print;' ../../src/ s/\d{10}.\d{3}\s+..\d+/TIMESTAMP\t9.999/g; # Version s/201\d{5}/VERSION/g; + # [123] [abc] [ABC] + s/^[123] [abc] [ABC]$/123 abc ABC/g; # Remote script s/(PARALLEL_PID\D+)\d+/${1}000000/g; # /usr/bin/time -f %e From 03a69b38ce7aa3b91fe21b1e4895b768cb3a5622 Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Wed, 20 May 2015 22:37:10 +0200 Subject: [PATCH 14/14] Fixed bug #45144: Does not work on Solaris "zone" containers. --- src/parallel | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/parallel b/src/parallel index d94b45c6..9b005fe1 100755 --- a/src/parallel +++ b/src/parallel @@ -3707,6 +3707,10 @@ sub which { ::debug("init", "shell path $shellpath\n"); $shellpath and last; } + if($testpid == $parent_of_ref->{$testpid}) { + # In Solaris zones, the PPID of the zsched process is itself + last; + } $testpid = $parent_of_ref->{$testpid}; } return $shellpath;