Merge tag 'det'

Conflicts:
	src/parallel

doc/release_new_version

@@ -231,7 +231,7 @@ New in this release:
 
 * GNU Parallel was cited in: Experimental study on the Wind Farm Substation Cable Installation Problem http://i11www.iti.uni-karlsruhe.de/_media/teaching/theses/ma-schmitz-14.pdf
 
-* <<afventer>> Comparing the CarbonTracker and TM5-4DVar data assimilation systems for CO2 surface flux inversions http://www.atmos-chem-phys-discuss.net/15/8883/2015/acpd-15-8883-2015-discussion.html
+* <<afventer - rykket >> Comparing the CarbonTracker and TM5-4DVar data assimilation systems for CO2 surface flux inversions http://www.atmos-chem-phys-discuss.net/15/8883/2015/acpd-15-8883-2015-discussion.html
 
 * <<afventer opdatering>> CIDER: a pipeline for detecting waves of coordinated transcriptional regulation in gene expression time-course data http://biorxiv.org/content/biorxiv/early/2015/03/17/012518.full.pdf
 
@@ -247,10 +247,16 @@ taxator-tk http://algbio.cs.uni-duesseldorf.de/webapps/wa-download/ (check it)
 
 * GNU Parallel was used in: http://search.cpan.org/~ajpage/Bio-Roary-2.0.7/lib/Bio/Roary/JobRunner/Parallel.pm
 
+* GNU Parallel was used in: Rapid haploid SNP calling https://github.com/tseemann/snippy
+
+* << afventer svar fra Rachel >> GNU Parallel was used in: SISRS: Site Identification from Short Read Sequences https://github.com/rachelss/SISRS/
+
 * Pictures and Metadata http://www.ozzy.no/2015/02/05/pictures-and-metadata/
 
 * Task automation with bash and parallel https://biowize.wordpress.com/2015/03/23/task-automation-with-bash-and-parallel/
 
+* How To: Speed Up File Transfers in Linux using RSync with GNU Parallel http://www.yourownlinux.com/2015/04/speed-up-file-transfers-using-rsync-with-gnu-parallel.html
+
 * Mirroring a Git Repository https://avacariu.me/articles/mirroring-a-git-repository.html
 
 * Add an ENVI header to JAXA Global Mangrove Watch PALSAR tiles https://spectraldifferences.wordpress.com/2015/03/24/add-an-envi-header-to-jaxa-global-mangrove-watch-palsar-tiles/

src/parallel

@@ -1051,7 +1051,7 @@ sub parse_options {
 
 sub init_globals {
     # Defaults:
-    $Global::version = 20150409;
+    $Global::version = 20150416;
     $Global::progname = 'parallel';
     $Global::infinity = 2**31;
     $Global::debug = 0;
@@ -2986,7 +2986,7 @@ sub reaper {
         $job or next;
         $job->set_exitstatus($? >> 8);
         $job->set_exitsignal($? & 127);
-        debug("run", "died (", $job->exitstatus(), "): ", $job->seq());
+        debug("run", "seq ",$job->seq()," died (", $job->exitstatus(), ")");
         $job->set_endtime(::now());
         if($stiff == $Global::tty_taken) {
             # The process that died had the tty => release it
@@ -3031,6 +3031,72 @@ sub reaper {
 
 sub __USAGE__ {}
 
+sub killall {
+    # Kill all jobs
+    # Send all jobs TERM
+    # Wait
+    # Send all jobs TERM
+    # Wait
+    # Send all jobs KILL
+    # Send all (grand*)children KILL
+
+    $Global::start_no_new_jobs ||= 1;
+    # pids of the all children and (grand*)children
+    # before we start the blood bath
+    my @family_pids = family_pids(keys %Global::running);
+    # Send jobs TERM
+    kill "TERM", keys %Global::running;
+    # Wait up to 200 ms
+    # Send jobs TERM (again)
+    my $sleepsum = 0;
+    my $sleep = 0;
+    for (; kill(0, keys %Global::running) and $sleepsum < 200;
+	 $sleepsum += $sleep) {
+	# This can change %Global::running
+	$sleep = ::reap_usleep($sleep);
+    }
+    kill "TERM", keys %Global::running;
+    # Wait up to 200 ms
+    # Send jobs KILL
+    $sleepsum = 0;
+    $sleep = 0;
+    for (; kill(0, keys %Global::running) and $sleepsum < 200;
+	 $sleepsum += $sleep) {
+	# This can change %Global::running
+	$sleep = ::reap_usleep($sleep);
+    }
+    kill "KILL", keys %Global::running;
+    # Send all (grand*)children KILL (if there are any left)
+    kill "KILL", @family_pids;
+}
+
+sub family_pids {
+    # Find the pids with this->pid as (grand)*parent
+    # Input:
+    #   @parents = pids of parents
+    # Returns:
+    #   @pids = pids of (grand)*children
+    my @parents = @_;
+    my @pids;
+
+    my ($children_of_ref, $parent_of_ref, $name_of_ref) = ::pid_table();
+
+    my @more = @parents;
+    # While more (grand)*children
+    while(@more) {
+	my @m;
+	push @pids, @more;
+	for my $parent (@more) {
+	    if($children_of_ref->{$parent}) {
+		# add the children of this parent
+		push @m, @{$children_of_ref->{$parent}};
+	    }
+	}
+	@more = @m;
+    }
+    return (@pids);
+}
+
 sub wait_and_exit {
     # If we do not wait, we sometimes get segfault
     # Returns: N/A
@@ -3038,9 +3104,7 @@ sub wait_and_exit {
     unlink keys %Global::unlink;
     if($error) {
 	# Kill all without printing
-	for my $job (values %Global::running) {
+	killall();
-	    $job->kill();
-	}
     }
     for (keys %Global::unkilled_children) {
         kill 9, $_;
@@ -3359,13 +3423,14 @@ sub multiply_binary_prefix {
 }
 
 {
-    my ($disk_full_fh, $b8193, $name);
+    my ($disk_full_fh, $b8193, $error_printed);
     sub exit_if_disk_full {
 	# Checks if $TMPDIR is full by writing 8kb to a tmpfile
 	# If the disk is full: Exit immediately.
 	# Returns:
 	#   N/A
 	if(not $disk_full_fh) {
+	    my $name;
 	    ($disk_full_fh, $name) = ::tmpfile(SUFFIX => ".df");
 	    # Separate unlink due to NFS dealing badly with File::Temp
 	    unlink $name;
@@ -3392,8 +3457,11 @@ sub multiply_binary_prefix {
 	   or
 	   tell $disk_full_fh != 8193) {
 	    # On raspbian the disk can be full except for 10 chars.
+	    if(not $error_printed) {
 		::error("Output is incomplete. Cannot append to buffer file in $ENV{'TMPDIR'}. Is the disk full?\n");
 		::error("Change \$TMPDIR with --tmpdir or use --compress.\n");
+		$error_printed = 1;
+	    }
 	    ::wait_and_exit(255);
 	}
 	truncate $disk_full_fh, 0;
@@ -3507,6 +3575,9 @@ sub which {
 	    # Filter for SysV-style `ps`
 	    my $sysv = q( ps -ef | perl -ane '1..1 and /^(.*)CO?MM?A?N?D/ and $s=length $1;).
 		q(s/^.{$s}//; print "@F[1,2] $_"' );
+	    # Crazy msys: ' is not accepted on the cmd line, but " are treated as '
+	    my $msys = q( ps -ef | perl -ane "1..1 and /^(.*)CO?MM?A?N?D/ and $s=length $1;).
+		q(s/^.{$s}//; print qq{@F[1,2] $_}" );
 	    # BSD-style `ps`
 	    my $bsd = q(ps -o pid,ppid,command -ax);
 	    %pid_parentpid_cmd =
@@ -3521,7 +3592,7 @@ sub which {
 	     'hpux' => $sysv,
 	     'linux' => $sysv,
 	     'mirbsd' => $bsd,
-	     'msys' => $sysv,
+	     'msys' => $msys,
 	     'MSWin32' => $sysv,
 	     'netbsd' => $bsd,
 	     'nto' => $sysv,
@@ -4441,8 +4512,10 @@ sub compute_number_of_processes {
 	    # The child takes one process slot
 	    # It will be killed later
 	    $SIG{TERM} = $Global::original_sig{TERM};
-	    sleep 10000000;
+	    # Exec 'sleep' to save RAM
-	    exit(0);
+	    # 32000 seconds should be plenty, and should be supported
+	    # on all platforms.
+	    exec("sleep","32000");
 	} else {
 	    # Failed to spawn
             $max_system_proc_reached = 1;
@@ -5702,8 +5775,8 @@ sub filter_through_compress {
 	# The tmpfile is used to tell the reader that the writer has started,
 	# so unlink it to start with.
 	unlink $self->fh($fdno,'name');
-	my $wpid = open(my $fdw,"|-", empty_input_detector().
+        my $wpid = open(my $fdw,"|-", "(".empty_input_detector().
-			"| ($opt::compress_program) >>".
+			"| ($opt::compress_program)) >>".
 			$self->fh($fdno,'name')) || die $?;
 	$self->set_fh($fdno,'w',$fdw);
 	$self->set_fh($fdno,'wpid',$wpid);
@@ -5927,6 +6000,7 @@ sub kill {
     my $self = shift;
     my @signals = @_;
     my @family_pids = $self->family_pids();
+
     # Record this jobs as failed
     $self->set_exitstatus(-1);
     # Send two TERMs to give time to clean up
@@ -6999,7 +7073,7 @@ sub print {
 	    }
 	    next;
 	}
-	::debug("print", "File descriptor $fdno (", $self->fh($fdno,"name"), "):");
+	::debug("print", "File descriptor $fdno (", $self->fh($fdno,"name"), "):\n");
 	if($opt::files) {
 	    $self->files_print($fdno,$in_fh,$out_fd);
 	} elsif($opt::linebuffer) {
@@ -7024,14 +7098,18 @@ sub files_print {
     my $self = shift;
     my ($fdno,$in_fh,$out_fd) = @_;
 
-    # If --compress: $in_fh must be closed first.
+    # If the job is dead: close printing fh. Needed for --compress
     close $self->fh($fdno,"w");
     if($? and $opt::compress) {
-	::error($opt::decompress_program." failed.\n");
+	::error($opt::compress_program." failed.\n");
 	$self->set_exitstatus(255);
     }
+    if($opt::compress) {
+	# Kill the decompressor which will not be needed
+	CORE::kill "TERM", $self->fh($fdno,"rpid");
+    }
     close $in_fh;
-    # TODO if $?: die(?)
+
     if($opt::pipe and $self->virgin()) {
 	# Nothing was printed to this job:
 	# cleanup unused tmp files if --files was set
@@ -7054,7 +7132,7 @@ sub linebuffer_print {
 	# If the job is dead: close printing fh. Needed for --compress
 	close $self->fh($fdno,"w");
 	if($? and $opt::compress) {
-	    ::error($opt::decompress_program." failed.\n");
+	    ::error($opt::compress_program." failed.\n");
 	    $self->set_exitstatus(255);
 	}
 	if($opt::compress) {
@@ -7123,7 +7201,10 @@ sub linebuffer_print {
 	} else {
 	    # decompress done: close fh
 	    close $in_fh;
-	    # TODO if $?: die(?)
+	    if($? and $opt::compress) {
+		::error($opt::decompress_program." failed.\n");
+		$self->set_exitstatus(255);
+	    }
 	}
     }
 }
@@ -7134,7 +7215,7 @@ sub tag_print {
     my $buf;
     close $self->fh($fdno,"w");
     if($? and $opt::compress) {
-	::error($opt::decompress_program." failed.\n");
+	::error($opt::compress_program." failed.\n");
 	$self->set_exitstatus(255);
     }
     seek $in_fh, 0, 0;
@@ -7163,6 +7244,10 @@ sub tag_print {
 	$self->add_returnsize($outputlength);
     }
     close $in_fh;
+    if($? and $opt::compress) {
+	::error($opt::decompress_program." failed.\n");
+	$self->set_exitstatus(255);
+    }
 }
 
 sub normal_print {
@@ -7171,7 +7256,7 @@ sub normal_print {
     my $buf;
     close $self->fh($fdno,"w");
     if($? and $opt::compress) {
-	::error($opt::decompress_program." failed.\n");
+	::error($opt::compress_program." failed.\n");
 	$self->set_exitstatus(255);
     }
     seek $in_fh, 0, 0;
@@ -7193,6 +7278,10 @@ sub normal_print {
 	$self->add_returnsize($outputlength);
     }
     close $in_fh;
+    if($? and $opt::compress) {
+	::error($opt::decompress_program." failed.\n");
+	$self->set_exitstatus(255);
+    }
 }
 
 sub print_joblog {

testsuite/Makefile

@@ -7,7 +7,7 @@ testsuite: 3
 	make stopvm
 
 1: ../src/parallel tests-to-run/* wanted-results/* prereqlocal startdb prereqremote
-	time bash Start.sh - mem || true
+	TRIES=1 time bash Start.sh - mem || true
 	touch ~/.parallel/will-cite
 	make stopvm
 

testsuite/tests-to-run/parallel-local-0.3s.sh

@@ -84,6 +84,56 @@ echo '### bug #44546: If --compress-program fails: fail'
   parallel --line-buffer --compress-program false echo \;ls ::: /no-existing; echo $?
   parallel --compress-program false echo \;ls ::: /no-existing; echo $?
 
+echo 'bug #41613: --compress --line-buffer - no newline';
+  echo 'pipe compress tagstring'
+  perl -e 'print "O"'| parallel --compress --tagstring {#} --pipe --line-buffer cat;  echo "K"
+  echo 'pipe compress notagstring'
+  perl -e 'print "O"'| parallel --compress --pipe --line-buffer cat;  echo "K"
+  echo 'pipe nocompress tagstring'
+  perl -e 'print "O"'| parallel --tagstring {#} --pipe --line-buffer cat;  echo "K"
+  echo 'pipe nocompress notagstring'
+  perl -e 'print "O"'| parallel --pipe --line-buffer cat;  echo "K"
+  echo 'nopipe compress tagstring'
+  parallel --compress --tagstring {#} --line-buffer echo {} O ::: -n;  echo "K"
+  echo 'nopipe compress notagstring'
+  parallel --compress --line-buffer echo {} O ::: -n;  echo "K"
+  echo 'nopipe nocompress tagstring'
+  parallel --tagstring {#} --line-buffer echo {} O ::: -n;  echo "K"
+  echo 'nopipe nocompress notagstring'
+  parallel --line-buffer echo {} O ::: -n;  echo "K"
+
+echo 'Compress with failing (de)compressor'
+  parallel -k --tag --compress --compress-program 'cat;true'  --decompress-program 'cat;true'  echo ::: tag true true
+  parallel -k --tag --compress --compress-program 'cat;false' --decompress-program 'cat;true'  echo ::: tag false true
+  parallel -k --tag --compress --compress-program 'cat;false' --decompress-program 'cat;false' echo ::: tag false false
+  parallel -k --tag --compress --compress-program 'cat;true'  --decompress-program 'cat;false' echo ::: tag true false
+  parallel -k --compress --compress-program 'cat;true'  --decompress-program 'cat;true'  echo ::: true true
+  parallel -k --compress --compress-program 'cat;false' --decompress-program 'cat;true'  echo ::: false true
+  parallel -k --compress --compress-program 'cat;false' --decompress-program 'cat;false' echo ::: false false
+  parallel -k --compress --compress-program 'cat;true'  --decompress-program 'cat;false' echo ::: true false
+  parallel -k --line-buffer --compress --compress-program 'cat;true'  --decompress-program 'cat;true'  echo ::: line-buffer true true
+  parallel -k --line-buffer --compress --compress-program 'cat;false' --decompress-program 'cat;true'  echo ::: line-buffer false true
+  parallel -k --line-buffer --compress --compress-program 'cat;false' --decompress-program 'cat;false' echo ::: line-buffer false false
+  parallel -k --tag --line-buffer --compress --compress-program 'cat;true'  --decompress-program 'cat;false' echo ::: tag line-buffer true false
+  parallel -k --tag --line-buffer --compress --compress-program 'cat;true'  --decompress-program 'cat;true'  echo ::: tag line-buffer true true
+  parallel -k --tag --line-buffer --compress --compress-program 'cat;false' --decompress-program 'cat;true'  echo ::: tag line-buffer false true
+  parallel -k --tag --line-buffer --compress --compress-program 'cat;false' --decompress-program 'cat;false' echo ::: tag line-buffer false false
+  parallel -k --tag --line-buffer --compress --compress-program 'cat;true'  --decompress-program 'cat;false' echo ::: tag line-buffer true false
+  parallel -k --files --compress --compress-program 'cat;true'  --decompress-program 'cat;true'  echo ::: files true true   | parallel rm
+  parallel -k --files --compress --compress-program 'cat;false' --decompress-program 'cat;true'  echo ::: files false true  | parallel rm
+  parallel -k --files --compress --compress-program 'cat;false' --decompress-program 'cat;false' echo ::: files false false | parallel rm
+  parallel -k --files --compress --compress-program 'cat;true'  --decompress-program 'cat;false' echo ::: files true false  | parallel rm
+
+echo 'bug #44250: pxz complains File format not recognized but decompresses anyway'
+  # The first line dumps core if run from make file. Why?!
+  stdout parallel --compress --compress-program pxz ls /{} ::: OK-if-missing-file
+  stdout parallel --compress --compress-program pixz --decompress-program 'pixz -d' ls /{}  ::: OK-if-missing-file
+  stdout parallel --compress --compress-program pixz --decompress-program 'pixz -d' true ::: OK-if-no-output
+  stdout parallel --compress --compress-program pxz true ::: OK-if-no-output
+
+echo 'bug #41613: --compress --line-buffer no newline';
+  perl -e 'print "It worked"'| parallel --pipe --compress --line-buffer cat; echo
+
 echo '### bug #44614: --pipepart --header off by one'
   seq 10 >/tmp/parallel_44616; 
     parallel --pipepart -a /tmp/parallel_44616 -k --block 5 'echo foo; cat'; 

testsuite/tests-to-run/parallel-local9.sh

@@ -8,16 +8,6 @@ NICEPAR="nice nice parallel"
 export NICEPAR
 
 cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | stdout parallel -vj4 -k --joblog /tmp/jl-`basename $0` -L1
-echo 'bug #44250: pxz complains File format not recognized but decompresses anyway'
-  # The first line dumps core if run from make file. Why?!
-  stdout parallel --compress --compress-program pxz ls /{} ::: OK-if-missing-file
-  stdout parallel --compress --compress-program pixz --decompress-program 'pixz -d' ls /{}  ::: OK-if-missing-file
-  stdout parallel --compress --compress-program pixz --decompress-program 'pixz -d' true ::: OK-if-no-output
-  stdout parallel --compress --compress-program pxz true ::: OK-if-no-output
-
-echo 'bug #41613: --compress --line-buffer no newline';
-  perl -e 'print "It worked"'| $NICEPAR --pipe --compress --line-buffer cat; echo
-
 echo 'bug #41613: --compress --line-buffer no --tagstring';
   diff 
     <(nice perl -e 'for("x011".."x110"){print "$_\t", ("\n", map { rand } (1..100000)) }'| 
@@ -40,24 +30,6 @@ echo 'bug #41613: --compress --line-buffer with --tagstring';
     >/dev/null 
   || (echo 'Good: --line-buffer matters'; false) && echo 'Bad: --line-buffer not working'
 
-echo 'bug #41613: --compress --line-buffer - no newline';
-  echo 'pipe compress tagstring'
-  perl -e 'print "O"'| $NICEPAR --compress --tagstring {#} --pipe --line-buffer cat;  echo "K"
-  echo 'pipe compress notagstring'
-  perl -e 'print "O"'| $NICEPAR --compress --pipe --line-buffer cat;  echo "K"
-  echo 'pipe nocompress tagstring'
-  perl -e 'print "O"'| $NICEPAR --tagstring {#} --pipe --line-buffer cat;  echo "K"
-  echo 'pipe nocompress notagstring'
-  perl -e 'print "O"'| $NICEPAR --pipe --line-buffer cat;  echo "K"
-  echo 'nopipe compress tagstring'
-  $NICEPAR --compress --tagstring {#} --line-buffer echo {} O ::: -n;  echo "K"
-  echo 'nopipe compress notagstring'
-  $NICEPAR --compress --line-buffer echo {} O ::: -n;  echo "K"
-  echo 'nopipe nocompress tagstring'
-  $NICEPAR --tagstring {#} --line-buffer echo {} O ::: -n;  echo "K"
-  echo 'nopipe nocompress notagstring'
-  $NICEPAR --line-buffer echo {} O ::: -n;  echo "K"
-
 echo 'bug #41412: --timeout + --delay causes deadlock';
   seq 10 | parallel -j10 --timeout 1 --delay .3 echo;
   parallel -j3 --timeout 1 --delay 2 echo ::: 1 2 3;

testsuite/tests-to-run/parallel-polarhome.sh

@@ -16,7 +16,6 @@ echo '### Tests on polarhome machines'
 echo 'Setup on polarhome machines'
 stdout parallel -kj0 ssh -oLogLevel=quiet {} mkdir -p bin ::: $POLAR &
 
-
 test_empty_cmd() {
     echo '### Test if empty command in process list causes problems'
     perl -e '$0=" ";sleep 1' &
@@ -36,7 +35,8 @@ copy_and_test() {
       perl -pe 's:/[a-z0-9_]+.arg:/XXXXXXXX.arg:gi; s/\d\d\d\d/0000/gi;'
 }
 export -f copy_and_test
-stdout parallel -j0 -k --retries 5 --timeout 80 --delay 0.1 --tag  -v copy_and_test {} ::: $POLAR
+# 20150414 --timeout 80 -> 40
+stdout parallel -j0 -k --retries 5 --timeout 40 --delay 0.1 --tag  -v copy_and_test {} ::: $POLAR
 
 cat /tmp/test_empty_cmd
 rm /tmp/test_empty_cmd

testsuite/tests-to-run/parallel-tutorial.sh

@@ -15,6 +15,8 @@ perl -ne '$/="\n\n"; /^Output/../^[^O]\S/ and next; /^  / and print;' ../../src/
             s/zenity/zenity --timeout=12/;
             s:/usr/bin/time:/usr/bin/time -f %e:;
             s:ignored_vars:ignored_vars|sort:;
+            # Remove \n to join all joblogs into the previous block
+            s:cat /tmp/log\n:cat /tmp/log;:;
             # When parallelized: Sleep to make sure the abc-files are made
             /%head1/ and $_.="sleep .3\n\n"x10;
 ' |
@@ -44,6 +46,10 @@ perl -ne '$/="\n\n"; /^Output/../^[^O]\S/ and next; /^  / and print;' ../../src/
             s/\+ wc.*\n//;
             # + command_X | (Bash outputs these in random order)
             s/.*command_[ABC].*\n//;
+            # Due to multiple jobs "Second started" often ends up wrong
+            s/Second started\n//;
+            # Due to multiple jobs "tried 2" often ends up wrong
+            s/tried 2\n//;
 '
 # 3+3 .par files (from --files), 1 .tms-file from tmux attach
 ls /tmp/par*.par /var/tmp/par*.par /tmp/*.tms /tmp/*.tmx 2>/dev/null | wc -l

testsuite/wanted-results/parallel-local-0.3s

@@ -104,30 +104,177 @@ echo '### bug #44546: If --compress-program fails: fail'
 ### bug #44546: If --compress-program fails: fail
   parallel --line-buffer --compress-program false echo \;ls ::: /no-existing; echo $?
 1
-parallel: Error: false -dc failed.
+parallel: Error: false failed.
-parallel: Error: false -dc failed.
+parallel: Error: false failed.
   parallel --tag --line-buffer --compress-program false echo \;ls ::: /no-existing; echo $?
 1
-parallel: Error: false -dc failed.
+parallel: Error: false failed.
-parallel: Error: false -dc failed.
+parallel: Error: false failed.
   (parallel --files --tag --line-buffer --compress-program false echo \;sleep 1\;ls ::: /no-existing; echo $?) | tail -n1
 1
-parallel: Error: false -dc failed.
+parallel: Error: false failed.
-parallel: Error: false -dc failed.
+parallel: Error: false failed.
-parallel: Error: false -dc failed.
+parallel: Error: false failed.
-parallel: Error: false -dc failed.
+parallel: Error: false failed.
   parallel --tag --compress-program false echo \;ls ::: /no-existing; echo $?
 1
-parallel: Error: false -dc failed.
+parallel: Error: false failed.
-parallel: Error: false -dc failed.
+parallel: Error: false failed.
   parallel --line-buffer --compress-program false echo \;ls ::: /no-existing; echo $?
 1
-parallel: Error: false -dc failed.
+parallel: Error: false failed.
-parallel: Error: false -dc failed.
+parallel: Error: false failed.
   parallel --compress-program false echo \;ls ::: /no-existing; echo $?
 1
-parallel: Error: false -dc failed.
+parallel: Error: false failed.
-parallel: Error: false -dc failed.
+parallel: Error: false failed.
+echo 'bug #41613: --compress --line-buffer - no newline';   echo 'pipe compress tagstring'
+bug #41613: --compress --line-buffer - no newline
+pipe compress tagstring
+  perl -e 'print "O"'| parallel --compress --tagstring {#} --pipe --line-buffer cat;  echo "K"
+1	OK
+  echo 'pipe compress notagstring'
+pipe compress notagstring
+  perl -e 'print "O"'| parallel --compress --pipe --line-buffer cat;  echo "K"
+OK
+  echo 'pipe nocompress tagstring'
+pipe nocompress tagstring
+  perl -e 'print "O"'| parallel --tagstring {#} --pipe --line-buffer cat;  echo "K"
+1	OK
+  echo 'pipe nocompress notagstring'
+pipe nocompress notagstring
+  perl -e 'print "O"'| parallel --pipe --line-buffer cat;  echo "K"
+OK
+  echo 'nopipe compress tagstring'
+nopipe compress tagstring
+  parallel --compress --tagstring {#} --line-buffer echo {} O ::: -n;  echo "K"
+1	OK
+  echo 'nopipe compress notagstring'
+nopipe compress notagstring
+  parallel --compress --line-buffer echo {} O ::: -n;  echo "K"
+OK
+  echo 'nopipe nocompress tagstring'
+nopipe nocompress tagstring
+  parallel --tagstring {#} --line-buffer echo {} O ::: -n;  echo "K"
+1	OK
+  echo 'nopipe nocompress notagstring'
+nopipe nocompress notagstring
+  parallel --line-buffer echo {} O ::: -n;  echo "K"
+OK
+echo 'Compress with failing (de)compressor'
+Compress with failing (de)compressor
+  parallel -k --tag --compress --compress-program 'cat;true'  --decompress-program 'cat;true'  echo ::: tag true true
+tag	tag
+true	true
+true	true
+  parallel -k --tag --compress --compress-program 'cat;false' --decompress-program 'cat;true'  echo ::: tag false true
+tag	tag
+false	false
+true	true
+parallel: Error: cat;false failed.
+parallel: Error: cat;false failed.
+parallel: Error: cat;false failed.
+  parallel -k --tag --compress --compress-program 'cat;false' --decompress-program 'cat;false' echo ::: tag false false
+tag	tag
+false	false
+false	false
+parallel: Error: cat;false failed.
+parallel: Error: cat;false failed.
+parallel: Error: cat;false failed.
+  parallel -k --tag --compress --compress-program 'cat;true'  --decompress-program 'cat;false' echo ::: tag true false
+tag	tag
+true	true
+false	false
+  parallel -k --compress --compress-program 'cat;true'  --decompress-program 'cat;true'  echo ::: true true
+true
+true
+  parallel -k --compress --compress-program 'cat;false' --decompress-program 'cat;true'  echo ::: false true
+false
+true
+parallel: Error: cat;false failed.
+parallel: Error: cat;false failed.
+  parallel -k --compress --compress-program 'cat;false' --decompress-program 'cat;false' echo ::: false false
+false
+false
+parallel: Error: cat;false failed.
+parallel: Error: cat;false failed.
+  parallel -k --compress --compress-program 'cat;true'  --decompress-program 'cat;false' echo ::: true false
+true
+false
+  parallel -k --line-buffer --compress --compress-program 'cat;true'  --decompress-program 'cat;true'  echo ::: line-buffer true true
+line-buffer
+true
+true
+  parallel -k --line-buffer --compress --compress-program 'cat;false' --decompress-program 'cat;true'  echo ::: line-buffer false true
+line-buffer
+false
+true
+parallel: Error: cat;false failed.
+parallel: Error: cat;false failed.
+parallel: Error: cat;false failed.
+  parallel -k --line-buffer --compress --compress-program 'cat;false' --decompress-program 'cat;false' echo ::: line-buffer false false
+line-buffer
+false
+false
+parallel: Error: cat;false failed.
+parallel: Error: cat;false failed.
+parallel: Error: cat;false failed.
+  parallel -k --tag --line-buffer --compress --compress-program 'cat;true'  --decompress-program 'cat;false' echo ::: tag line-buffer true false
+tag	tag
+line-buffer	line-buffer
+true	true
+false	false
+  parallel -k --tag --line-buffer --compress --compress-program 'cat;true'  --decompress-program 'cat;true'  echo ::: tag line-buffer true true
+tag	tag
+line-buffer	line-buffer
+true	true
+true	true
+  parallel -k --tag --line-buffer --compress --compress-program 'cat;false' --decompress-program 'cat;true'  echo ::: tag line-buffer false true
+tag	tag
+line-buffer	line-buffer
+false	false
+true	true
+parallel: Error: cat;false failed.
+parallel: Error: cat;false failed.
+parallel: Error: cat;false failed.
+parallel: Error: cat;false failed.
+  parallel -k --tag --line-buffer --compress --compress-program 'cat;false' --decompress-program 'cat;false' echo ::: tag line-buffer false false
+tag	tag
+line-buffer	line-buffer
+false	false
+false	false
+parallel: Error: cat;false failed.
+parallel: Error: cat;false failed.
+parallel: Error: cat;false failed.
+parallel: Error: cat;false failed.
+  parallel -k --tag --line-buffer --compress --compress-program 'cat;true'  --decompress-program 'cat;false' echo ::: tag line-buffer true false
+tag	tag
+line-buffer	line-buffer
+true	true
+false	false
+  parallel -k --files --compress --compress-program 'cat;true'  --decompress-program 'cat;true'  echo ::: files true true   | parallel rm
+  parallel -k --files --compress --compress-program 'cat;false' --decompress-program 'cat;true'  echo ::: files false true  | parallel rm
+parallel: Error: cat;false failed.
+parallel: Error: cat;false failed.
+parallel: Error: cat;false failed.
+  parallel -k --files --compress --compress-program 'cat;false' --decompress-program 'cat;false' echo ::: files false false | parallel rm
+parallel: Error: cat;false failed.
+parallel: Error: cat;false failed.
+parallel: Error: cat;false failed.
+  parallel -k --files --compress --compress-program 'cat;true'  --decompress-program 'cat;false' echo ::: files true false  | parallel rm
+echo 'bug #44250: pxz complains File format not recognized but decompresses anyway'
+bug #44250: pxz complains File format not recognized but decompresses anyway
+  # The first line dumps core if run from make file. Why?!
+  stdout parallel --compress --compress-program pxz ls /{} ::: OK-if-missing-file
+Segmentation fault (core dumped)
+parallel: Error: pxz failed.
+  stdout parallel --compress --compress-program pixz --decompress-program 'pixz -d' ls /{}  ::: OK-if-missing-file
+ls: cannot access /OK-if-missing-file: No such file or directory
+  stdout parallel --compress --compress-program pixz --decompress-program 'pixz -d' true ::: OK-if-no-output
+  stdout parallel --compress --compress-program pxz true ::: OK-if-no-output
+echo 'bug #41613: --compress --line-buffer no newline';   perl -e 'print "It worked"'| parallel --pipe --compress --line-buffer cat; echo
+bug #41613: --compress --line-buffer no newline
+It worked
 echo '### bug #44614: --pipepart --header off by one'
 ### bug #44614: --pipepart --header off by one
   seq 10 >/tmp/parallel_44616;     parallel --pipepart -a /tmp/parallel_44616 -k --block 5 'echo foo; cat';     parallel --pipepart -a /tmp/parallel_44616 -k --block 2 --regexp --recend 3'\n' 'echo foo; cat';     rm /tmp/parallel_44616

testsuite/wanted-results/parallel-local9

@@ -1,17 +1,3 @@
-echo 'bug #44250: pxz complains File format not recognized but decompresses anyway'
-bug #44250: pxz complains File format not recognized but decompresses anyway
-  # The first line dumps core if run from make file. Why?!
-  stdout parallel --compress --compress-program pxz ls /{} ::: OK-if-missing-file
-Segmentation fault (core dumped)
-parallel: Error: pxz -dc failed.
-  stdout parallel --compress --compress-program pixz --decompress-program 'pixz -d' ls /{}  ::: OK-if-missing-file
-ls: cannot access /OK-if-missing-file: No such file or directory
-  stdout parallel --compress --compress-program pixz --decompress-program 'pixz -d' true ::: OK-if-no-output
-  stdout parallel --compress --compress-program pxz true ::: OK-if-no-output
-echo 'bug #41613: --compress --line-buffer no newline';
-bug #41613: --compress --line-buffer no newline
-  perl -e 'print "It worked"'| $NICEPAR --pipe --compress --line-buffer cat; echo
-It worked
 echo 'bug #41613: --compress --line-buffer no --tagstring';
 bug #41613: --compress --line-buffer no --tagstring
   diff     <(nice perl -e 'for("x011".."x110"){print "$_\t", ("\n", map { rand } (1..100000)) }'|       $NICEPAR -N10 -L1 --pipe -j6 --block 20M --compress       pv -qL 1000000 | perl -pe 's/(....).*/$1/')     <(nice perl -e 'for("x011".."x110"){print "$_\t", ("\n", map { rand } (1..100000)) }'|       $NICEPAR -N10 -L1 --pipe -j6 --block 20M --compress --line-buffer       pv -qL 1000000 | perl -pe 's/(....).*/$1/')     >/dev/null   || (echo 'Good: --line-buffer matters'; false) && echo 'Bad: --line-buffer not working'
@@ -20,40 +6,6 @@ echo 'bug #41613: --compress --line-buffer with --tagstring';
 bug #41613: --compress --line-buffer with --tagstring
   diff     <(nice perl -e 'for("x011".."x110"){print "$_\t", ("\n", map { rand } (1..100000)) }'|       $NICEPAR -N10 -L1 --pipe -j6 --block 20M --compress --tagstring {#}       pv -qL 1000000 | perl -pe 's/(....).*/$1/')     <(nice perl -e 'for("x011".."x110"){print "$_\t", ("\n", map { rand } (1..100000)) }'|       $NICEPAR -N10 -L1 --pipe -j6 --block 20M --compress --tagstring {#} --line-buffer       pv -qL 1000000 | perl -pe 's/(....).*/$1/')     >/dev/null   || (echo 'Good: --line-buffer matters'; false) && echo 'Bad: --line-buffer not working'
 Good: --line-buffer matters
-echo 'bug #41613: --compress --line-buffer - no newline';
-bug #41613: --compress --line-buffer - no newline
-  echo 'pipe compress tagstring'
-pipe compress tagstring
-  perl -e 'print "O"'| $NICEPAR --compress --tagstring {#} --pipe --line-buffer cat;  echo "K"
-1	OK
-  echo 'pipe compress notagstring'
-pipe compress notagstring
-  perl -e 'print "O"'| $NICEPAR --compress --pipe --line-buffer cat;  echo "K"
-OK
-  echo 'pipe nocompress tagstring'
-pipe nocompress tagstring
-  perl -e 'print "O"'| $NICEPAR --tagstring {#} --pipe --line-buffer cat;  echo "K"
-1	OK
-  echo 'pipe nocompress notagstring'
-pipe nocompress notagstring
-  perl -e 'print "O"'| $NICEPAR --pipe --line-buffer cat;  echo "K"
-OK
-  echo 'nopipe compress tagstring'
-nopipe compress tagstring
-  $NICEPAR --compress --tagstring {#} --line-buffer echo {} O ::: -n;  echo "K"
-1	OK
-  echo 'nopipe compress notagstring'
-nopipe compress notagstring
-  $NICEPAR --compress --line-buffer echo {} O ::: -n;  echo "K"
-OK
-  echo 'nopipe nocompress tagstring'
-nopipe nocompress tagstring
-  $NICEPAR --tagstring {#} --line-buffer echo {} O ::: -n;  echo "K"
-1	OK
-  echo 'nopipe nocompress notagstring'
-nopipe nocompress notagstring
-  $NICEPAR --line-buffer echo {} O ::: -n;  echo "K"
-OK
 echo 'bug #41412: --timeout + --delay causes deadlock';
 bug #41412: --timeout + --delay causes deadlock
   seq 10 | parallel -j10 --timeout 1 --delay .3 echo;

testsuite/wanted-results/parallel-tutorial

@@ -440,30 +440,30 @@ Computer:jobs running/jobs completed/%of started jobs/Average seconds to complet
 
   seq 1000 | parallel -j10 --bar '(echo -n {};sleep 0.1)' 2> >(zenity --timeout=12 --progress --auto-kill)
 BASE64  parallel --joblog /tmp/log exit  ::: 1 2 3 0 
-  cat /tmp/log
+  cat /tmp/log;
-Seq	Host	Starttime	JobRuntime	Send	Receive	Exitval	Signal	Command
-1	:	TIMESTAMP	9.999	0	0	1	0	exit 1
-2	:	TIMESTAMP	9.999	0	0	2	0	exit 2
-3	:	TIMESTAMP	9.999	0	0	3	0	exit 3
-4	:	TIMESTAMP	9.999	0	0	0	0	exit 0
   parallel --joblog /tmp/log exit  ::: 1 2 3 0 
-  cat /tmp/log
+  cat /tmp/log;  parallel --resume --joblog /tmp/log exit  ::: 1 2 3 0 0 0
-  parallel --resume --joblog /tmp/log exit  ::: 1 2 3 0 0 0
+  cat /tmp/log;
-  cat /tmp/log
-Seq	Host	Starttime	JobRuntime	Send	Receive	Exitval	Signal	Command
-1	:	TIMESTAMP	9.999	0	0	1	0	exit 1
-2	:	TIMESTAMP	9.999	0	0	2	0	exit 2
-3	:	TIMESTAMP	9.999	0	0	3	0	exit 3
-4	:	TIMESTAMP	9.999	0	0	0	0	exit 0
-Seq	Host	Starttime	JobRuntime	Send	Receive	Exitval	Signal	Command
-1	:	TIMESTAMP	9.999	0	0	1	0	exit 1
-2	:	TIMESTAMP	9.999	0	0	2	0	exit 2
-3	:	TIMESTAMP	9.999	0	0	3	0	exit 3
-4	:	TIMESTAMP	9.999	0	0	0	0	exit 0
-5	:	TIMESTAMP	9.999	0	0	0	0	exit 0
-6	:	TIMESTAMP	9.999	0	0	0	0	exit 0
   parallel --resume-failed --joblog /tmp/log exit  ::: 1 2 3 0 0 0
-  cat /tmp/log
+  cat /tmp/log;
+  parallel -j2 --halt 1 echo {}\; exit {} ::: 0 0 1 2 3
+Seq	Host	Starttime	JobRuntime	Send	Receive	Exitval	Signal	Command
+1	:	TIMESTAMP	9.999	0	0	1	0	exit 1
+2	:	TIMESTAMP	9.999	0	0	2	0	exit 2
+3	:	TIMESTAMP	9.999	0	0	3	0	exit 3
+4	:	TIMESTAMP	9.999	0	0	0	0	exit 0
+Seq	Host	Starttime	JobRuntime	Send	Receive	Exitval	Signal	Command
+1	:	TIMESTAMP	9.999	0	0	1	0	exit 1
+2	:	TIMESTAMP	9.999	0	0	2	0	exit 2
+3	:	TIMESTAMP	9.999	0	0	3	0	exit 3
+4	:	TIMESTAMP	9.999	0	0	0	0	exit 0
+Seq	Host	Starttime	JobRuntime	Send	Receive	Exitval	Signal	Command
+1	:	TIMESTAMP	9.999	0	0	1	0	exit 1
+2	:	TIMESTAMP	9.999	0	0	2	0	exit 2
+3	:	TIMESTAMP	9.999	0	0	3	0	exit 3
+4	:	TIMESTAMP	9.999	0	0	0	0	exit 0
+5	:	TIMESTAMP	9.999	0	0	0	0	exit 0
+6	:	TIMESTAMP	9.999	0	0	0	0	exit 0
 Seq	Host	Starttime	JobRuntime	Send	Receive	Exitval	Signal	Command
 1	:	TIMESTAMP	9.999	0	0	1	0	exit 1
 2	:	TIMESTAMP	9.999	0	0	2	0	exit 2
@@ -474,7 +474,6 @@ Seq	Host	Starttime	JobRuntime	Send	Receive	Exitval	Signal	Command
 1	:	TIMESTAMP	9.999	0	0	1	0	exit 1
 2	:	TIMESTAMP	9.999	0	0	2	0	exit 2
 3	:	TIMESTAMP	9.999	0	0	3	0	exit 3
-  parallel -j2 --halt 1 echo {}\; exit {} ::: 0 0 1 2 3
 0
 0
 1
@@ -507,12 +506,9 @@ completed 1
 completed 2
 completed 0
 tried 1
-tried 2
 tried 0
 tried 1
-tried 2
 tried 1
-tried 2
   parallel --load 100% echo load is less than {} job per cpu ::: 1 
 load is less than 1 job per cpu
   parallel --noswap echo the system is not swapping ::: now
@@ -664,6 +660,7 @@ TEXTDOMAIN
 TEXTDOMAINDIR
 TIMEOUT
 TMPDIR
+TRIES
 UPSTART_EVENTS
 UPSTART_INSTANCE
 UPSTART_JOB
@@ -883,14 +880,12 @@ The second finished running in the foreground
 First started
 The first finished
   sem --id my_id -u 'echo Second started; sleep 10; echo The second finished'
-Second started
   sem --jobs 3 --id my_id -u 'echo First started; sleep 5; echo The first finished' &&
   sem --jobs 3 --id my_id -u 'echo Second started; sleep 6; echo The second finished' &&
   sem --jobs 3 --id my_id -u 'echo Third started; sleep 7; echo The third finished' &&
   sem --jobs 3 --id my_id -u 'echo Fourth started; sleep 8; echo The fourth finished' &&
   sem --wait --id my_id
 First started
-Second started
 The first finished
 Third started
 The second finished
@@ -936,7 +931,7 @@ This helps funding further development; and it won't cost you a cent.
 If you pay 10000 EUR you should feel free to use GNU Parallel without citing.
 
   parallel --version
-GNU parallel 20150403
+GNU parallel 20150415
 Copyright (C) 2007,2008,2009,2010,2011,2012,2013,2014,2015 Ole Tange
 and Free Software Foundation, Inc.
 License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
@@ -948,7 +943,7 @@ Web site: http://www.gnu.org/software/parallel
 When using programs that use GNU Parallel to process data for publication
 please cite as described in 'parallel --bibtex'.
   parallel --minversion 20130722 && echo Your version is at least 20130722.
-20150403
+20150415
 Your version is at least 20130722.
   parallel --bibtex
 Academic tradition requires you to cite works you base your article on.