parallel: Minimize time when --compress leaves a file. Passes testsuite.

doc/release_new_version

@@ -231,7 +231,7 @@ New in this release:
 
 * GNU Parallel was cited in: Experimental study on the Wind Farm Substation Cable Installation Problem http://i11www.iti.uni-karlsruhe.de/_media/teaching/theses/ma-schmitz-14.pdf
 
-* <<afventer>> Comparing the CarbonTracker and TM5-4DVar data assimilation systems for CO2 surface flux inversions http://www.atmos-chem-phys-discuss.net/15/8883/2015/acpd-15-8883-2015-discussion.html
+* <<afventer - rykket >> Comparing the CarbonTracker and TM5-4DVar data assimilation systems for CO2 surface flux inversions http://www.atmos-chem-phys-discuss.net/15/8883/2015/acpd-15-8883-2015-discussion.html
 
 * <<afventer opdatering>> CIDER: a pipeline for detecting waves of coordinated transcriptional regulation in gene expression time-course data http://biorxiv.org/content/biorxiv/early/2015/03/17/012518.full.pdf
 
@@ -247,10 +247,16 @@ taxator-tk http://algbio.cs.uni-duesseldorf.de/webapps/wa-download/ (check it)
 
 * GNU Parallel was used in: http://search.cpan.org/~ajpage/Bio-Roary-2.0.7/lib/Bio/Roary/JobRunner/Parallel.pm
 
+* GNU Parallel was used in: Rapid haploid SNP calling https://github.com/tseemann/snippy
+
+* << afventer svar fra Rachel >> GNU Parallel was used in: SISRS: Site Identification from Short Read Sequences https://github.com/rachelss/SISRS/
+
 * Pictures and Metadata http://www.ozzy.no/2015/02/05/pictures-and-metadata/
 
 * Task automation with bash and parallel https://biowize.wordpress.com/2015/03/23/task-automation-with-bash-and-parallel/
 
+* How To: Speed Up File Transfers in Linux using RSync with GNU Parallel http://www.yourownlinux.com/2015/04/speed-up-file-transfers-using-rsync-with-gnu-parallel.html
+
 * Mirroring a Git Repository https://avacariu.me/articles/mirroring-a-git-repository.html
 
 * Add an ENVI header to JAXA Global Mangrove Watch PALSAR tiles https://spectraldifferences.wordpress.com/2015/03/24/add-an-envi-header-to-jaxa-global-mangrove-watch-palsar-tiles/

src/parallel

@@ -5576,18 +5576,26 @@ sub slot {
 	#   $cattail = perl program for: cattail "decompress program" writerpid [file_to_decompress or stdin] [file_to_unlink]
 	if(not $cattail) {
 	    $cattail = q{
-		# cat followed by tail.
+		# cat followed by tail (possibly with rm as soon at the file is opened)
 		# If $writerpid dead: finish after this round
 		use Fcntl;
 
 		$|=1;
 
 		my ($cmd, $writerpid, $read_file, $unlink_file) = @ARGV;
+		while(defined $unlink_file and not -e $unlink_file) {
+		    # Writer has not opened file, so we cannot open it
+		    $sleep = ($sleep < 30) ? ($sleep * 1.001 + 0.01) : ($sleep);
+		    usleep($sleep);
+		}
+
 		if($read_file) {
 		    open(IN,"<",$read_file) || die("cattail: Cannot open $read_file");
 		} else {
 		    *IN = *STDIN;
 		}
+		# The writer and we have both opened the file, so it is safe to unlink it
+		unlink $unlink_file;
 
 		my $first_round = 1;
 		my $flags;
@@ -5601,9 +5609,6 @@ sub slot {
 		    my $writer_running = kill 0, $writerpid;
 		    $read = sysread(IN,$buf,32768);
 		    if($read) {
-			# We can unlink the file now: The writer has written something
-			-e $unlink_file and unlink $unlink_file;
-
 			if($first_round) {
 			    # Only start the command if there any input to process
 			    $first_round = 0;
@@ -5621,7 +5626,6 @@ sub slot {
 		    } else {
 			if(eof(IN) and not $writer_running) {
 			    # Writer dead: There will never be more to read => exit
-			    -e $unlink_file and unlink $unlink_file;
 			    exit;
 			}
 			# TODO This could probably be done more efficiently using select(2)
@@ -5768,8 +5772,11 @@ sub filter_through_compress {
     my $cattail = cattail();
 
     for my $fdno (1,2) {
-	my $wpid = open(my $fdw,"|-", empty_input_detector().
+	# The tmpfile is used to tell the reader that the writer has started,
-			"| ($opt::compress_program) >>".
+	# so unlink it to start with.
+	unlink $self->fh($fdno,'name');
+        my $wpid = open(my $fdw,"|-", "(".empty_input_detector().
+			"| ($opt::compress_program)) >>".
 			$self->fh($fdno,'name')) || die $?;
 	$self->set_fh($fdno,'w',$fdw);
 	$self->set_fh($fdno,'wpid',$wpid);
@@ -7091,14 +7098,18 @@ sub files_print {
     my $self = shift;
     my ($fdno,$in_fh,$out_fd) = @_;
 
-    # If --compress: $in_fh must be closed first.
+    # If the job is dead: close printing fh. Needed for --compress
     close $self->fh($fdno,"w");
     if($? and $opt::compress) {
-	::error($opt::decompress_program." failed.\n");
+	::error($opt::compress_program." failed.\n");
 	$self->set_exitstatus(255);
     }
+    if($opt::compress) {
+	# Kill the decompressor which will not be needed
+	CORE::kill "TERM", $self->fh($fdno,"rpid");
+    }
     close $in_fh;
-    # TODO if $?: die(?)
+
     if($opt::pipe and $self->virgin()) {
 	# Nothing was printed to this job:
 	# cleanup unused tmp files if --files was set
@@ -7121,7 +7132,7 @@ sub linebuffer_print {
 	# If the job is dead: close printing fh. Needed for --compress
 	close $self->fh($fdno,"w");
 	if($? and $opt::compress) {
-	    ::error($opt::decompress_program." failed.\n");
+	    ::error($opt::compress_program." failed.\n");
 	    $self->set_exitstatus(255);
 	}
 	if($opt::compress) {
@@ -7190,7 +7201,10 @@ sub linebuffer_print {
 	} else {
 	    # decompress done: close fh
 	    close $in_fh;
-	    # TODO if $?: die(?)
+	    if($? and $opt::compress) {
+		::error($opt::decompress_program." failed.\n");
+		$self->set_exitstatus(255);
+	    }
 	}
     }
 }
@@ -7201,7 +7215,7 @@ sub tag_print {
     my $buf;
     close $self->fh($fdno,"w");
     if($? and $opt::compress) {
-	::error($opt::decompress_program." failed.\n");
+	::error($opt::compress_program." failed.\n");
 	$self->set_exitstatus(255);
     }
     seek $in_fh, 0, 0;
@@ -7230,6 +7244,10 @@ sub tag_print {
 	$self->add_returnsize($outputlength);
     }
     close $in_fh;
+    if($? and $opt::compress) {
+	::error($opt::decompress_program." failed.\n");
+	$self->set_exitstatus(255);
+    }
 }
 
 sub normal_print {
@@ -7238,7 +7256,7 @@ sub normal_print {
     my $buf;
     close $self->fh($fdno,"w");
     if($? and $opt::compress) {
-	::error($opt::decompress_program." failed.\n");
+	::error($opt::compress_program." failed.\n");
 	$self->set_exitstatus(255);
     }
     seek $in_fh, 0, 0;
@@ -7260,6 +7278,10 @@ sub normal_print {
 	$self->add_returnsize($outputlength);
     }
     close $in_fh;
+    if($? and $opt::compress) {
+	::error($opt::decompress_program." failed.\n");
+	$self->set_exitstatus(255);
+    }
 }
 
 sub print_joblog {

testsuite/Makefile

@@ -7,7 +7,7 @@ testsuite: 3
 	make stopvm
 
 1: ../src/parallel tests-to-run/* wanted-results/* prereqlocal startdb prereqremote
-	time bash Start.sh - mem || true
+	TRIES=1 time bash Start.sh - mem || true
 	touch ~/.parallel/will-cite
 	make stopvm
 

testsuite/tests-to-run/parallel-local-0.3s.sh

@@ -84,6 +84,56 @@ echo '### bug #44546: If --compress-program fails: fail'
   parallel --line-buffer --compress-program false echo \;ls ::: /no-existing; echo $?
   parallel --compress-program false echo \;ls ::: /no-existing; echo $?
 
+echo 'bug #41613: --compress --line-buffer - no newline';
+  echo 'pipe compress tagstring'
+  perl -e 'print "O"'| parallel --compress --tagstring {#} --pipe --line-buffer cat;  echo "K"
+  echo 'pipe compress notagstring'
+  perl -e 'print "O"'| parallel --compress --pipe --line-buffer cat;  echo "K"
+  echo 'pipe nocompress tagstring'
+  perl -e 'print "O"'| parallel --tagstring {#} --pipe --line-buffer cat;  echo "K"
+  echo 'pipe nocompress notagstring'
+  perl -e 'print "O"'| parallel --pipe --line-buffer cat;  echo "K"
+  echo 'nopipe compress tagstring'
+  parallel --compress --tagstring {#} --line-buffer echo {} O ::: -n;  echo "K"
+  echo 'nopipe compress notagstring'
+  parallel --compress --line-buffer echo {} O ::: -n;  echo "K"
+  echo 'nopipe nocompress tagstring'
+  parallel --tagstring {#} --line-buffer echo {} O ::: -n;  echo "K"
+  echo 'nopipe nocompress notagstring'
+  parallel --line-buffer echo {} O ::: -n;  echo "K"
+
+echo 'Compress with failing (de)compressor'
+  parallel -k --tag --compress --compress-program 'cat;true'  --decompress-program 'cat;true'  echo ::: tag true true
+  parallel -k --tag --compress --compress-program 'cat;false' --decompress-program 'cat;true'  echo ::: tag false true
+  parallel -k --tag --compress --compress-program 'cat;false' --decompress-program 'cat;false' echo ::: tag false false
+  parallel -k --tag --compress --compress-program 'cat;true'  --decompress-program 'cat;false' echo ::: tag true false
+  parallel -k --compress --compress-program 'cat;true'  --decompress-program 'cat;true'  echo ::: true true
+  parallel -k --compress --compress-program 'cat;false' --decompress-program 'cat;true'  echo ::: false true
+  parallel -k --compress --compress-program 'cat;false' --decompress-program 'cat;false' echo ::: false false
+  parallel -k --compress --compress-program 'cat;true'  --decompress-program 'cat;false' echo ::: true false
+  parallel -k --line-buffer --compress --compress-program 'cat;true'  --decompress-program 'cat;true'  echo ::: line-buffer true true
+  parallel -k --line-buffer --compress --compress-program 'cat;false' --decompress-program 'cat;true'  echo ::: line-buffer false true
+  parallel -k --line-buffer --compress --compress-program 'cat;false' --decompress-program 'cat;false' echo ::: line-buffer false false
+  parallel -k --tag --line-buffer --compress --compress-program 'cat;true'  --decompress-program 'cat;false' echo ::: tag line-buffer true false
+  parallel -k --tag --line-buffer --compress --compress-program 'cat;true'  --decompress-program 'cat;true'  echo ::: tag line-buffer true true
+  parallel -k --tag --line-buffer --compress --compress-program 'cat;false' --decompress-program 'cat;true'  echo ::: tag line-buffer false true
+  parallel -k --tag --line-buffer --compress --compress-program 'cat;false' --decompress-program 'cat;false' echo ::: tag line-buffer false false
+  parallel -k --tag --line-buffer --compress --compress-program 'cat;true'  --decompress-program 'cat;false' echo ::: tag line-buffer true false
+  parallel -k --files --compress --compress-program 'cat;true'  --decompress-program 'cat;true'  echo ::: files true true   | parallel rm
+  parallel -k --files --compress --compress-program 'cat;false' --decompress-program 'cat;true'  echo ::: files false true  | parallel rm
+  parallel -k --files --compress --compress-program 'cat;false' --decompress-program 'cat;false' echo ::: files false false | parallel rm
+  parallel -k --files --compress --compress-program 'cat;true'  --decompress-program 'cat;false' echo ::: files true false  | parallel rm
+
+echo 'bug #44250: pxz complains File format not recognized but decompresses anyway'
+  # The first line dumps core if run from make file. Why?!
+  stdout parallel --compress --compress-program pxz ls /{} ::: OK-if-missing-file
+  stdout parallel --compress --compress-program pixz --decompress-program 'pixz -d' ls /{}  ::: OK-if-missing-file
+  stdout parallel --compress --compress-program pixz --decompress-program 'pixz -d' true ::: OK-if-no-output
+  stdout parallel --compress --compress-program pxz true ::: OK-if-no-output
+
+echo 'bug #41613: --compress --line-buffer no newline';
+  perl -e 'print "It worked"'| parallel --pipe --compress --line-buffer cat; echo
+
 echo '### bug #44614: --pipepart --header off by one'
   seq 10 >/tmp/parallel_44616; 
     parallel --pipepart -a /tmp/parallel_44616 -k --block 5 'echo foo; cat'; 

testsuite/tests-to-run/parallel-local9.sh

@@ -8,16 +8,6 @@ NICEPAR="nice nice parallel"
 export NICEPAR
 
 cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | stdout parallel -vj4 -k --joblog /tmp/jl-`basename $0` -L1
-echo 'bug #44250: pxz complains File format not recognized but decompresses anyway'
-  # The first line dumps core if run from make file. Why?!
-  stdout parallel --compress --compress-program pxz ls /{} ::: OK-if-missing-file
-  stdout parallel --compress --compress-program pixz --decompress-program 'pixz -d' ls /{}  ::: OK-if-missing-file
-  stdout parallel --compress --compress-program pixz --decompress-program 'pixz -d' true ::: OK-if-no-output
-  stdout parallel --compress --compress-program pxz true ::: OK-if-no-output
-
-echo 'bug #41613: --compress --line-buffer no newline';
-  perl -e 'print "It worked"'| $NICEPAR --pipe --compress --line-buffer cat; echo
-
 echo 'bug #41613: --compress --line-buffer no --tagstring';
   diff 
     <(nice perl -e 'for("x011".."x110"){print "$_\t", ("\n", map { rand } (1..100000)) }'| 
@@ -40,24 +30,6 @@ echo 'bug #41613: --compress --line-buffer with --tagstring';
     >/dev/null 
   || (echo 'Good: --line-buffer matters'; false) && echo 'Bad: --line-buffer not working'
 
-echo 'bug #41613: --compress --line-buffer - no newline';
-  echo 'pipe compress tagstring'
-  perl -e 'print "O"'| $NICEPAR --compress --tagstring {#} --pipe --line-buffer cat;  echo "K"
-  echo 'pipe compress notagstring'
-  perl -e 'print "O"'| $NICEPAR --compress --pipe --line-buffer cat;  echo "K"
-  echo 'pipe nocompress tagstring'
-  perl -e 'print "O"'| $NICEPAR --tagstring {#} --pipe --line-buffer cat;  echo "K"
-  echo 'pipe nocompress notagstring'
-  perl -e 'print "O"'| $NICEPAR --pipe --line-buffer cat;  echo "K"
-  echo 'nopipe compress tagstring'
-  $NICEPAR --compress --tagstring {#} --line-buffer echo {} O ::: -n;  echo "K"
-  echo 'nopipe compress notagstring'
-  $NICEPAR --compress --line-buffer echo {} O ::: -n;  echo "K"
-  echo 'nopipe nocompress tagstring'
-  $NICEPAR --tagstring {#} --line-buffer echo {} O ::: -n;  echo "K"
-  echo 'nopipe nocompress notagstring'
-  $NICEPAR --line-buffer echo {} O ::: -n;  echo "K"
-
 echo 'bug #41412: --timeout + --delay causes deadlock';
   seq 10 | parallel -j10 --timeout 1 --delay .3 echo;
   parallel -j3 --timeout 1 --delay 2 echo ::: 1 2 3;

testsuite/tests-to-run/parallel-tutorial.sh

@@ -46,6 +46,10 @@ perl -ne '$/="\n\n"; /^Output/../^[^O]\S/ and next; /^  / and print;' ../../src/
             s/\+ wc.*\n//;
             # + command_X | (Bash outputs these in random order)
             s/.*command_[ABC].*\n//;
+            # Due to multiple jobs "Second started" often ends up wrong
+            s/Second started\n//;
+            # Due to multiple jobs "tried 2" often ends up wrong
+            s/tried 2\n//;
 '
 # 3+3 .par files (from --files), 1 .tms-file from tmux attach
 ls /tmp/par*.par /var/tmp/par*.par /tmp/*.tms /tmp/*.tmx 2>/dev/null | wc -l

testsuite/wanted-results/parallel-local-0.3s

@@ -104,30 +104,177 @@ echo '### bug #44546: If --compress-program fails: fail'
 ### bug #44546: If --compress-program fails: fail
   parallel --line-buffer --compress-program false echo \;ls ::: /no-existing; echo $?
 1
-parallel: Error: false -dc failed.
+parallel: Error: false failed.
-parallel: Error: false -dc failed.
+parallel: Error: false failed.
   parallel --tag --line-buffer --compress-program false echo \;ls ::: /no-existing; echo $?
 1
-parallel: Error: false -dc failed.
+parallel: Error: false failed.
-parallel: Error: false -dc failed.
+parallel: Error: false failed.
   (parallel --files --tag --line-buffer --compress-program false echo \;sleep 1\;ls ::: /no-existing; echo $?) | tail -n1
 1
-parallel: Error: false -dc failed.
+parallel: Error: false failed.
-parallel: Error: false -dc failed.
+parallel: Error: false failed.
-parallel: Error: false -dc failed.
+parallel: Error: false failed.
-parallel: Error: false -dc failed.
+parallel: Error: false failed.
   parallel --tag --compress-program false echo \;ls ::: /no-existing; echo $?
 1
-parallel: Error: false -dc failed.
+parallel: Error: false failed.
-parallel: Error: false -dc failed.
+parallel: Error: false failed.
   parallel --line-buffer --compress-program false echo \;ls ::: /no-existing; echo $?
 1
-parallel: Error: false -dc failed.
+parallel: Error: false failed.
-parallel: Error: false -dc failed.
+parallel: Error: false failed.
   parallel --compress-program false echo \;ls ::: /no-existing; echo $?
 1
-parallel: Error: false -dc failed.
+parallel: Error: false failed.
-parallel: Error: false -dc failed.
+parallel: Error: false failed.
+echo 'bug #41613: --compress --line-buffer - no newline';   echo 'pipe compress tagstring'
+bug #41613: --compress --line-buffer - no newline
+pipe compress tagstring
+  perl -e 'print "O"'| parallel --compress --tagstring {#} --pipe --line-buffer cat;  echo "K"
+1	OK
+  echo 'pipe compress notagstring'
+pipe compress notagstring
+  perl -e 'print "O"'| parallel --compress --pipe --line-buffer cat;  echo "K"
+OK
+  echo 'pipe nocompress tagstring'
+pipe nocompress tagstring
+  perl -e 'print "O"'| parallel --tagstring {#} --pipe --line-buffer cat;  echo "K"
+1	OK
+  echo 'pipe nocompress notagstring'
+pipe nocompress notagstring
+  perl -e 'print "O"'| parallel --pipe --line-buffer cat;  echo "K"
+OK
+  echo 'nopipe compress tagstring'
+nopipe compress tagstring
+  parallel --compress --tagstring {#} --line-buffer echo {} O ::: -n;  echo "K"
+1	OK
+  echo 'nopipe compress notagstring'
+nopipe compress notagstring
+  parallel --compress --line-buffer echo {} O ::: -n;  echo "K"
+OK
+  echo 'nopipe nocompress tagstring'
+nopipe nocompress tagstring
+  parallel --tagstring {#} --line-buffer echo {} O ::: -n;  echo "K"
+1	OK
+  echo 'nopipe nocompress notagstring'
+nopipe nocompress notagstring
+  parallel --line-buffer echo {} O ::: -n;  echo "K"
+OK
+echo 'Compress with failing (de)compressor'
+Compress with failing (de)compressor
+  parallel -k --tag --compress --compress-program 'cat;true'  --decompress-program 'cat;true'  echo ::: tag true true
+tag	tag
+true	true
+true	true
+  parallel -k --tag --compress --compress-program 'cat;false' --decompress-program 'cat;true'  echo ::: tag false true
+tag	tag
+false	false
+true	true
+parallel: Error: cat;false failed.
+parallel: Error: cat;false failed.
+parallel: Error: cat;false failed.
+  parallel -k --tag --compress --compress-program 'cat;false' --decompress-program 'cat;false' echo ::: tag false false
+tag	tag
+false	false
+false	false
+parallel: Error: cat;false failed.
+parallel: Error: cat;false failed.
+parallel: Error: cat;false failed.
+  parallel -k --tag --compress --compress-program 'cat;true'  --decompress-program 'cat;false' echo ::: tag true false
+tag	tag
+true	true
+false	false
+  parallel -k --compress --compress-program 'cat;true'  --decompress-program 'cat;true'  echo ::: true true
+true
+true
+  parallel -k --compress --compress-program 'cat;false' --decompress-program 'cat;true'  echo ::: false true
+false
+true
+parallel: Error: cat;false failed.
+parallel: Error: cat;false failed.
+  parallel -k --compress --compress-program 'cat;false' --decompress-program 'cat;false' echo ::: false false
+false
+false
+parallel: Error: cat;false failed.
+parallel: Error: cat;false failed.
+  parallel -k --compress --compress-program 'cat;true'  --decompress-program 'cat;false' echo ::: true false
+true
+false
+  parallel -k --line-buffer --compress --compress-program 'cat;true'  --decompress-program 'cat;true'  echo ::: line-buffer true true
+line-buffer
+true
+true
+  parallel -k --line-buffer --compress --compress-program 'cat;false' --decompress-program 'cat;true'  echo ::: line-buffer false true
+line-buffer
+false
+true
+parallel: Error: cat;false failed.
+parallel: Error: cat;false failed.
+parallel: Error: cat;false failed.
+  parallel -k --line-buffer --compress --compress-program 'cat;false' --decompress-program 'cat;false' echo ::: line-buffer false false
+line-buffer
+false
+false
+parallel: Error: cat;false failed.
+parallel: Error: cat;false failed.
+parallel: Error: cat;false failed.
+  parallel -k --tag --line-buffer --compress --compress-program 'cat;true'  --decompress-program 'cat;false' echo ::: tag line-buffer true false
+tag	tag
+line-buffer	line-buffer
+true	true
+false	false
+  parallel -k --tag --line-buffer --compress --compress-program 'cat;true'  --decompress-program 'cat;true'  echo ::: tag line-buffer true true
+tag	tag
+line-buffer	line-buffer
+true	true
+true	true
+  parallel -k --tag --line-buffer --compress --compress-program 'cat;false' --decompress-program 'cat;true'  echo ::: tag line-buffer false true
+tag	tag
+line-buffer	line-buffer
+false	false
+true	true
+parallel: Error: cat;false failed.
+parallel: Error: cat;false failed.
+parallel: Error: cat;false failed.
+parallel: Error: cat;false failed.
+  parallel -k --tag --line-buffer --compress --compress-program 'cat;false' --decompress-program 'cat;false' echo ::: tag line-buffer false false
+tag	tag
+line-buffer	line-buffer
+false	false
+false	false
+parallel: Error: cat;false failed.
+parallel: Error: cat;false failed.
+parallel: Error: cat;false failed.
+parallel: Error: cat;false failed.
+  parallel -k --tag --line-buffer --compress --compress-program 'cat;true'  --decompress-program 'cat;false' echo ::: tag line-buffer true false
+tag	tag
+line-buffer	line-buffer
+true	true
+false	false
+  parallel -k --files --compress --compress-program 'cat;true'  --decompress-program 'cat;true'  echo ::: files true true   | parallel rm
+  parallel -k --files --compress --compress-program 'cat;false' --decompress-program 'cat;true'  echo ::: files false true  | parallel rm
+parallel: Error: cat;false failed.
+parallel: Error: cat;false failed.
+parallel: Error: cat;false failed.
+  parallel -k --files --compress --compress-program 'cat;false' --decompress-program 'cat;false' echo ::: files false false | parallel rm
+parallel: Error: cat;false failed.
+parallel: Error: cat;false failed.
+parallel: Error: cat;false failed.
+  parallel -k --files --compress --compress-program 'cat;true'  --decompress-program 'cat;false' echo ::: files true false  | parallel rm
+echo 'bug #44250: pxz complains File format not recognized but decompresses anyway'
+bug #44250: pxz complains File format not recognized but decompresses anyway
+  # The first line dumps core if run from make file. Why?!
+  stdout parallel --compress --compress-program pxz ls /{} ::: OK-if-missing-file
+Segmentation fault (core dumped)
+parallel: Error: pxz failed.
+  stdout parallel --compress --compress-program pixz --decompress-program 'pixz -d' ls /{}  ::: OK-if-missing-file
+ls: cannot access /OK-if-missing-file: No such file or directory
+  stdout parallel --compress --compress-program pixz --decompress-program 'pixz -d' true ::: OK-if-no-output
+  stdout parallel --compress --compress-program pxz true ::: OK-if-no-output
+echo 'bug #41613: --compress --line-buffer no newline';   perl -e 'print "It worked"'| parallel --pipe --compress --line-buffer cat; echo
+bug #41613: --compress --line-buffer no newline
+It worked
 echo '### bug #44614: --pipepart --header off by one'
 ### bug #44614: --pipepart --header off by one
   seq 10 >/tmp/parallel_44616;     parallel --pipepart -a /tmp/parallel_44616 -k --block 5 'echo foo; cat';     parallel --pipepart -a /tmp/parallel_44616 -k --block 2 --regexp --recend 3'\n' 'echo foo; cat';     rm /tmp/parallel_44616

testsuite/wanted-results/parallel-local9

@@ -1,17 +1,3 @@
-echo 'bug #44250: pxz complains File format not recognized but decompresses anyway'
-bug #44250: pxz complains File format not recognized but decompresses anyway
-  # The first line dumps core if run from make file. Why?!
-  stdout parallel --compress --compress-program pxz ls /{} ::: OK-if-missing-file
-Segmentation fault (core dumped)
-parallel: Error: pxz -dc failed.
-  stdout parallel --compress --compress-program pixz --decompress-program 'pixz -d' ls /{}  ::: OK-if-missing-file
-ls: cannot access /OK-if-missing-file: No such file or directory
-  stdout parallel --compress --compress-program pixz --decompress-program 'pixz -d' true ::: OK-if-no-output
-  stdout parallel --compress --compress-program pxz true ::: OK-if-no-output
-echo 'bug #41613: --compress --line-buffer no newline';
-bug #41613: --compress --line-buffer no newline
-  perl -e 'print "It worked"'| $NICEPAR --pipe --compress --line-buffer cat; echo
-It worked
 echo 'bug #41613: --compress --line-buffer no --tagstring';
 bug #41613: --compress --line-buffer no --tagstring
   diff     <(nice perl -e 'for("x011".."x110"){print "$_\t", ("\n", map { rand } (1..100000)) }'|       $NICEPAR -N10 -L1 --pipe -j6 --block 20M --compress       pv -qL 1000000 | perl -pe 's/(....).*/$1/')     <(nice perl -e 'for("x011".."x110"){print "$_\t", ("\n", map { rand } (1..100000)) }'|       $NICEPAR -N10 -L1 --pipe -j6 --block 20M --compress --line-buffer       pv -qL 1000000 | perl -pe 's/(....).*/$1/')     >/dev/null   || (echo 'Good: --line-buffer matters'; false) && echo 'Bad: --line-buffer not working'
@@ -20,40 +6,6 @@ echo 'bug #41613: --compress --line-buffer with --tagstring';
 bug #41613: --compress --line-buffer with --tagstring
   diff     <(nice perl -e 'for("x011".."x110"){print "$_\t", ("\n", map { rand } (1..100000)) }'|       $NICEPAR -N10 -L1 --pipe -j6 --block 20M --compress --tagstring {#}       pv -qL 1000000 | perl -pe 's/(....).*/$1/')     <(nice perl -e 'for("x011".."x110"){print "$_\t", ("\n", map { rand } (1..100000)) }'|       $NICEPAR -N10 -L1 --pipe -j6 --block 20M --compress --tagstring {#} --line-buffer       pv -qL 1000000 | perl -pe 's/(....).*/$1/')     >/dev/null   || (echo 'Good: --line-buffer matters'; false) && echo 'Bad: --line-buffer not working'
 Good: --line-buffer matters
-echo 'bug #41613: --compress --line-buffer - no newline';
-bug #41613: --compress --line-buffer - no newline
-  echo 'pipe compress tagstring'
-pipe compress tagstring
-  perl -e 'print "O"'| $NICEPAR --compress --tagstring {#} --pipe --line-buffer cat;  echo "K"
-1	OK
-  echo 'pipe compress notagstring'
-pipe compress notagstring
-  perl -e 'print "O"'| $NICEPAR --compress --pipe --line-buffer cat;  echo "K"
-OK
-  echo 'pipe nocompress tagstring'
-pipe nocompress tagstring
-  perl -e 'print "O"'| $NICEPAR --tagstring {#} --pipe --line-buffer cat;  echo "K"
-1	OK
-  echo 'pipe nocompress notagstring'
-pipe nocompress notagstring
-  perl -e 'print "O"'| $NICEPAR --pipe --line-buffer cat;  echo "K"
-OK
-  echo 'nopipe compress tagstring'
-nopipe compress tagstring
-  $NICEPAR --compress --tagstring {#} --line-buffer echo {} O ::: -n;  echo "K"
-1	OK
-  echo 'nopipe compress notagstring'
-nopipe compress notagstring
-  $NICEPAR --compress --line-buffer echo {} O ::: -n;  echo "K"
-OK
-  echo 'nopipe nocompress tagstring'
-nopipe nocompress tagstring
-  $NICEPAR --tagstring {#} --line-buffer echo {} O ::: -n;  echo "K"
-1	OK
-  echo 'nopipe nocompress notagstring'
-nopipe nocompress notagstring
-  $NICEPAR --line-buffer echo {} O ::: -n;  echo "K"
-OK
 echo 'bug #41412: --timeout + --delay causes deadlock';
 bug #41412: --timeout + --delay causes deadlock
   seq 10 | parallel -j10 --timeout 1 --delay .3 echo;

testsuite/wanted-results/parallel-tutorial

@@ -506,12 +506,9 @@ completed 1
 completed 2
 completed 0
 tried 1
-tried 2
 tried 0
 tried 1
-tried 2
 tried 1
-tried 2
   parallel --load 100% echo load is less than {} job per cpu ::: 1 
 load is less than 1 job per cpu
   parallel --noswap echo the system is not swapping ::: now
@@ -663,6 +660,7 @@ TEXTDOMAIN
 TEXTDOMAINDIR
 TIMEOUT
 TMPDIR
+TRIES
 UPSTART_EVENTS
 UPSTART_INSTANCE
 UPSTART_JOB
@@ -882,14 +880,12 @@ The second finished running in the foreground
 First started
 The first finished
   sem --id my_id -u 'echo Second started; sleep 10; echo The second finished'
-Second started
   sem --jobs 3 --id my_id -u 'echo First started; sleep 5; echo The first finished' &&
   sem --jobs 3 --id my_id -u 'echo Second started; sleep 6; echo The second finished' &&
   sem --jobs 3 --id my_id -u 'echo Third started; sleep 7; echo The third finished' &&
   sem --jobs 3 --id my_id -u 'echo Fourth started; sleep 8; echo The fourth finished' &&
   sem --wait --id my_id
 First started
-Second started
 The first finished
 Third started
 The second finished