Fixed too many spawned jobs in --pipe --files.

src/parallel

@@ -4907,19 +4907,27 @@ sub print {
 
     my $self = shift;
     ::debug(">>joboutput ".$self->replaced()."\n");
-    if($opt::pipe and $self->virgin()) {
+#    if($opt::pipe and $self->virgin()) {
-	# Nothing was printed to this job:
+#	# Nothing was printed to this job:
-	# cleanup tmp files if --files was set
+#	# cleanup tmp files if --files was set
-	unlink $self->fh(1,"name");
+#	close $self->fh(0,"w");
-	return;
+#	for my $fdno (1,2) {
-    }
+#	    close $self->fh($fdno,"w");
+#	    close $self->fh($fdno,"r");
+#	    # Unlink files that are empty
+#	    unlink $self->fh($fdno,"name");
+#	}
+#	return;
+#    }
     if($opt::dryrun) {
 	# Nothing was printed to this job:
 	# cleanup tmp files if --files was set
 	unlink $self->fh(1,"name");
     }
-
+    if($opt::pipe and $self->virgin()) {
+    } else {
 	if($Global::joblog) { $self->print_joblog() }
+    }
 
     # Printing is only relevant for grouped output.
     $Global::grouped or return;
@@ -4957,7 +4965,14 @@ sub print {
 	if($opt::files) {
 	    # If --compress: $in_fh must be closed first.
 	    close $self->fh($fdno,"w");
-	    if($fdno == 1 and $self->fh($fdno,"name")) {
+	    close $in_fh;
+	    if($opt::pipe and $self->virgin()) {
+		# Nothing was printed to this job:                                                                                           # cleanup tmp files if --files was set
+		for my $fdno (1,2) {
+		    unlink $self->fh($fdno,"name");
+		    unlink $self->fh($fdno,"unlink");
+		}
+	    } elsif($fdno == 1 and $self->fh($fdno,"name")) {
 		print $out_fd $self->fh($fdno,"name"),"\n";
 	    }
 	} elsif($opt::linebuffer) {

testsuite/tests-to-run/parallel-local-100s.sh

@@ -6,7 +6,7 @@
 cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -k -j4 -L1
 echo '### Test if we can deal with output > 4 GB'
 ##  echo | niceload --io 10 parallel -q perl -e '"\$a=\"x\"x1000000;for(0..4300){print \$a}"' | md5sum
-  echo | parallel -q perl -e '$a="x"x1000000;for(0..4300){print $a}' | md5sum
+  echo | parallel --tmpdir /dev/shm -q perl -e '$a="x"x1000000;for(0..4300){print $a}' | md5sum
 
 echo '**'
 

testsuite/tests-to-run/parallel-local114.sh

@@ -1,14 +1,36 @@
 #!/bin/bash
 
 cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -j0 -k -L1
+echo "### --line-buffer"
+  seq 10 | parallel -j20 --line-buffer  'seq {} 10 | pv -qL 10' > /tmp/parallel_l$$; 
+  seq 10 | parallel -j20                'seq {} 10 | pv -qL 10' > /tmp/parallel_$$; 
+  cat /tmp/parallel_l$$ | wc; 
+  diff /tmp/parallel_$$ /tmp/parallel_l$$ >/dev/null ; echo These must diff: $?
+
 echo "### --pipe --line-buffer"
-  seq 200| parallel -N10 -L1 --pipe  -j20 --line-buffer --tagstring {#} pv -qL 10 > /tmp/parallel_$$; 
+  seq 200| parallel -N10 -L1 --pipe  -j20 --line-buffer --tagstring {#} pv -qL 10 > /tmp/parallel_pl$$; 
-  cat /tmp/parallel_$$ | wc; 
+  seq 200| parallel -N10 -L1 --pipe  -j20               --tagstring {#} pv -qL 10 > /tmp/parallel_p$$; 
-  diff <(sort /tmp/parallel_$$) /tmp/parallel_$$ >/dev/null ; echo These must diff: $?
+  cat /tmp/parallel_pl$$ | wc; 
+  diff /tmp/parallel_p$$ /tmp/parallel_pl$$ >/dev/null ; echo These must diff: $?
 
 echo "### --pipe --line-buffer --compress (fails)"
 #  seq 200| parallel -N10 -L1 --pipe  -j20 --line-buffer --compress --tagstring {#} pv -qL 10 | wc
 
+echo "### bug #41482: --pipe --compress blocks at different -j/seq combinations"
+  seq 1 | parallel -k -j2 --compress -N1 -L1 --pipe cat;
+  echo echo 1-4 + 1-4
+    seq 4 | parallel -k -j3 --compress -N1 -L1 -vv echo;
+  echo 4 times wc to stderr to stdout
+    (seq 4 | parallel -k -j3 --compress -N1 -L1 --pipe wc '>&2') 2>&1 >/dev/null
+  echo 1 2 3 4
+    seq 4 | parallel -k -j3 --compress echo;
+  echo 1 2 3 4
+    seq 4 | parallel -k -j1 --compress echo;
+  echo 1 2
+    seq 2 | parallel -k -j1 --compress echo;
+  echo 1 2 3
+    seq 3 | parallel -k -j2 --compress -N1 -L1 --pipe cat;
+
 echo "### bug #41609: --compress fails"
   seq 12 | parallel --compress --compress-program bzip2 -k seq {} 1000000 | md5sum
   seq 12 | parallel --compress -k seq {} 1000000 | md5sum

testsuite/wanted-results/parallel-local114

@@ -1,7 +1,43 @@
+### --line-buffer
+     55      55     120
+These must diff: 1
 ### --pipe --line-buffer
     200     400    1202
 These must diff: 1
 ### --pipe --line-buffer --compress (fails)
+### bug #41482: --pipe --compress blocks at different -j/seq combinations
+1
+echo 1-4 + 1-4
+echo 1
+1
+echo 2
+2
+echo 3
+3
+echo 4
+4
+4 times wc to stderr to stdout
+      1       1       2
+      1       1       2
+      1       1       2
+      1       1       2
+1 2 3 4
+1
+2
+3
+4
+1 2 3 4
+1
+2
+3
+4
+1 2
+1
+2
+1 2 3
+1
+2
+3
 ### bug #41609: --compress fails
 24812dd0f24a26d08a780f988b9d5ad2  -
 24812dd0f24a26d08a780f988b9d5ad2  -