parallel: Implemented setpgrp. Passes testsuite except for some csh tests.

doc/boxplot-runtime

@@ -40,20 +40,19 @@ measure() {
 
     PATH=/tmp/bin:$PATH
     cd /tmp/bin
-#    ls parallel-* | shuf | parallel -j$CORES --joblog /tmp/joblog$CORES-$INNER-$OUTER.csv 'seq '$INNER' | {2} true' :::: <(seq $OUTER) - 
+    ls parallel-* | parallel --shuf -j$CORES --joblog ~/tmp/joblog$CORES-$INNER-$OUTER.csv 'seq '$INNER' | {2} true' :::: <(seq $OUTER) - 
-    ls parallel-* | parallel --shuf -j$CORES --joblog /tmp/joblog$CORES-$INNER-$OUTER.csv 'seq '$INNER' | {2} true' :::: <(seq $OUTER) - 
 
     killall forever
 
     Rscript - <<_
-      jl<-read.csv("/tmp/joblog$CORES-$INNER-$OUTER.csv",sep="\t");
+      jl<-read.csv("$HOME/tmp/joblog$CORES-$INNER-$OUTER.csv",sep="\t");
       jl\$Command <- as.factor(substr(jl\$Command, 12, nchar(as.character(jl\$Command))-5))
       pdf("/tmp/boxplot.pdf");
       par(cex.axis=0.5);
       boxplot(JobRuntime/$INNER*1000~Command,data=jl,las=2,outline=F,
               ylab="milliseconds/job",main="GNU Parallel performance\n$OUTER trials each running $INNER");
 _
-    cp /tmp/boxplot.pdf /tmp/boxplot-j$CORES-${GHZ}ghz-$OUTER-${INNER}v$VERSION.pdf
+    cp /tmp/boxplot.pdf $HOME/tmp/boxplot-j$CORES-${GHZ}ghz-$OUTER-${INNER}v$VERSION.pdf
     evince /tmp/boxplot.pdf
 }
 

doc/release_new_version

@@ -232,7 +232,7 @@ New in this release:
 
 * --retry-failed will retry all failed jobs in a joblog. It will ignore any command given.
 
-* --ssh and $PARALLEL_SSH can be used to set the command used for ssh. The command is assume to behave the same as ssh.
+* --ssh and $PARALLEL_SSH can be used to set the command used for ssh. The command is assumed to behave the same way as ssh.
 
 * --fifo now works in csh, too.
 
@@ -250,7 +250,7 @@ New in this release:
 
 * GNU Parallel was cited in: A composite genome approach to identify phylogenetically informative data from next-generation sequencing http://www.biomedcentral.com/content/pdf/s12859-015-0632-y.pdf
 
-Tutorial: Lorenz-Mie theory for 2D scattering and resonance calculations http://arxiv.org/pdf/1505.07691.pdf
+* GNU Parallel was cited in: Tutorial: Lorenz-Mie theory for 2D scattering and resonance calculations http://arxiv.org/pdf/1505.07691.pdf
 
 * <<Har angiveligt submittet ny version - afventer opdatering>> GNU Parallel was used (unfortunately without citation) in: MUGBAS: a species free gene-based programme suite for post-GWAS analysis http://www.ncbi.nlm.nih.gov/pubmed/25765345
 
@@ -264,6 +264,8 @@ Tutorial: Lorenz-Mie theory for 2D scattering and resonance calculations http://
 
 * GNU Parallel is used in: https://github.com/d2207197/local-mapreduce
 
+* GNU Parallel is used in: wiki-edit-news https://github.com/albert-decatur/wiki-edit-news
+
 * GNU Parallel is used in: A Fingerprint Identification System https://curve.carleton.ca/system/files/theses/28733.pdf
 
 * Job ad asking for GNU Parallel experience: http://searchjobs.intel.com/gdansk-pol/software-validation-engineer/63A06826DAF24797AB414DC146201C2E/job/

src/parallel

@@ -890,6 +890,7 @@ sub parse_options {
     if(@opt::v) { $Global::verbose = $#opt::v+1; } # Convert -v -v to v=2
     $Global::debug = $opt::D;
     $Global::shell = $ENV{'PARALLEL_SHELL'} || parent_shell($$) || $ENV{'SHELL'} || "/bin/sh";
+    $Global::cshell = $Global::shell =~ m:/csh:;
     if(defined $opt::X) { $Global::ContextReplace = 1; }
     if(defined $opt::silent) { $Global::verbose = 0; }
     if(defined $opt::0) { $/ = "\0"; }
@@ -908,6 +909,7 @@ sub parse_options {
     if(defined $opt::colsep) { $Global::trim = 'lr'; }
     if(defined $opt::header) { $opt::colsep = defined $opt::colsep ? $opt::colsep : "\t"; }
     if(defined $opt::trim) { $Global::trim = $opt::trim; }
+    if(defined $opt::roundrobin) { $opt::pipe = 1; }
     if(defined $opt::arg_sep) { $Global::arg_sep = $opt::arg_sep; }
     if(defined $opt::arg_file_sep) { $Global::arg_file_sep = $opt::arg_file_sep; }
     if(defined $opt::number_of_cpus) { print SSHLogin::no_of_cpus(),"\n"; wait_and_exit(0); }
@@ -1259,7 +1261,7 @@ sub parse_semaphore {
     if(defined $opt::semaphorename) { $Global::semaphore = 1; }
     if(defined $opt::fg) { $Global::semaphore = 1; }
     if(defined $opt::bg) { $Global::semaphore = 1; }
-    if(defined $opt::wait) { $Global::semaphore = 1; }
+    if(defined $opt::wait) { $Global::semaphore = 1; @ARGV = "true"; }
     if($Global::semaphore) {
 	if(@opt::a) {
 	    # A semaphore does not take input from neither stdin nor file
@@ -1660,7 +1662,7 @@ sub shell_quote_scalar {
 	if(defined $a) {
 	    if(($a =~ s/'/''/g)
 	       +
-	       ($a =~ s/[\n\002-\011\013-\032\\\#\?\`\(\)\{\}\[\]\^\*\<\=\>\~\|\; \"\!\$\&\202-\377]+/'$&'/go)) {
+	       ($a =~ s/[\n\002-\011\013-\032\\\#\?\`\(\)\{\}\[\]\^\*\<\=\>\~\|\; \"\!\$\&\'\202-\377]+/'$&'/go)) {
 		# A string was replaced
 		# No need to test for "" or \0
 	    } elsif($a eq "") {
@@ -6321,7 +6323,7 @@ sub total_failed {
 	    # * cat > fifo
 	    # * waitpid to get the exit code from $command
 	    # * be less than 1000 chars long
-	    $script = "perl -e ". ::shell_quote_scalar
+	    $script = "perl -e '".
 		(::spacefree
 		 (0, q{
 		     ($s,$c,$f) = @ARGV;
@@ -6340,7 +6342,7 @@ sub total_failed {
 		     # Cleanup
 		     unlink $f;
 		     exit $?/256;
-		  }));
+		  }))."'";
 	}
 	return $script;
     }
@@ -6446,6 +6448,13 @@ sub wrapped {
 	    # Wrap command with 'tmux'
 	    $command = $self->tmux_wrap($command);
 	}
+	if($Global::cshell
+	   and
+	   length $command > 499) {
+	    # csh does not like words longer than 1000 (499 quoted)
+	    $command = "perl -e '".base64_zip_eval()."' ".
+		join" ",string_zip_base64('exec "'.::perl_quote_scalar($command).'"');
+	}
 	$self->{'wrapped'} = $command;
     }
     return $self->{'wrapped'};
@@ -6954,18 +6963,62 @@ sub parentdirs_of {
     return @parents;
 }
 
+sub open3_setpgrp_external {
+    my ($in,$command) = @_;
+    my $pid;
+    my @setpgrp_wrap = ('perl','-e',"setpgrp\;exec '$Global::shell', '-c', \@ARGV");
+    # The eval is needed to catch exception from open3
+    eval {
+	$pid = ::open3($in, ">&OUT", ">&ERR", @setpgrp_wrap, $command)
+	    || ::die_bug("open3-$in");
+	1;
+    };
+    return $pid;
+}
+
+sub open3_setpgrp_internal {
+    my ($in,$command) = @_;
+    my $pid;
+    # The eval is needed to catch exception from open3
+    eval {
+	if(not $pid = ::open3($in, ">&OUT", ">&ERR", "-")) {
+	    # Each child gets its own process group to make it safe to killall
+	    setpgrp(0,0);
+	    exec("exec $Global::shell -c ".::shell_quote_scalar($command))
+		|| ::die_bug("open3-$in $command");
+	}
+    };
+    return $pid;
+}
+
+sub open3_setpgrp {
+    # If the OS supports open3(x,x,x,"-") use that
+    eval{ ::open3(Symbol::gensym(), ">&STDOUT", ">&STDERR", "-")};
+#    if($@) {
+    if(1) {
+	# Does not support open3(x,x,x,"-")
+	*open3_setpgrp = \&open3_setpgrp_external;
+    } else {
+	# Supports open3(x,x,x,"-")
+	*open3_setpgrp = \&open3_setpgrp_internal;
+    }
+    # The sub is now redefined. Call it
+    return open3_setpgrp(@_);
+}
+
 sub start {
     # Setup STDOUT and STDERR for a job and start it.
     # Returns:
     #   job-object or undef if job not to run
+
     my $job = shift;
     # Get the shell command to be executed (possibly with ssh infront).
     my $command = $job->wrapped();
-
+    my $pid;
+    
     if($Global::interactive or $Global::stderr_verbose) {
 	$command = interactive_start($command);
     }
-    my $pid;
     $job->openoutputfiles();
     my($stdout_fh,$stderr_fh) = ($job->fh(1,"w"),$job->fh(2,"w"));
     local (*IN,*OUT,*ERR);
@@ -6978,21 +7031,33 @@ sub start {
     if($opt::dryrun) {
 	$command = "true";
     }
-    # Each child gets its own process group to make it safe to killall
-    my @setpgrp_wrap = ('perl','-e',"setpgrp\;exec '$Global::shell', '-c', \@ARGV");
     $ENV{'PARALLEL_SEQ'} = $job->seq();
     $ENV{'PARALLEL_PID'} = $$;
     $ENV{'PARALLEL_TMP'} = ::tmpname("par");
     ::debug("run", $Global::total_running, " processes . Starting (",
 	    $job->seq(), "): $command\n");
+    $command = "exec $Global::shell -c ".::shell_quote_scalar($command);
+
     if($opt::pipe) {
 	my ($stdin_fh);
+#	$pid = open3_setpgrp($stdin_fh,$command);
+#	$pid = open3_setpgrp($stdin_fh,$command);
 	# The eval is needed to catch exception from open3
-	eval {
+    my @setpgrp_wrap = ('perl','-e',"setpgrp\;exec '$Global::shell', '-c', \@ARGV");
-	    $pid = ::open3($stdin_fh, ">&OUT", ">&ERR", @setpgrp_wrap, $command) ||
+    # The eval is needed to catch exception from open3
-		::die_bug("open3-pipe");
+    eval {
-	    1;
+	$pid = ::open3($stdin_fh, ">&OUT", ">&ERR", @setpgrp_wrap, $command)
-	};
+	    || ::die_bug("open3-stdin");
+	1;
+    };
+#	eval {
+#	    if(not $pid = ::open3($stdin_fh, ">&OUT", ">&ERR", "-")) {
+#		# Each child gets its own process group to make it safe to killall
+#		setpgrp(0,0);
+#		exec($command) || ::die_bug("open3-pipe");
+#	    }
+#	    1;
+#	};
 	$job->set_fh(0,"w",$stdin_fh);
     } elsif ($opt::tty and not $Global::tty_taken and -c "/dev/tty" and
 	     open(my $devtty_fh, "<", "/dev/tty")) {
@@ -7000,7 +7065,7 @@ sub start {
 	*IN = $devtty_fh;
 	# The eval is needed to catch exception from open3
 	eval {
-	    $pid = ::open3("<&IN", ">&OUT", ">&ERR", $Global::shell, "-c", $command) ||
+	    $pid = ::open3("<&IN", ">&OUT", ">&ERR", $command) ||
 		::die_bug("open3-/dev/tty");
 	    $Global::tty_taken = $pid;
 	    close $devtty_fh;
@@ -7011,22 +7076,46 @@ sub start {
 	# Give STDIN to the first job if using -a (but only if running
 	# locally - otherwise CTRL-C does not work for other jobs Bug#36585)
 	*IN = *STDIN;
+#	$pid = open3_setpgrp_internal("<&IN",$command);
+#	$pid = open3_setpgrp("<&IN",$command);
 	# The eval is needed to catch exception from open3
-	eval {
+    my @setpgrp_wrap = ('perl','-e',"setpgrp\;exec '$Global::shell', '-c', \@ARGV");
-	    $pid = ::open3("<&IN", ">&OUT", ">&ERR", @setpgrp_wrap, $command) ||
+    # The eval is needed to catch exception from open3
-		::die_bug("open3-a");
+    eval {
-	    1;
+	$pid = ::open3("<&IN", ">&OUT", ">&ERR", @setpgrp_wrap, $command)
-	};
+	    || ::die_bug("open3-IN");
+	1;
+    };
+#	eval {
+#	    if(not $pid = ::open3("<&IN", ">&OUT", ">&ERR", "-")) {
+#		# Each child gets its own process group to make it safe to killall
+#		setpgrp(0,0);
+#		exec($command) || ::die_bug("open3-a");
+#	    }
+#	    1;
+#	};
 	# Re-open to avoid complaining
 	open(STDIN, "<&", $Global::original_stdin)
 	    or ::die_bug("dup-\$Global::original_stdin: $!");
     } else {
+#	$pid = open3_setpgrp_internal(::gensym,$command);
+#	$pid = open3_setpgrp(::gensym,$command);
 	# The eval is needed to catch exception from open3
-	eval {
+    my @setpgrp_wrap = ('perl','-e',"setpgrp\;exec '$Global::shell', '-c', \@ARGV");
-	    $pid = ::open3(::gensym, ">&OUT", ">&ERR", @setpgrp_wrap, $command) ||
+    # The eval is needed to catch exception from open3
-		::die_bug("open3-gensym");
+    eval {
-	    1;
+	$pid = ::open3(::gensym, ">&OUT", ">&ERR", @setpgrp_wrap, $command)
-	};
+	    || ::die_bug("open3-gens");
+	1;
+    };
+#	eval {
+#	    if(not $pid = ::open3(::gensym, ">&OUT", ">&ERR", "-")) {
+#		# Each child gets its own process group to make it safe to killall
+#		setpgrp(0,0);
+#		exec($command) || ::die_bug("open3-gensym $command");
+#	    }
+#	    1;
+#	};
     }
     if($pid) {
 	# A job was started
@@ -8022,7 +8111,6 @@ sub len {
     }
     # If we are using --env, add the prefix for that, too.
     $len += $Global::envvarlen;
-
     return $len;
 }
 
@@ -8500,14 +8588,15 @@ sub max_length {
 	}
 	$Limits::Command::line_max_len = tmux_length($cached_limit);
 	if($opt::max_chars) {
-	    if($opt::max_chars <= $cached_limit) {
+	    if($opt::max_chars * 2 <= $cached_limit) {
-		$Limits::Command::line_max_len = $opt::max_chars;
+		# $opt::max_chars quoting causes the length to double
+		$Limits::Command::line_max_len = $opt::max_chars * 2;
 	    } else {
 		::warning("Value for -s option should be < $cached_limit.");
 	    }
 	}
     }
-    return $Limits::Command::line_max_len;
+    return int($Limits::Command::line_max_len/2);
 }
 
 sub real_max_length {

src/parallel.pod

@@ -1557,6 +1557,8 @@ time to initialize.
 B<--keep-order> will not work with B<--round-robin> as it is
 impossible to track which input block corresponds to which output.
 
+B<--round-robin> implies B<--pipe>.
+
 
 =item B<--rpl> 'I<tag> I<perl expression>'
 
@@ -2369,6 +2371,18 @@ If B<my_program> fails a red FAIL will be printed followed by the failing
 command; otherwise a green OK will be printed followed by the command.
 
 
+=head1 EXAMPLE: Log rotate
+
+Log rotation renames a logfile to an extension with a higher number:
+log.1 becomes log.2, log.2 becomes log.3, and so on. The oldest log is
+removed. To avoid overwriting files the process starts backwards from
+the high number to the low number.  This will keep 10 old versions of
+the log:
+
+  seq 9 -1 1 | parallel -j1 mv log.{} log.'{= $_++ =}'
+  mv log log.1
+
+
 =head1 EXAMPLE: Removing file extension when processing files
 
 When processing files removing the file extension using B<{.}> is

testsuite/Start.sh

@@ -39,6 +39,9 @@ export -f run_test
 # Create a monitor script
 echo forever pstree -lp $$ >/tmp/monitor
 chmod 755 /tmp/monitor
+# log rotate
+parallel -j1 mv testsuite.log.{} testsuite.log.'{= $_++ =}' ::: 10 9 8 7 6 5 4 3 2 1
+mv testsuite.log testsuite.log.1
 date
 mkdir -p actual-results
 ls -t tests-to-run/*${1}*.sh | egrep -v "${2}" |
@@ -51,5 +54,4 @@ else
   rm -rf src-passing-testsuite
   cp -a ../src src-passing-testsuite
 fi
-rm testsuite.log
 date

testsuite/tests-to-run/parallel-local-ssh1.sh

@@ -24,7 +24,7 @@ echo '### bug #41805: Idea: propagate --env for parallel --number-of-cores'
   echo '** bug #41805 done'
 
 echo '### Deal with long command lines on remote servers'
-  perl -e 'print((("\""x10000)."\n")x10)' | parallel -j1 -S lo -N 10000 echo {} |wc
+  perl -e 'print((("\""x5000)."\n")x10)' | parallel -j1 -S lo -N 10000 echo {} |wc
 
 echo '### Test bug #34241: --pipe should not spawn unneeded processes'
   seq 5 | ssh csh@lo parallel -k --block 5 --pipe -j10 cat\\\;echo Block_end

testsuite/tests-to-run/parallel-local-ssh4.sh

@@ -14,16 +14,16 @@ echo '### zsh'
 
 echo '### csh'
   echo "3 big vars run remotely - length(base64) > 1000"
-  ssh csh@lo 'setenv A `seq 200|xargs`; 
+  stdout ssh csh@lo 'setenv A `seq 200|xargs`; 
-              setenv B `seq 200 -1 1|xargs`; 
+                     setenv B `seq 200 -1 1|xargs`; 
-              setenv C `seq 300 -2 1|xargs`; 
+                     setenv C `seq 300 -2 1|xargs`; 
-              parallel -Scsh@lo --env A,B,C -k echo \$\{\}\|wc ::: A B C'
+                     parallel -Scsh@lo --env A,B,C -k echo \$\{\}\|wc ::: A B C'
 
   echo "3 big vars run locally"
-  ssh csh@lo 'setenv A `seq 200|xargs`; 
+  stdout ssh csh@lo 'setenv A `seq 200|xargs`; 
-              setenv B `seq 200 -1 1|xargs`; 
+                     setenv B `seq 200 -1 1|xargs`; 
-              setenv C `seq 300 -2 1|xargs`; 
+                     setenv C `seq 300 -2 1|xargs`; 
-              parallel --env A,B,C -k echo \$\{\}\|wc ::: A B C'
+                     parallel --env A,B,C -k echo \$\{\}\|wc ::: A B C'
 
 echo '### Test tmux works on different shells'
   parallel -Scsh@lo,tcsh@lo,parallel@lo,zsh@lo --tmux echo ::: 1 2 3 4; echo $?

testsuite/tests-to-run/parallel-local7.sh

@@ -62,7 +62,7 @@ perl -e 'print map { ($_, map { pack("c*",$_) } grep { $_>=1 && $_!=10 } 0..$_),
 
 echo '### Test output ascii'
   rm -f /tmp/paralocal7*; 
-  perl -e 'print map { ($_, map { pack("c*",$_) } grep { $_!=10 } 1..$_),"\n" } 1..255' | stdout parallel --tmux echo {}'>>/tmp/paralocal7{%}' | par_tmux_filter; 
+  perl -e 'print map { ($_, map { pack("c*",$_) } grep { $_>=1 && $_!=10 } $_-10..$_),"\n" } 1..255' | stdout parallel --tmux echo {}'>>/tmp/paralocal7{%}' | par_tmux_filter; 
   sort /tmp/paralocal7* | md5sum
 
 echo '### Test critical lengths. Must not block'

testsuite/tests-to-run/parallel-tutorial.sh

@@ -5,6 +5,7 @@ cd testsuite 2>/dev/null
 rm -rf tmp
 mkdir tmp
 cd tmp
+touch ~/.parallel/will-cite
 echo '### test parallel_tutorial'
 rm -f /tmp/runs
 export SERVER1=parallel@lo

testsuite/wanted-results/parallel-local-0.3s

@@ -359,12 +359,8 @@ pdksh	"#&/
 pdksh	()*=?'
 posh	"#&/
 posh	()*=?'
-rbash	"#&/
-rbash	()*=?'
 rc	"#&/
 rc	()*=?'
-rzsh	"#&/
-rzsh	()*=?'
 sash	"#&/
 sash	()*=?'
 sh	"#&/
@@ -377,6 +373,8 @@ yash	"#&/
 yash	()*=?'
 zsh	"#&/
 zsh	()*=?'
+rbash	rbash: line 0: exec: restricted
+rzsh	zsh:exec:1: rzsh: restricted
 echo '**'
 **
 ### 1 .par file from --files expected

testsuite/wanted-results/parallel-local-ssh1

@@ -34,8 +34,8 @@ HOME=/home/tange
 ** bug #41805 done
 echo '### Deal with long command lines on remote servers'
 ### Deal with long command lines on remote servers
-  perl -e 'print((("\""x10000)."\n")x10)' | parallel -j1 -S lo -N 10000 echo {} |wc
+  perl -e 'print((("\""x5000)."\n")x10)' | parallel -j1 -S lo -N 10000 echo {} |wc
-      4      10  100010
+      4      10   50010
 echo '### Test bug #34241: --pipe should not spawn unneeded processes'
 ### Test bug #34241: --pipe should not spawn unneeded processes
   seq 5 | ssh csh@lo parallel -k --block 5 --pipe -j10 cat\\\;echo Block_end

testsuite/wanted-results/parallel-local-ssh4

@@ -8,13 +8,13 @@ echo '### csh'
 ### csh
   echo "3 big vars run remotely - length(base64) > 1000"
 3 big vars run remotely - length(base64) > 1000
-  ssh csh@lo 'setenv A `seq 200|xargs`;               setenv B `seq 200 -1 1|xargs`;               setenv C `seq 300 -2 1|xargs`;               parallel -Scsh@lo --env A,B,C -k echo \$\{\}\|wc ::: A B C'
+  stdout ssh csh@lo 'setenv A `seq 200|xargs`;               setenv B `seq 200 -1 1|xargs`;               setenv C `seq 300 -2 1|xargs`;               parallel -Scsh@lo --env A,B,C -k echo \$\{\}\|wc ::: A B C'
       1     200     692
       1     200     692
       1     150     547
   echo "3 big vars run locally"
 3 big vars run locally
-  ssh csh@lo 'setenv A `seq 200|xargs`;               setenv B `seq 200 -1 1|xargs`;               setenv C `seq 300 -2 1|xargs`;               parallel --env A,B,C -k echo \$\{\}\|wc ::: A B C'
+  stdout ssh csh@lo 'setenv A `seq 200|xargs`;               setenv B `seq 200 -1 1|xargs`;               setenv C `seq 300 -2 1|xargs`;               parallel --env A,B,C -k echo \$\{\}\|wc ::: A B C'
       1     200     692
       1     200     692
       1     150     547
@@ -48,7 +48,7 @@ echo '### works'
 ### works
   stdout parallel -Sparallel@lo --tmux echo ::: \\\\\\\"\\\\\\\"\\\;\@ | grep -v 'See output'
   stdout parallel -Sparallel@lo --tmux echo ::: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx | grep -v 'See output'
-parallel: Error: Command line too long (402 >= 236) at input 0: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx...
+parallel: Error: Command line too long (402 >= 118) at input 0: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx...
 echo '### These blocked due to length'
 ### These blocked due to length
   stdout parallel -Slo      --tmux echo ::: \\\\\\\"\\\\\\\"\\\;\@ | grep -v 'See output'

testsuite/wanted-results/parallel-local13

@@ -456,6 +456,7 @@ echo '###  true < 32767-ys.xi'
 ###  true < 32767-ys.xi
 stdout xargs true < 32767-ys.xi
 stdout parallel -k true < 32767-ys.xi
+parallel: Error: Command line too long (98306 >= 65535) at input 0: y y y y y y y y y y y y y y y y y y y y y y y y y ...
 echo '###  true < 16383-ys.xi'
 ###  true < 16383-ys.xi
 stdout xargs true < 16383-ys.xi

testsuite/wanted-results/parallel-local19

@@ -107,24 +107,24 @@ seq 1 6 | parallel -k printf '{}.gif\\n' | parallel -j1 -kX echo a{}b{.}c{.}
 a1.gifb1c1 a2.gifb2c2 a3.gifb3c3 a4.gifb4c4 a5.gifb5c5 a6.gifb6c6
 echo '### Test -m with 60000 args';   seq 1 60000 | perl -pe 's/$/.gif/' |   parallel -j1 -km echo a{}b{.}c{.} |   tee >(wc) >(md5sum) >/dev/null;   wait
 ### Test -m with 60000 args
-565b18712ac24f5c46f6e64cf4548733  -
+0891d955e7e2659ee6a88a96f3309b5c  -
-     10  179980 1286692
+     20  179960 1286702
 echo '### Test -X with 60000 args';   seq 1 60000 | perl -pe 's/$/.gif/' |   parallel -j1 -kX echo a{}b{.}c{.} |   tee >(wc) >(md5sum) >/dev/null;   wait
 ### Test -X with 60000 args
-f09f466888e1c737eb581f41cb0087ba  -
+af6c8c69409b5cbc55aeb95174e23c37  -
-     11   60000 1346682
+     21   60000 1346682
 echo '### Test -X with 60000 args and 5 expansions'
 ### Test -X with 60000 args and 5 expansions
 seq 1 60000 | perl -pe 's/$/.gif/' | parallel -j1 -kX echo a{}b{.}c{.}{.}{} | wc -l
-17
+34
 seq 1 60000 | perl -pe 's/$/.gif/' | parallel -j1 -kX echo a{}b{.}c{.}{.} | wc -l
-13
+25
 seq 1 60000 | perl -pe 's/$/.gif/' | parallel -j1 -kX echo a{}b{.}c{.} | wc -l
-11
+21
 seq 1 60000 | perl -pe 's/$/.gif/' | parallel -j1 -kX echo a{}b{.}c | wc -l
-9
+17
 seq 1 60000 | perl -pe 's/$/.gif/' | parallel -j1 -kX echo a{}b | wc -l
-6
+11
 echo '### Test {.} does not repeat more than {}'
 ### Test {.} does not repeat more than {}
 seq 1 15 | perl -pe 's/$/.gif/'   | parallel -j1 -s 80 -kX echo a{}b{.}c{.}
@@ -146,13 +146,13 @@ a14.gif 15.gifb14 15c14 15
 echo '### Test -I with shell meta chars'
 ### Test -I with shell meta chars
 seq 1 60000 | parallel -j1 -I :: -X echo a::b::c:: | wc -l
-9
+17
 seq 1 60000 | parallel -j1 -I '<>' -X echo 'a<>b<>c<>' | wc -l
-9
+17
 seq 1 60000 | parallel -j1 -I '<' -X echo 'a<b<c<' | wc -l
-9
+17
 seq 1 60000 | parallel -j1 -I '>' -X echo 'a>b>c>' | wc -l
-9
+17
 echo '### Test {.}'
 ### Test {.}
 echo a | parallel -qX echo  "'"{.}"' "

testsuite/wanted-results/parallel-local22

@@ -1,16 +1,16 @@
 echo '### Test of xargs -m command lines > 130k';   seq 1 60000 | parallel -m -j1 echo a{}b{}c | tee >(wc >/tmp/awc$$) >(sort | md5sum) >/tmp/a$$;   wait;   CHAR=$(cat /tmp/a$$ | wc -c);   LINES=$(cat /tmp/a$$ | wc -l);   echo "Chars per line:" $(echo "$CHAR/$LINES" | bc);   cat /tmp/awc$$;   rm /tmp/a$$ /tmp/awc$$
 ### Test of xargs -m command lines > 130k
-31d9274be5fdc2de59487cb05ba57776  -
+cb2184f70b8e9058cae1413426f8a9ae  -
-Chars per line: 116300
+Chars per line: 63437
-      6  119994  697800
+     11  119989  697810
 echo '### Test of xargs -X command lines > 130k';   seq 1 60000 | parallel -X -j1 echo a{}b{}c | tee >(wc >/tmp/bwc$$) >(sort | (sleep 1; md5sum)) >/tmp/b$$;   wait;   CHAR=$(cat /tmp/b$$ | wc -c);   LINES=$(cat /tmp/b$$ | wc -l);   echo "Chars per line:" $(echo "$CHAR/$LINES" | bc);   cat /tmp/bwc$$;   rm /tmp/b$$ /tmp/bwc$$
 ### Test of xargs -X command lines > 130k
-22074f9acada52462defb18ba912d744  -
+514f33181f346d416a1ada982b43a542  -
-Chars per line: 116826
+Chars per line: 62906
-      7   60000  817788
+     13   60000  817788
 echo '### Test of xargs -m command lines > 130k';   seq 1 60000 | parallel -k -j1 -m echo | md5sum
 ### Test of xargs -m command lines > 130k
-b35d8e49be8d94899b719c40d3f1f4bb  -
+1e763b036fe666d229520dad45610e6c  -
 echo '### This causes problems if we kill child processes';   seq 2 40 | parallel -j 0 seq 1 10  | sort | md5sum
 ### This causes problems if we kill child processes
 437c0d47a99b9a7c5bcb1d132f94c2e6  -
@@ -191,8 +191,10 @@ which static-sh => shell path /bin/static-sh
 which tcsh => shell path /usr/bin/tcsh
 which yash => shell path /usr/bin/yash
 which zsh => shell path /usr/bin/zsh
+/bin/rbash: line 0: exec: restricted
 Local configuration error occurred.
 Contact the systems administrator for further assistance.
+zsh:exec:1: /bin/rzsh: restricted
   echo '## Started directly from perl'
 ## Started directly from perl
   perl -e 'system(qw(parallel -Dinit echo ::: 1))' | grep which;   rm -f /tmp/par*.par

testsuite/wanted-results/parallel-local23

@@ -86,8 +86,9 @@ please cite as described in 'parallel --bibtex'.
 echo '### bug #39787: --xargs broken'
 ### bug #39787: --xargs broken
   nice perl -e 'for(1..30000){print "$_\n"}' | $NICEPAR --xargs -k echo  | perl -ne 'print length $_,"\n"'
-131064
+65526
-37830
+65526
+37842
 echo '### --delay should grow by 3 sec per arg'
 ### --delay should grow by 3 sec per arg
   stdout /usr/bin/time -f %e parallel --delay 3 true ::: 1 2 | perl -ne '$_ >= 3 and $_ <= 8 and print "OK\n"'

testsuite/wanted-results/parallel-local7

@@ -31,51 +31,51 @@ See output with: tmux1.9 -S /tmp/parallel-local7/tmsXXXXX attach
 See output with: tmux1.9 -S /tmp/parallel-local7/tmsXXXXX attach
 0
   seq 1000 1100 | TMUX=tmux1.9 par_tmux
-See output with: tmux1.9 -S /tmp/parallel-local7/tmsXXXXX attach
+parallel: Error: Command line too long (2023 >= 2023) at input 0: 1007
-0
+255
   seq 1100 1200 | TMUX=tmux1.9 par_tmux
-See output with: tmux1.9 -S /tmp/parallel-local7/tmsXXXXX attach
+parallel: Error: Command line too long (2209 >= 2023) at input 0: 1100
-0
+255
   seq 1200 1300 | TMUX=tmux1.9 par_tmux
-See output with: tmux1.9 -S /tmp/parallel-local7/tmsXXXXX attach
+parallel: Error: Command line too long (2409 >= 2023) at input 0: 1200
-0
+255
   seq 1300 1400 | TMUX=tmux1.9 par_tmux
-See output with: tmux1.9 -S /tmp/parallel-local7/tmsXXXXX attach
+parallel: Error: Command line too long (2609 >= 2023) at input 0: 1300
-0
+255
   seq 1400 1500 | TMUX=tmux1.9 par_tmux
-See output with: tmux1.9 -S /tmp/parallel-local7/tmsXXXXX attach
+parallel: Error: Command line too long (2809 >= 2023) at input 0: 1400
-0
+255
   seq 1500 1600 | TMUX=tmux1.9 par_tmux
-See output with: tmux1.9 -S /tmp/parallel-local7/tmsXXXXX attach
+parallel: Error: Command line too long (3009 >= 2023) at input 0: 1500
-0
+255
   seq 1600 1700 | TMUX=tmux1.9 par_tmux
-See output with: tmux1.9 -S /tmp/parallel-local7/tmsXXXXX attach
+parallel: Error: Command line too long (3209 >= 2023) at input 0: 1600
-0
+255
   seq 1700 1800 | TMUX=tmux1.9 par_tmux
-See output with: tmux1.9 -S /tmp/parallel-local7/tmsXXXXX attach
+parallel: Error: Command line too long (3409 >= 2023) at input 0: 1700
-0
+255
   seq 1800 1900 | TMUX=tmux1.9 par_tmux
-See output with: tmux1.9 -S /tmp/parallel-local7/tmsXXXXX attach
+parallel: Error: Command line too long (3609 >= 2023) at input 0: 1800
-0
+255
   seq 1900 2000 | TMUX=tmux1.9 par_tmux
-See output with: tmux1.9 -S /tmp/parallel-local7/tmsXXXXX attach
+parallel: Error: Command line too long (3809 >= 2023) at input 0: 1900
-0
+255
   seq 2000 2018 | TMUX=tmux1.9 par_tmux
-See output with: tmux1.9 -S /tmp/parallel-local7/tmsXXXXX attach
+parallel: Error: Command line too long (4009 >= 2023) at input 0: 2000
-0
+255
 echo '### tmux1.9 fails'
 ### tmux1.9 fails
   echo 2019 | TMUX=tmux1.9 par_tmux
-parallel: Error: Command line too long (4047 >= 4047) at input 0: 2019
+parallel: Error: Command line too long (4047 >= 2023) at input 0: 2019
 255
   echo 2020 | TMUX=tmux1.9 par_tmux
-parallel: Error: Command line too long (4049 >= 4047) at input 0: 2020
+parallel: Error: Command line too long (4049 >= 2023) at input 0: 2020
 255
   echo 2021 | TMUX=tmux1.9 par_tmux
-parallel: Error: Command line too long (4051 >= 4047) at input 0: 2021
+parallel: Error: Command line too long (4051 >= 2023) at input 0: 2021
 255
   echo 2022 | TMUX=tmux1.9 par_tmux
-parallel: Error: Command line too long (4053 >= 4047) at input 0: 2022
+parallel: Error: Command line too long (4053 >= 2023) at input 0: 2022
 255
 echo '### tmux1.8'
 ### tmux1.8
@@ -87,34 +87,36 @@ See output with: tmux1.8 -S /tmp/parallel-local7/tmsXXXXX attach
 0
   seq 101 150 | TMUX=tmux1.8 par_tmux
 See output with: tmux1.8 -S /tmp/parallel-local7/tmsXXXXX attach
-0
+parallel: Error: Command line too long (236 >= 236) at input 0: 114
+255
   seq 151 200 | TMUX=tmux1.8 par_tmux
-See output with: tmux1.8 -S /tmp/parallel-local7/tmsXXXXX attach
+parallel: Error: Command line too long (310 >= 236) at input 0: 151
-0
+255
   seq 201 231 | TMUX=tmux1.8 par_tmux
-See output with: tmux1.8 -S /tmp/parallel-local7/tmsXXXXX attach
+parallel: Error: Command line too long (410 >= 236) at input 0: 201
-0
+255
 echo '### tmux1.8 fails'
 ### tmux1.8 fails
   echo 232 | TMUX=tmux1.8 par_tmux
-parallel: Error: Command line too long (472 >= 472) at input 0: 232
+parallel: Error: Command line too long (472 >= 236) at input 0: 232
 255
   echo 233 | TMUX=tmux1.8 par_tmux
-parallel: Error: Command line too long (474 >= 472) at input 0: 233
+parallel: Error: Command line too long (474 >= 236) at input 0: 233
 255
   echo 234 | TMUX=tmux1.8 par_tmux
-parallel: Error: Command line too long (476 >= 472) at input 0: 234
+parallel: Error: Command line too long (476 >= 236) at input 0: 234
 255
   echo 235 | TMUX=tmux1.8 par_tmux
-parallel: Error: Command line too long (478 >= 472) at input 0: 235
+parallel: Error: Command line too long (478 >= 236) at input 0: 235
 255
   echo 236 | TMUX=tmux1.8 par_tmux
-parallel: Error: Command line too long (480 >= 472) at input 0: 236
+parallel: Error: Command line too long (480 >= 236) at input 0: 236
 255
 echo '### tmux1.8 0..255 ascii'
 ### tmux1.8 0..255 ascii
 perl -e 'print map { ($_, map { pack("c*",$_) } grep { $_>=1 && $_!=10 } 0..$_),"\n" } 0..255' |    TMUX=tmux1.8 stdout parallel --tmux --timeout 3 echo | par_tmux_filter; echo $?
 See output with: tmux1.8 -S /tmp/parallel-local7/tmsXXXXX attach
+parallel: Error: Command line too long (236 >= 236) at input 0: 155
	
 !"#$%&'()*+,-./0...
 0
 echo '### tmux1.9 0..255 ascii'
 ### tmux1.9 0..255 ascii
@@ -123,16 +125,17 @@ See output with: tmux1.9 -S /tmp/parallel-local7/tmsXXXXX attach
 0
 echo '### Test output ascii'
 ### Test output ascii
-  rm -f /tmp/paralocal7*;   perl -e 'print map { ($_, map { pack("c*",$_) } grep { $_!=10 } 1..$_),"\n" } 1..255' | stdout parallel --tmux echo {}'>>/tmp/paralocal7{%}' | par_tmux_filter;   sort /tmp/paralocal7* | md5sum
+  rm -f /tmp/paralocal7*;   perl -e 'print map { ($_, map { pack("c*",$_) } grep { $_>=1 && $_!=10 } $_-10..$_),"\n" } 1..255' | stdout parallel --tmux echo {}'>>/tmp/paralocal7{%}' | par_tmux_filter;   sort /tmp/paralocal7* | md5sum
 See output with: tmux -S /tmp/parallel-local7/tmsXXXXX attach
-a7ee232967c8aab2edf227169e8cdce0  -
+053c7e8e945ef7641fc63bc309ac069d  -
 echo '### Test critical lengths. Must not block'
 ### Test critical lengths. Must not block
   seq 70 130  | TMUX=tmux1.8 stdout parallel --tmux echo '{}{=$_="&"x$_=}' | par_tmux_filter
 See output with: tmux1.8 -S /tmp/parallel-local7/tmsXXXXX attach
+parallel: Error: Command line too long (236 >= 236) at input 0: 114
   seq 70 130  | TMUX=tmux1.9 stdout parallel --tmux echo '{}{=$_="&"x$_=}' | par_tmux_filter
 See output with: tmux1.9 -S /tmp/parallel-local7/tmsXXXXX attach
   seq 280 425 | TMUX=tmux1.8 stdout parallel --tmux echo '{}{=$_="a"x$_=}' | par_tmux_filter
-See output with: tmux1.8 -S /tmp/parallel-local7/tmsXXXXX attach
+parallel: Error: Command line too long (288 >= 236) at input 0: 280
   seq 280 425 | TMUX=tmux1.9 stdout parallel --tmux echo '{}{=$_="a"x$_=}' | par_tmux_filter
 See output with: tmux1.9 -S /tmp/parallel-local7/tmsXXXXX attach

testsuite/wanted-results/parallel-local9

@@ -403,12 +403,12 @@ a1b1^c1[.}c
 echo "### BUG: The length for -X is not close to max (131072)"
 ### BUG: The length for -X is not close to max (131072)
 seq 1 4000 | nice parallel -k -X echo {.} aa {}{.} {}{}d{} {}dd{}d{.} |head -n 1 |wc
-      1   12821  131060
+      1    6861   65500
 echo "### BUG: empty lines with --show-limit"
 ### BUG: empty lines with --show-limit
 echo | $NICEPAR --show-limits
 Maximal size of command: 131071
-Maximal used size of command: 131071
+Maximal used size of command: 65535
 
 Execution of  will continue now, and it will try to read its input
 and run commands; if this is not what you wanted to happen, please

testsuite/wanted-results/parallel-tutorial

@@ -241,11 +241,11 @@ f1=B f2=D
 f1=A f2=B
 f1=C f2=D
   cat num30000 | parallel --xargs echo | wc -l
-2
+3
   cat num30000 | parallel --xargs -s 10000 echo | wc -l
 17
   cat num30000 | parallel --jobs 4 -m echo | wc -l
-5
+6
   parallel --jobs 4 -m echo ::: 1 2 3 4 5 6 7 8 9 10
 1 2 3
 4 5 6