From cb20cd2c8ed1c1ba71cd3fb9b7df02238e34f38b Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Sun, 21 Jun 2015 10:39:48 +0200 Subject: [PATCH] parallel: Implemented setpgrp. Passes testsuite except for some csh tests. --- doc/boxplot-runtime | 7 +- doc/release_new_version | 6 +- src/parallel | 145 ++++++++++++++---- src/parallel.pod | 14 ++ testsuite/Start.sh | 4 +- testsuite/tests-to-run/parallel-local-ssh1.sh | 2 +- testsuite/tests-to-run/parallel-local-ssh4.sh | 16 +- testsuite/tests-to-run/parallel-local7.sh | 2 +- testsuite/tests-to-run/parallel-tutorial.sh | 1 + testsuite/wanted-results/parallel-local-0.3s | 6 +- testsuite/wanted-results/parallel-local-ssh1 | 4 +- testsuite/wanted-results/parallel-local-ssh4 | 6 +- testsuite/wanted-results/parallel-local114 | Bin 11572 -> 11572 bytes testsuite/wanted-results/parallel-local13 | 1 + testsuite/wanted-results/parallel-local19 | 26 ++-- testsuite/wanted-results/parallel-local22 | 16 +- testsuite/wanted-results/parallel-local23 | 5 +- testsuite/wanted-results/parallel-local7 | 81 +++++----- testsuite/wanted-results/parallel-local9 | 4 +- testsuite/wanted-results/parallel-tutorial | 4 +- 20 files changed, 231 insertions(+), 119 deletions(-) diff --git a/doc/boxplot-runtime b/doc/boxplot-runtime index 41eaf6cd..0261faab 100644 --- a/doc/boxplot-runtime +++ b/doc/boxplot-runtime @@ -40,20 +40,19 @@ measure() { PATH=/tmp/bin:$PATH cd /tmp/bin -# ls parallel-* | shuf | parallel -j$CORES --joblog /tmp/joblog$CORES-$INNER-$OUTER.csv 'seq '$INNER' | {2} true' :::: <(seq $OUTER) - - ls parallel-* | parallel --shuf -j$CORES --joblog /tmp/joblog$CORES-$INNER-$OUTER.csv 'seq '$INNER' | {2} true' :::: <(seq $OUTER) - + ls parallel-* | parallel --shuf -j$CORES --joblog ~/tmp/joblog$CORES-$INNER-$OUTER.csv 'seq '$INNER' | {2} true' :::: <(seq $OUTER) - killall forever Rscript - <<_ - jl<-read.csv("/tmp/joblog$CORES-$INNER-$OUTER.csv",sep="\t"); + jl<-read.csv("$HOME/tmp/joblog$CORES-$INNER-$OUTER.csv",sep="\t"); jl\$Command <- as.factor(substr(jl\$Command, 12, nchar(as.character(jl\$Command))-5)) pdf("/tmp/boxplot.pdf"); par(cex.axis=0.5); boxplot(JobRuntime/$INNER*1000~Command,data=jl,las=2,outline=F, ylab="milliseconds/job",main="GNU Parallel performance\n$OUTER trials each running $INNER"); _ - cp /tmp/boxplot.pdf /tmp/boxplot-j$CORES-${GHZ}ghz-$OUTER-${INNER}v$VERSION.pdf + cp /tmp/boxplot.pdf $HOME/tmp/boxplot-j$CORES-${GHZ}ghz-$OUTER-${INNER}v$VERSION.pdf evince /tmp/boxplot.pdf } diff --git a/doc/release_new_version b/doc/release_new_version index fa77e1cf..d151317c 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -232,7 +232,7 @@ New in this release: * --retry-failed will retry all failed jobs in a joblog. It will ignore any command given. -* --ssh and $PARALLEL_SSH can be used to set the command used for ssh. The command is assume to behave the same as ssh. +* --ssh and $PARALLEL_SSH can be used to set the command used for ssh. The command is assumed to behave the same way as ssh. * --fifo now works in csh, too. @@ -250,7 +250,7 @@ New in this release: * GNU Parallel was cited in: A composite genome approach to identify phylogenetically informative data from next-generation sequencing http://www.biomedcentral.com/content/pdf/s12859-015-0632-y.pdf -Tutorial: Lorenz-Mie theory for 2D scattering and resonance calculations http://arxiv.org/pdf/1505.07691.pdf +* GNU Parallel was cited in: Tutorial: Lorenz-Mie theory for 2D scattering and resonance calculations http://arxiv.org/pdf/1505.07691.pdf * <> GNU Parallel was used (unfortunately without citation) in: MUGBAS: a species free gene-based programme suite for post-GWAS analysis http://www.ncbi.nlm.nih.gov/pubmed/25765345 @@ -264,6 +264,8 @@ Tutorial: Lorenz-Mie theory for 2D scattering and resonance calculations http:// * GNU Parallel is used in: https://github.com/d2207197/local-mapreduce +* GNU Parallel is used in: wiki-edit-news https://github.com/albert-decatur/wiki-edit-news + * GNU Parallel is used in: A Fingerprint Identification System https://curve.carleton.ca/system/files/theses/28733.pdf * Job ad asking for GNU Parallel experience: http://searchjobs.intel.com/gdansk-pol/software-validation-engineer/63A06826DAF24797AB414DC146201C2E/job/ diff --git a/src/parallel b/src/parallel index f098311c..d2754200 100755 --- a/src/parallel +++ b/src/parallel @@ -890,6 +890,7 @@ sub parse_options { if(@opt::v) { $Global::verbose = $#opt::v+1; } # Convert -v -v to v=2 $Global::debug = $opt::D; $Global::shell = $ENV{'PARALLEL_SHELL'} || parent_shell($$) || $ENV{'SHELL'} || "/bin/sh"; + $Global::cshell = $Global::shell =~ m:/csh:; if(defined $opt::X) { $Global::ContextReplace = 1; } if(defined $opt::silent) { $Global::verbose = 0; } if(defined $opt::0) { $/ = "\0"; } @@ -908,6 +909,7 @@ sub parse_options { if(defined $opt::colsep) { $Global::trim = 'lr'; } if(defined $opt::header) { $opt::colsep = defined $opt::colsep ? $opt::colsep : "\t"; } if(defined $opt::trim) { $Global::trim = $opt::trim; } + if(defined $opt::roundrobin) { $opt::pipe = 1; } if(defined $opt::arg_sep) { $Global::arg_sep = $opt::arg_sep; } if(defined $opt::arg_file_sep) { $Global::arg_file_sep = $opt::arg_file_sep; } if(defined $opt::number_of_cpus) { print SSHLogin::no_of_cpus(),"\n"; wait_and_exit(0); } @@ -1259,7 +1261,7 @@ sub parse_semaphore { if(defined $opt::semaphorename) { $Global::semaphore = 1; } if(defined $opt::fg) { $Global::semaphore = 1; } if(defined $opt::bg) { $Global::semaphore = 1; } - if(defined $opt::wait) { $Global::semaphore = 1; } + if(defined $opt::wait) { $Global::semaphore = 1; @ARGV = "true"; } if($Global::semaphore) { if(@opt::a) { # A semaphore does not take input from neither stdin nor file @@ -1660,7 +1662,7 @@ sub shell_quote_scalar { if(defined $a) { if(($a =~ s/'/''/g) + - ($a =~ s/[\n\002-\011\013-\032\\\#\?\`\(\)\{\}\[\]\^\*\<\=\>\~\|\; \"\!\$\&\202-\377]+/'$&'/go)) { + ($a =~ s/[\n\002-\011\013-\032\\\#\?\`\(\)\{\}\[\]\^\*\<\=\>\~\|\; \"\!\$\&\'\202-\377]+/'$&'/go)) { # A string was replaced # No need to test for "" or \0 } elsif($a eq "") { @@ -6321,7 +6323,7 @@ sub total_failed { # * cat > fifo # * waitpid to get the exit code from $command # * be less than 1000 chars long - $script = "perl -e ". ::shell_quote_scalar + $script = "perl -e '". (::spacefree (0, q{ ($s,$c,$f) = @ARGV; @@ -6340,7 +6342,7 @@ sub total_failed { # Cleanup unlink $f; exit $?/256; - })); + }))."'"; } return $script; } @@ -6446,6 +6448,13 @@ sub wrapped { # Wrap command with 'tmux' $command = $self->tmux_wrap($command); } + if($Global::cshell + and + length $command > 499) { + # csh does not like words longer than 1000 (499 quoted) + $command = "perl -e '".base64_zip_eval()."' ". + join" ",string_zip_base64('exec "'.::perl_quote_scalar($command).'"'); + } $self->{'wrapped'} = $command; } return $self->{'wrapped'}; @@ -6954,18 +6963,62 @@ sub parentdirs_of { return @parents; } +sub open3_setpgrp_external { + my ($in,$command) = @_; + my $pid; + my @setpgrp_wrap = ('perl','-e',"setpgrp\;exec '$Global::shell', '-c', \@ARGV"); + # The eval is needed to catch exception from open3 + eval { + $pid = ::open3($in, ">&OUT", ">&ERR", @setpgrp_wrap, $command) + || ::die_bug("open3-$in"); + 1; + }; + return $pid; +} + +sub open3_setpgrp_internal { + my ($in,$command) = @_; + my $pid; + # The eval is needed to catch exception from open3 + eval { + if(not $pid = ::open3($in, ">&OUT", ">&ERR", "-")) { + # Each child gets its own process group to make it safe to killall + setpgrp(0,0); + exec("exec $Global::shell -c ".::shell_quote_scalar($command)) + || ::die_bug("open3-$in $command"); + } + }; + return $pid; +} + +sub open3_setpgrp { + # If the OS supports open3(x,x,x,"-") use that + eval{ ::open3(Symbol::gensym(), ">&STDOUT", ">&STDERR", "-")}; +# if($@) { + if(1) { + # Does not support open3(x,x,x,"-") + *open3_setpgrp = \&open3_setpgrp_external; + } else { + # Supports open3(x,x,x,"-") + *open3_setpgrp = \&open3_setpgrp_internal; + } + # The sub is now redefined. Call it + return open3_setpgrp(@_); +} + sub start { # Setup STDOUT and STDERR for a job and start it. # Returns: # job-object or undef if job not to run + my $job = shift; # Get the shell command to be executed (possibly with ssh infront). my $command = $job->wrapped(); - + my $pid; + if($Global::interactive or $Global::stderr_verbose) { $command = interactive_start($command); } - my $pid; $job->openoutputfiles(); my($stdout_fh,$stderr_fh) = ($job->fh(1,"w"),$job->fh(2,"w")); local (*IN,*OUT,*ERR); @@ -6978,21 +7031,33 @@ sub start { if($opt::dryrun) { $command = "true"; } - # Each child gets its own process group to make it safe to killall - my @setpgrp_wrap = ('perl','-e',"setpgrp\;exec '$Global::shell', '-c', \@ARGV"); $ENV{'PARALLEL_SEQ'} = $job->seq(); $ENV{'PARALLEL_PID'} = $$; $ENV{'PARALLEL_TMP'} = ::tmpname("par"); ::debug("run", $Global::total_running, " processes . Starting (", $job->seq(), "): $command\n"); + $command = "exec $Global::shell -c ".::shell_quote_scalar($command); + if($opt::pipe) { my ($stdin_fh); +# $pid = open3_setpgrp($stdin_fh,$command); +# $pid = open3_setpgrp($stdin_fh,$command); # The eval is needed to catch exception from open3 - eval { - $pid = ::open3($stdin_fh, ">&OUT", ">&ERR", @setpgrp_wrap, $command) || - ::die_bug("open3-pipe"); - 1; - }; + my @setpgrp_wrap = ('perl','-e',"setpgrp\;exec '$Global::shell', '-c', \@ARGV"); + # The eval is needed to catch exception from open3 + eval { + $pid = ::open3($stdin_fh, ">&OUT", ">&ERR", @setpgrp_wrap, $command) + || ::die_bug("open3-stdin"); + 1; + }; +# eval { +# if(not $pid = ::open3($stdin_fh, ">&OUT", ">&ERR", "-")) { +# # Each child gets its own process group to make it safe to killall +# setpgrp(0,0); +# exec($command) || ::die_bug("open3-pipe"); +# } +# 1; +# }; $job->set_fh(0,"w",$stdin_fh); } elsif ($opt::tty and not $Global::tty_taken and -c "/dev/tty" and open(my $devtty_fh, "<", "/dev/tty")) { @@ -7000,7 +7065,7 @@ sub start { *IN = $devtty_fh; # The eval is needed to catch exception from open3 eval { - $pid = ::open3("<&IN", ">&OUT", ">&ERR", $Global::shell, "-c", $command) || + $pid = ::open3("<&IN", ">&OUT", ">&ERR", $command) || ::die_bug("open3-/dev/tty"); $Global::tty_taken = $pid; close $devtty_fh; @@ -7011,22 +7076,46 @@ sub start { # Give STDIN to the first job if using -a (but only if running # locally - otherwise CTRL-C does not work for other jobs Bug#36585) *IN = *STDIN; +# $pid = open3_setpgrp_internal("<&IN",$command); +# $pid = open3_setpgrp("<&IN",$command); # The eval is needed to catch exception from open3 - eval { - $pid = ::open3("<&IN", ">&OUT", ">&ERR", @setpgrp_wrap, $command) || - ::die_bug("open3-a"); - 1; - }; + my @setpgrp_wrap = ('perl','-e',"setpgrp\;exec '$Global::shell', '-c', \@ARGV"); + # The eval is needed to catch exception from open3 + eval { + $pid = ::open3("<&IN", ">&OUT", ">&ERR", @setpgrp_wrap, $command) + || ::die_bug("open3-IN"); + 1; + }; +# eval { +# if(not $pid = ::open3("<&IN", ">&OUT", ">&ERR", "-")) { +# # Each child gets its own process group to make it safe to killall +# setpgrp(0,0); +# exec($command) || ::die_bug("open3-a"); +# } +# 1; +# }; # Re-open to avoid complaining open(STDIN, "<&", $Global::original_stdin) or ::die_bug("dup-\$Global::original_stdin: $!"); } else { +# $pid = open3_setpgrp_internal(::gensym,$command); +# $pid = open3_setpgrp(::gensym,$command); # The eval is needed to catch exception from open3 - eval { - $pid = ::open3(::gensym, ">&OUT", ">&ERR", @setpgrp_wrap, $command) || - ::die_bug("open3-gensym"); - 1; - }; + my @setpgrp_wrap = ('perl','-e',"setpgrp\;exec '$Global::shell', '-c', \@ARGV"); + # The eval is needed to catch exception from open3 + eval { + $pid = ::open3(::gensym, ">&OUT", ">&ERR", @setpgrp_wrap, $command) + || ::die_bug("open3-gens"); + 1; + }; +# eval { +# if(not $pid = ::open3(::gensym, ">&OUT", ">&ERR", "-")) { +# # Each child gets its own process group to make it safe to killall +# setpgrp(0,0); +# exec($command) || ::die_bug("open3-gensym $command"); +# } +# 1; +# }; } if($pid) { # A job was started @@ -8022,7 +8111,6 @@ sub len { } # If we are using --env, add the prefix for that, too. $len += $Global::envvarlen; - return $len; } @@ -8500,14 +8588,15 @@ sub max_length { } $Limits::Command::line_max_len = tmux_length($cached_limit); if($opt::max_chars) { - if($opt::max_chars <= $cached_limit) { - $Limits::Command::line_max_len = $opt::max_chars; + if($opt::max_chars * 2 <= $cached_limit) { + # $opt::max_chars quoting causes the length to double + $Limits::Command::line_max_len = $opt::max_chars * 2; } else { ::warning("Value for -s option should be < $cached_limit."); } } } - return $Limits::Command::line_max_len; + return int($Limits::Command::line_max_len/2); } sub real_max_length { diff --git a/src/parallel.pod b/src/parallel.pod index 784d82f1..fe710f01 100644 --- a/src/parallel.pod +++ b/src/parallel.pod @@ -1557,6 +1557,8 @@ time to initialize. B<--keep-order> will not work with B<--round-robin> as it is impossible to track which input block corresponds to which output. +B<--round-robin> implies B<--pipe>. + =item B<--rpl> 'I I' @@ -2369,6 +2371,18 @@ If B fails a red FAIL will be printed followed by the failing command; otherwise a green OK will be printed followed by the command. +=head1 EXAMPLE: Log rotate + +Log rotation renames a logfile to an extension with a higher number: +log.1 becomes log.2, log.2 becomes log.3, and so on. The oldest log is +removed. To avoid overwriting files the process starts backwards from +the high number to the low number. This will keep 10 old versions of +the log: + + seq 9 -1 1 | parallel -j1 mv log.{} log.'{= $_++ =}' + mv log log.1 + + =head1 EXAMPLE: Removing file extension when processing files When processing files removing the file extension using B<{.}> is diff --git a/testsuite/Start.sh b/testsuite/Start.sh index 2d226256..b6bc3097 100644 --- a/testsuite/Start.sh +++ b/testsuite/Start.sh @@ -39,6 +39,9 @@ export -f run_test # Create a monitor script echo forever pstree -lp $$ >/tmp/monitor chmod 755 /tmp/monitor +# log rotate +parallel -j1 mv testsuite.log.{} testsuite.log.'{= $_++ =}' ::: 10 9 8 7 6 5 4 3 2 1 +mv testsuite.log testsuite.log.1 date mkdir -p actual-results ls -t tests-to-run/*${1}*.sh | egrep -v "${2}" | @@ -51,5 +54,4 @@ else rm -rf src-passing-testsuite cp -a ../src src-passing-testsuite fi -rm testsuite.log date diff --git a/testsuite/tests-to-run/parallel-local-ssh1.sh b/testsuite/tests-to-run/parallel-local-ssh1.sh index 6d044c83..cba49f1f 100644 --- a/testsuite/tests-to-run/parallel-local-ssh1.sh +++ b/testsuite/tests-to-run/parallel-local-ssh1.sh @@ -24,7 +24,7 @@ echo '### bug #41805: Idea: propagate --env for parallel --number-of-cores' echo '** bug #41805 done' echo '### Deal with long command lines on remote servers' - perl -e 'print((("\""x10000)."\n")x10)' | parallel -j1 -S lo -N 10000 echo {} |wc + perl -e 'print((("\""x5000)."\n")x10)' | parallel -j1 -S lo -N 10000 echo {} |wc echo '### Test bug #34241: --pipe should not spawn unneeded processes' seq 5 | ssh csh@lo parallel -k --block 5 --pipe -j10 cat\\\;echo Block_end diff --git a/testsuite/tests-to-run/parallel-local-ssh4.sh b/testsuite/tests-to-run/parallel-local-ssh4.sh index 4673d6a6..5bc4abcf 100644 --- a/testsuite/tests-to-run/parallel-local-ssh4.sh +++ b/testsuite/tests-to-run/parallel-local-ssh4.sh @@ -14,16 +14,16 @@ echo '### zsh' echo '### csh' echo "3 big vars run remotely - length(base64) > 1000" - ssh csh@lo 'setenv A `seq 200|xargs`; - setenv B `seq 200 -1 1|xargs`; - setenv C `seq 300 -2 1|xargs`; - parallel -Scsh@lo --env A,B,C -k echo \$\{\}\|wc ::: A B C' + stdout ssh csh@lo 'setenv A `seq 200|xargs`; + setenv B `seq 200 -1 1|xargs`; + setenv C `seq 300 -2 1|xargs`; + parallel -Scsh@lo --env A,B,C -k echo \$\{\}\|wc ::: A B C' echo "3 big vars run locally" - ssh csh@lo 'setenv A `seq 200|xargs`; - setenv B `seq 200 -1 1|xargs`; - setenv C `seq 300 -2 1|xargs`; - parallel --env A,B,C -k echo \$\{\}\|wc ::: A B C' + stdout ssh csh@lo 'setenv A `seq 200|xargs`; + setenv B `seq 200 -1 1|xargs`; + setenv C `seq 300 -2 1|xargs`; + parallel --env A,B,C -k echo \$\{\}\|wc ::: A B C' echo '### Test tmux works on different shells' parallel -Scsh@lo,tcsh@lo,parallel@lo,zsh@lo --tmux echo ::: 1 2 3 4; echo $? diff --git a/testsuite/tests-to-run/parallel-local7.sh b/testsuite/tests-to-run/parallel-local7.sh index ffc45f32..a3a5ef5f 100755 --- a/testsuite/tests-to-run/parallel-local7.sh +++ b/testsuite/tests-to-run/parallel-local7.sh @@ -62,7 +62,7 @@ perl -e 'print map { ($_, map { pack("c*",$_) } grep { $_>=1 && $_!=10 } 0..$_), echo '### Test output ascii' rm -f /tmp/paralocal7*; - perl -e 'print map { ($_, map { pack("c*",$_) } grep { $_!=10 } 1..$_),"\n" } 1..255' | stdout parallel --tmux echo {}'>>/tmp/paralocal7{%}' | par_tmux_filter; + perl -e 'print map { ($_, map { pack("c*",$_) } grep { $_>=1 && $_!=10 } $_-10..$_),"\n" } 1..255' | stdout parallel --tmux echo {}'>>/tmp/paralocal7{%}' | par_tmux_filter; sort /tmp/paralocal7* | md5sum echo '### Test critical lengths. Must not block' diff --git a/testsuite/tests-to-run/parallel-tutorial.sh b/testsuite/tests-to-run/parallel-tutorial.sh index 393b2ed8..a2ff0992 100644 --- a/testsuite/tests-to-run/parallel-tutorial.sh +++ b/testsuite/tests-to-run/parallel-tutorial.sh @@ -5,6 +5,7 @@ cd testsuite 2>/dev/null rm -rf tmp mkdir tmp cd tmp +touch ~/.parallel/will-cite echo '### test parallel_tutorial' rm -f /tmp/runs export SERVER1=parallel@lo diff --git a/testsuite/wanted-results/parallel-local-0.3s b/testsuite/wanted-results/parallel-local-0.3s index b396e7de..226d8758 100644 --- a/testsuite/wanted-results/parallel-local-0.3s +++ b/testsuite/wanted-results/parallel-local-0.3s @@ -359,12 +359,8 @@ pdksh "#&/ pdksh ()*=?' posh "#&/ posh ()*=?' -rbash "#&/ -rbash ()*=?' rc "#&/ rc ()*=?' -rzsh "#&/ -rzsh ()*=?' sash "#&/ sash ()*=?' sh "#&/ @@ -377,6 +373,8 @@ yash "#&/ yash ()*=?' zsh "#&/ zsh ()*=?' +rbash rbash: line 0: exec: restricted +rzsh zsh:exec:1: rzsh: restricted echo '**' ** ### 1 .par file from --files expected diff --git a/testsuite/wanted-results/parallel-local-ssh1 b/testsuite/wanted-results/parallel-local-ssh1 index 5d426ab8..27388194 100644 --- a/testsuite/wanted-results/parallel-local-ssh1 +++ b/testsuite/wanted-results/parallel-local-ssh1 @@ -34,8 +34,8 @@ HOME=/home/tange ** bug #41805 done echo '### Deal with long command lines on remote servers' ### Deal with long command lines on remote servers - perl -e 'print((("\""x10000)."\n")x10)' | parallel -j1 -S lo -N 10000 echo {} |wc - 4 10 100010 + perl -e 'print((("\""x5000)."\n")x10)' | parallel -j1 -S lo -N 10000 echo {} |wc + 4 10 50010 echo '### Test bug #34241: --pipe should not spawn unneeded processes' ### Test bug #34241: --pipe should not spawn unneeded processes seq 5 | ssh csh@lo parallel -k --block 5 --pipe -j10 cat\\\;echo Block_end diff --git a/testsuite/wanted-results/parallel-local-ssh4 b/testsuite/wanted-results/parallel-local-ssh4 index 7439baa0..8e167abf 100644 --- a/testsuite/wanted-results/parallel-local-ssh4 +++ b/testsuite/wanted-results/parallel-local-ssh4 @@ -8,13 +8,13 @@ echo '### csh' ### csh echo "3 big vars run remotely - length(base64) > 1000" 3 big vars run remotely - length(base64) > 1000 - ssh csh@lo 'setenv A `seq 200|xargs`; setenv B `seq 200 -1 1|xargs`; setenv C `seq 300 -2 1|xargs`; parallel -Scsh@lo --env A,B,C -k echo \$\{\}\|wc ::: A B C' + stdout ssh csh@lo 'setenv A `seq 200|xargs`; setenv B `seq 200 -1 1|xargs`; setenv C `seq 300 -2 1|xargs`; parallel -Scsh@lo --env A,B,C -k echo \$\{\}\|wc ::: A B C' 1 200 692 1 200 692 1 150 547 echo "3 big vars run locally" 3 big vars run locally - ssh csh@lo 'setenv A `seq 200|xargs`; setenv B `seq 200 -1 1|xargs`; setenv C `seq 300 -2 1|xargs`; parallel --env A,B,C -k echo \$\{\}\|wc ::: A B C' + stdout ssh csh@lo 'setenv A `seq 200|xargs`; setenv B `seq 200 -1 1|xargs`; setenv C `seq 300 -2 1|xargs`; parallel --env A,B,C -k echo \$\{\}\|wc ::: A B C' 1 200 692 1 200 692 1 150 547 @@ -48,7 +48,7 @@ echo '### works' ### works stdout parallel -Sparallel@lo --tmux echo ::: \\\\\\\"\\\\\\\"\\\;\@ | grep -v 'See output' stdout parallel -Sparallel@lo --tmux echo ::: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx | grep -v 'See output' -parallel: Error: Command line too long (402 >= 236) at input 0: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx... +parallel: Error: Command line too long (402 >= 118) at input 0: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx... echo '### These blocked due to length' ### These blocked due to length stdout parallel -Slo --tmux echo ::: \\\\\\\"\\\\\\\"\\\;\@ | grep -v 'See output' diff --git a/testsuite/wanted-results/parallel-local114 b/testsuite/wanted-results/parallel-local114 index 04a48e30b858dc2e481aeb9e9747c9c6ce28cff6..5d64a02cfd5be7af9d3b718e5e1fa7a9f6515ef2 100644 GIT binary patch delta 134 zcmWm5u?@m75CA|?LKCG<^Cc7rzyEE&X%Qt1knWw>0tq1qqtG!0QnCSqunOY(o7$;; zzp8zgEDD!klH_QdbefTo*!m(GH3O^{yORzNxa9#Z=WB+gk3rCE5La99#1zJFb)11i hE<`9;h<(rK6}8tWHZiGV`#jXm9O<8IG~>PgG(R=eB**{& delta 134 zcmdlIwIyo9EPhvG!xT#+bCaZ0)3lUiqm)!rOA`z8= 65535) at input 0: y y y y y y y y y y y y y y y y y y y y y y y y y ... echo '### true < 16383-ys.xi' ### true < 16383-ys.xi stdout xargs true < 16383-ys.xi diff --git a/testsuite/wanted-results/parallel-local19 b/testsuite/wanted-results/parallel-local19 index b64f61dc..e048ef78 100644 --- a/testsuite/wanted-results/parallel-local19 +++ b/testsuite/wanted-results/parallel-local19 @@ -107,24 +107,24 @@ seq 1 6 | parallel -k printf '{}.gif\\n' | parallel -j1 -kX echo a{}b{.}c{.} a1.gifb1c1 a2.gifb2c2 a3.gifb3c3 a4.gifb4c4 a5.gifb5c5 a6.gifb6c6 echo '### Test -m with 60000 args'; seq 1 60000 | perl -pe 's/$/.gif/' | parallel -j1 -km echo a{}b{.}c{.} | tee >(wc) >(md5sum) >/dev/null; wait ### Test -m with 60000 args -565b18712ac24f5c46f6e64cf4548733 - - 10 179980 1286692 +0891d955e7e2659ee6a88a96f3309b5c - + 20 179960 1286702 echo '### Test -X with 60000 args'; seq 1 60000 | perl -pe 's/$/.gif/' | parallel -j1 -kX echo a{}b{.}c{.} | tee >(wc) >(md5sum) >/dev/null; wait ### Test -X with 60000 args -f09f466888e1c737eb581f41cb0087ba - - 11 60000 1346682 +af6c8c69409b5cbc55aeb95174e23c37 - + 21 60000 1346682 echo '### Test -X with 60000 args and 5 expansions' ### Test -X with 60000 args and 5 expansions seq 1 60000 | perl -pe 's/$/.gif/' | parallel -j1 -kX echo a{}b{.}c{.}{.}{} | wc -l -17 +34 seq 1 60000 | perl -pe 's/$/.gif/' | parallel -j1 -kX echo a{}b{.}c{.}{.} | wc -l -13 +25 seq 1 60000 | perl -pe 's/$/.gif/' | parallel -j1 -kX echo a{}b{.}c{.} | wc -l -11 +21 seq 1 60000 | perl -pe 's/$/.gif/' | parallel -j1 -kX echo a{}b{.}c | wc -l -9 +17 seq 1 60000 | perl -pe 's/$/.gif/' | parallel -j1 -kX echo a{}b | wc -l -6 +11 echo '### Test {.} does not repeat more than {}' ### Test {.} does not repeat more than {} seq 1 15 | perl -pe 's/$/.gif/' | parallel -j1 -s 80 -kX echo a{}b{.}c{.} @@ -146,13 +146,13 @@ a14.gif 15.gifb14 15c14 15 echo '### Test -I with shell meta chars' ### Test -I with shell meta chars seq 1 60000 | parallel -j1 -I :: -X echo a::b::c:: | wc -l -9 +17 seq 1 60000 | parallel -j1 -I '<>' -X echo 'a<>b<>c<>' | wc -l -9 +17 seq 1 60000 | parallel -j1 -I '<' -X echo 'a' -X echo 'a>b>c>' | wc -l -9 +17 echo '### Test {.}' ### Test {.} echo a | parallel -qX echo "'"{.}"' " diff --git a/testsuite/wanted-results/parallel-local22 b/testsuite/wanted-results/parallel-local22 index 6bc10e2d..449ac20b 100644 --- a/testsuite/wanted-results/parallel-local22 +++ b/testsuite/wanted-results/parallel-local22 @@ -1,16 +1,16 @@ echo '### Test of xargs -m command lines > 130k'; seq 1 60000 | parallel -m -j1 echo a{}b{}c | tee >(wc >/tmp/awc$$) >(sort | md5sum) >/tmp/a$$; wait; CHAR=$(cat /tmp/a$$ | wc -c); LINES=$(cat /tmp/a$$ | wc -l); echo "Chars per line:" $(echo "$CHAR/$LINES" | bc); cat /tmp/awc$$; rm /tmp/a$$ /tmp/awc$$ ### Test of xargs -m command lines > 130k -31d9274be5fdc2de59487cb05ba57776 - -Chars per line: 116300 - 6 119994 697800 +cb2184f70b8e9058cae1413426f8a9ae - +Chars per line: 63437 + 11 119989 697810 echo '### Test of xargs -X command lines > 130k'; seq 1 60000 | parallel -X -j1 echo a{}b{}c | tee >(wc >/tmp/bwc$$) >(sort | (sleep 1; md5sum)) >/tmp/b$$; wait; CHAR=$(cat /tmp/b$$ | wc -c); LINES=$(cat /tmp/b$$ | wc -l); echo "Chars per line:" $(echo "$CHAR/$LINES" | bc); cat /tmp/bwc$$; rm /tmp/b$$ /tmp/bwc$$ ### Test of xargs -X command lines > 130k -22074f9acada52462defb18ba912d744 - -Chars per line: 116826 - 7 60000 817788 +514f33181f346d416a1ada982b43a542 - +Chars per line: 62906 + 13 60000 817788 echo '### Test of xargs -m command lines > 130k'; seq 1 60000 | parallel -k -j1 -m echo | md5sum ### Test of xargs -m command lines > 130k -b35d8e49be8d94899b719c40d3f1f4bb - +1e763b036fe666d229520dad45610e6c - echo '### This causes problems if we kill child processes'; seq 2 40 | parallel -j 0 seq 1 10 | sort | md5sum ### This causes problems if we kill child processes 437c0d47a99b9a7c5bcb1d132f94c2e6 - @@ -191,8 +191,10 @@ which static-sh => shell path /bin/static-sh which tcsh => shell path /usr/bin/tcsh which yash => shell path /usr/bin/yash which zsh => shell path /usr/bin/zsh +/bin/rbash: line 0: exec: restricted Local configuration error occurred. Contact the systems administrator for further assistance. +zsh:exec:1: /bin/rzsh: restricted echo '## Started directly from perl' ## Started directly from perl perl -e 'system(qw(parallel -Dinit echo ::: 1))' | grep which; rm -f /tmp/par*.par diff --git a/testsuite/wanted-results/parallel-local23 b/testsuite/wanted-results/parallel-local23 index 04052b32..f4474aea 100644 --- a/testsuite/wanted-results/parallel-local23 +++ b/testsuite/wanted-results/parallel-local23 @@ -86,8 +86,9 @@ please cite as described in 'parallel --bibtex'. echo '### bug #39787: --xargs broken' ### bug #39787: --xargs broken nice perl -e 'for(1..30000){print "$_\n"}' | $NICEPAR --xargs -k echo | perl -ne 'print length $_,"\n"' -131064 -37830 +65526 +65526 +37842 echo '### --delay should grow by 3 sec per arg' ### --delay should grow by 3 sec per arg stdout /usr/bin/time -f %e parallel --delay 3 true ::: 1 2 | perl -ne '$_ >= 3 and $_ <= 8 and print "OK\n"' diff --git a/testsuite/wanted-results/parallel-local7 b/testsuite/wanted-results/parallel-local7 index 31f70302..7b86c796 100644 --- a/testsuite/wanted-results/parallel-local7 +++ b/testsuite/wanted-results/parallel-local7 @@ -31,51 +31,51 @@ See output with: tmux1.9 -S /tmp/parallel-local7/tmsXXXXX attach See output with: tmux1.9 -S /tmp/parallel-local7/tmsXXXXX attach 0 seq 1000 1100 | TMUX=tmux1.9 par_tmux -See output with: tmux1.9 -S /tmp/parallel-local7/tmsXXXXX attach -0 +parallel: Error: Command line too long (2023 >= 2023) at input 0: 1007 +255 seq 1100 1200 | TMUX=tmux1.9 par_tmux -See output with: tmux1.9 -S /tmp/parallel-local7/tmsXXXXX attach -0 +parallel: Error: Command line too long (2209 >= 2023) at input 0: 1100 +255 seq 1200 1300 | TMUX=tmux1.9 par_tmux -See output with: tmux1.9 -S /tmp/parallel-local7/tmsXXXXX attach -0 +parallel: Error: Command line too long (2409 >= 2023) at input 0: 1200 +255 seq 1300 1400 | TMUX=tmux1.9 par_tmux -See output with: tmux1.9 -S /tmp/parallel-local7/tmsXXXXX attach -0 +parallel: Error: Command line too long (2609 >= 2023) at input 0: 1300 +255 seq 1400 1500 | TMUX=tmux1.9 par_tmux -See output with: tmux1.9 -S /tmp/parallel-local7/tmsXXXXX attach -0 +parallel: Error: Command line too long (2809 >= 2023) at input 0: 1400 +255 seq 1500 1600 | TMUX=tmux1.9 par_tmux -See output with: tmux1.9 -S /tmp/parallel-local7/tmsXXXXX attach -0 +parallel: Error: Command line too long (3009 >= 2023) at input 0: 1500 +255 seq 1600 1700 | TMUX=tmux1.9 par_tmux -See output with: tmux1.9 -S /tmp/parallel-local7/tmsXXXXX attach -0 +parallel: Error: Command line too long (3209 >= 2023) at input 0: 1600 +255 seq 1700 1800 | TMUX=tmux1.9 par_tmux -See output with: tmux1.9 -S /tmp/parallel-local7/tmsXXXXX attach -0 +parallel: Error: Command line too long (3409 >= 2023) at input 0: 1700 +255 seq 1800 1900 | TMUX=tmux1.9 par_tmux -See output with: tmux1.9 -S /tmp/parallel-local7/tmsXXXXX attach -0 +parallel: Error: Command line too long (3609 >= 2023) at input 0: 1800 +255 seq 1900 2000 | TMUX=tmux1.9 par_tmux -See output with: tmux1.9 -S /tmp/parallel-local7/tmsXXXXX attach -0 +parallel: Error: Command line too long (3809 >= 2023) at input 0: 1900 +255 seq 2000 2018 | TMUX=tmux1.9 par_tmux -See output with: tmux1.9 -S /tmp/parallel-local7/tmsXXXXX attach -0 +parallel: Error: Command line too long (4009 >= 2023) at input 0: 2000 +255 echo '### tmux1.9 fails' ### tmux1.9 fails echo 2019 | TMUX=tmux1.9 par_tmux -parallel: Error: Command line too long (4047 >= 4047) at input 0: 2019 +parallel: Error: Command line too long (4047 >= 2023) at input 0: 2019 255 echo 2020 | TMUX=tmux1.9 par_tmux -parallel: Error: Command line too long (4049 >= 4047) at input 0: 2020 +parallel: Error: Command line too long (4049 >= 2023) at input 0: 2020 255 echo 2021 | TMUX=tmux1.9 par_tmux -parallel: Error: Command line too long (4051 >= 4047) at input 0: 2021 +parallel: Error: Command line too long (4051 >= 2023) at input 0: 2021 255 echo 2022 | TMUX=tmux1.9 par_tmux -parallel: Error: Command line too long (4053 >= 4047) at input 0: 2022 +parallel: Error: Command line too long (4053 >= 2023) at input 0: 2022 255 echo '### tmux1.8' ### tmux1.8 @@ -87,34 +87,36 @@ See output with: tmux1.8 -S /tmp/parallel-local7/tmsXXXXX attach 0 seq 101 150 | TMUX=tmux1.8 par_tmux See output with: tmux1.8 -S /tmp/parallel-local7/tmsXXXXX attach -0 +parallel: Error: Command line too long (236 >= 236) at input 0: 114 +255 seq 151 200 | TMUX=tmux1.8 par_tmux -See output with: tmux1.8 -S /tmp/parallel-local7/tmsXXXXX attach -0 +parallel: Error: Command line too long (310 >= 236) at input 0: 151 +255 seq 201 231 | TMUX=tmux1.8 par_tmux -See output with: tmux1.8 -S /tmp/parallel-local7/tmsXXXXX attach -0 +parallel: Error: Command line too long (410 >= 236) at input 0: 201 +255 echo '### tmux1.8 fails' ### tmux1.8 fails echo 232 | TMUX=tmux1.8 par_tmux -parallel: Error: Command line too long (472 >= 472) at input 0: 232 +parallel: Error: Command line too long (472 >= 236) at input 0: 232 255 echo 233 | TMUX=tmux1.8 par_tmux -parallel: Error: Command line too long (474 >= 472) at input 0: 233 +parallel: Error: Command line too long (474 >= 236) at input 0: 233 255 echo 234 | TMUX=tmux1.8 par_tmux -parallel: Error: Command line too long (476 >= 472) at input 0: 234 +parallel: Error: Command line too long (476 >= 236) at input 0: 234 255 echo 235 | TMUX=tmux1.8 par_tmux -parallel: Error: Command line too long (478 >= 472) at input 0: 235 +parallel: Error: Command line too long (478 >= 236) at input 0: 235 255 echo 236 | TMUX=tmux1.8 par_tmux -parallel: Error: Command line too long (480 >= 472) at input 0: 236 +parallel: Error: Command line too long (480 >= 236) at input 0: 236 255 echo '### tmux1.8 0..255 ascii' ### tmux1.8 0..255 ascii perl -e 'print map { ($_, map { pack("c*",$_) } grep { $_>=1 && $_!=10 } 0..$_),"\n" } 0..255' | TMUX=tmux1.8 stdout parallel --tmux --timeout 3 echo | par_tmux_filter; echo $? See output with: tmux1.8 -S /tmp/parallel-local7/tmsXXXXX attach +parallel: Error: Command line too long (236 >= 236) at input 0: 155  !"#$%&'()*+,-./0... 0 echo '### tmux1.9 0..255 ascii' ### tmux1.9 0..255 ascii @@ -123,16 +125,17 @@ See output with: tmux1.9 -S /tmp/parallel-local7/tmsXXXXX attach 0 echo '### Test output ascii' ### Test output ascii - rm -f /tmp/paralocal7*; perl -e 'print map { ($_, map { pack("c*",$_) } grep { $_!=10 } 1..$_),"\n" } 1..255' | stdout parallel --tmux echo {}'>>/tmp/paralocal7{%}' | par_tmux_filter; sort /tmp/paralocal7* | md5sum + rm -f /tmp/paralocal7*; perl -e 'print map { ($_, map { pack("c*",$_) } grep { $_>=1 && $_!=10 } $_-10..$_),"\n" } 1..255' | stdout parallel --tmux echo {}'>>/tmp/paralocal7{%}' | par_tmux_filter; sort /tmp/paralocal7* | md5sum See output with: tmux -S /tmp/parallel-local7/tmsXXXXX attach -a7ee232967c8aab2edf227169e8cdce0 - +053c7e8e945ef7641fc63bc309ac069d - echo '### Test critical lengths. Must not block' ### Test critical lengths. Must not block seq 70 130 | TMUX=tmux1.8 stdout parallel --tmux echo '{}{=$_="&"x$_=}' | par_tmux_filter See output with: tmux1.8 -S /tmp/parallel-local7/tmsXXXXX attach +parallel: Error: Command line too long (236 >= 236) at input 0: 114 seq 70 130 | TMUX=tmux1.9 stdout parallel --tmux echo '{}{=$_="&"x$_=}' | par_tmux_filter See output with: tmux1.9 -S /tmp/parallel-local7/tmsXXXXX attach seq 280 425 | TMUX=tmux1.8 stdout parallel --tmux echo '{}{=$_="a"x$_=}' | par_tmux_filter -See output with: tmux1.8 -S /tmp/parallel-local7/tmsXXXXX attach +parallel: Error: Command line too long (288 >= 236) at input 0: 280 seq 280 425 | TMUX=tmux1.9 stdout parallel --tmux echo '{}{=$_="a"x$_=}' | par_tmux_filter See output with: tmux1.9 -S /tmp/parallel-local7/tmsXXXXX attach diff --git a/testsuite/wanted-results/parallel-local9 b/testsuite/wanted-results/parallel-local9 index 69755a1b..c6dc6804 100644 --- a/testsuite/wanted-results/parallel-local9 +++ b/testsuite/wanted-results/parallel-local9 @@ -403,12 +403,12 @@ a1b1^c1[.}c echo "### BUG: The length for -X is not close to max (131072)" ### BUG: The length for -X is not close to max (131072) seq 1 4000 | nice parallel -k -X echo {.} aa {}{.} {}{}d{} {}dd{}d{.} |head -n 1 |wc - 1 12821 131060 + 1 6861 65500 echo "### BUG: empty lines with --show-limit" ### BUG: empty lines with --show-limit echo | $NICEPAR --show-limits Maximal size of command: 131071 -Maximal used size of command: 131071 +Maximal used size of command: 65535 Execution of will continue now, and it will try to read its input and run commands; if this is not what you wanted to happen, please diff --git a/testsuite/wanted-results/parallel-tutorial b/testsuite/wanted-results/parallel-tutorial index 71601c89..f2bb88e9 100644 --- a/testsuite/wanted-results/parallel-tutorial +++ b/testsuite/wanted-results/parallel-tutorial @@ -241,11 +241,11 @@ f1=B f2=D f1=A f2=B f1=C f2=D cat num30000 | parallel --xargs echo | wc -l -2 +3 cat num30000 | parallel --xargs -s 10000 echo | wc -l 17 cat num30000 | parallel --jobs 4 -m echo | wc -l -5 +6 parallel --jobs 4 -m echo ::: 1 2 3 4 5 6 7 8 9 10 1 2 3 4 5 6