From d8ec46d2d824190814f4c561c505334ff0634906 Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Fri, 10 Apr 2015 01:20:51 +0200 Subject: [PATCH] parallel: Kill children if receving TERM or INT. --- NEWS | 4 - doc/release_new_version | 8 +- src/parallel | 134 ++-- src/parallel_design.pod | 15 +- src/parallel_tutorial.html | 26 +- src/parallel_tutorial.pod | 29 +- testsuite/Makefile | 12 +- testsuite/Start.sh | 47 +- testsuite/tests-to-run/parallel-local-0.3s.sh | 3 + testsuite/tests-to-run/parallel-local-3s.sh | 29 +- testsuite/tests-to-run/parallel-local-ssh4.sh | 2 +- testsuite/tests-to-run/parallel-local7.sh | 6 +- testsuite/tests-to-run/parallel-tutorial.sh | 11 +- testsuite/wanted-results/parallel-local-0.3s | 2 + testsuite/wanted-results/parallel-local-3s | 20 + testsuite/wanted-results/parallel-tutorial | 580 ++++++++++-------- 16 files changed, 539 insertions(+), 389 deletions(-) diff --git a/NEWS b/NEWS index 6b2acbdf..fc67c81c 100644 --- a/NEWS +++ b/NEWS @@ -105,10 +105,6 @@ New in this release: Performance Stream Parallel Processing http://www.cs.wustl.edu/~lip/pubs/pmam15_jbeard.pdf -* GNU Parallel was cited in: Towards Collaborative Exploration and - Analysis of Big Data from Mars: A Noachis Terra Case Study - http://link.springer.com/chapter/10.1007/978-3-319-13865-7_25 - * GNU Parallel was cited in: Quantifying properties of hot and dense QCD matter through systematic model-to-data comparison http://arxiv.org/pdf/1502.00339.pdf diff --git a/doc/release_new_version b/doc/release_new_version index 4793fbc2..309bd82d 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -209,9 +209,9 @@ cc:Tim Cuthbertson , Ryoichiro Suzuki , Jesse Alama -Subject: GNU Parallel 20150422 ('') released +Subject: GNU Parallel 20150422 ('Germanwings') released -GNU Parallel 20150422 ('') has been released. It is available for download at: http://ftp.gnu.org/gnu/parallel/ +GNU Parallel 20150422 ('Germanwings') has been released. It is available for download at: http://ftp.gnu.org/gnu/parallel/ Haiku of the month: @@ -221,6 +221,8 @@ New in this release: * GNU Parallel now has a DOI: https://dx.doi.org/10.5281/zenodo.16303 +* GNU Parallel was cited in: Scaling Machine Learning for Target Prediction in Drug Discovery using Apache Spark https://cris.cumulus.vub.ac.be/portal/files/5147244/spark.pdf + * GNU Parallel was cited in: Bayesian inference of protein structure from chemical shift data https://peerj.com/articles/861/ * GNU Parallel was cited in: Toward Enhanced Metadata Quality of Large-Scale Digital Libraries: Estimating Volume Time Range https://www.ideals.illinois.edu/bitstream/handle/2142/73656/186_ready.pdf @@ -255,6 +257,8 @@ taxator-tk http://algbio.cs.uni-duesseldorf.de/webapps/wa-download/ (check it) * GNU Parallel (Sebuah Uji Coba) http://kaka.prakasa.my.id/2014/09/04/gnu-parallel-sebuah-uji-coba/ +* 你见过的最牛逼的命令行程序是什么？ http://www.zhihu.com/question/29257300 + * Bug fixes and man page updates. GNU Parallel - For people who live life in the parallel lane. diff --git a/src/parallel b/src/parallel index 84ba1f5e..a8142623 100755 --- a/src/parallel +++ b/src/parallel @@ -49,29 +49,28 @@ if($Global::max_number_of_args) { my @command = @ARGV; -my @fhlist; +my @input_source_fh; if($opt::pipepart) { - @fhlist = map { open_or_exit($_) } "/dev/null"; + @input_source_fh = map { open_or_exit($_) } "/dev/null"; } else { - @fhlist = map { open_or_exit($_) } @opt::a; - if(not @fhlist and not $opt::pipe) { - @fhlist = (*STDIN); + @input_source_fh = map { open_or_exit($_) } @opt::a; + if(not @input_source_fh and not $opt::pipe) { + @input_source_fh = (*STDIN); } } if($opt::skip_first_line) { # Skip the first line for the first file handle - my $fh = $fhlist[0]; + my $fh = $input_source_fh[0]; <$fh>; } if($opt::header and not $opt::pipe) { - my $fh = $fhlist[0]; # split with colsep or \t # $header force $colsep = \t if undef? my $delimiter = $opt::colsep; - $delimiter ||= "\$"; + $delimiter ||= "\t"; my $id = 1; - for my $fh (@fhlist) { + for my $fh (@input_source_fh) { my $line = <$fh>; chomp($line); ::debug("init", "Delimiter: '$delimiter'"); @@ -89,7 +88,7 @@ if($opt::header and not $opt::pipe) { } } else { my $id = 1; - for my $fh (@fhlist) { + for my $fh (@input_source_fh) { $Global::input_source_header{$id} = $id; $id++; } @@ -101,7 +100,7 @@ if($opt::filter_hosts and (@opt::sshlogin or @opt::sshloginfile)) { } if($opt::nonall or $opt::onall) { - onall(@command); + onall(\@input_source_fh,@command); wait_and_exit(min(undef_as_zero($Global::exitstatus),254)); } @@ -109,7 +108,7 @@ if($opt::nonall or $opt::onall) { # multiple --transfer and --basefile with different /./ $Global::JobQueue = JobQueue->new( - \@command,\@fhlist,$Global::ContextReplace,$number_of_args,\@Global::ret_files); + \@command,\@input_source_fh,$Global::ContextReplace,$number_of_args,\@Global::ret_files); if($opt::eta or $opt::bar) { # Count the number of jobs before starting any @@ -1052,7 +1051,7 @@ sub parse_options { sub init_globals { # Defaults: - $Global::version = 20150329; + $Global::version = 20150403; $Global::progname = 'parallel'; $Global::infinity = 2**31; $Global::debug = 0; @@ -1086,7 +1085,6 @@ sub init_globals { ); # Modifiable copy of %Global::replace %Global::rpl = %Global::replace; - $Global::parens = "{==}"; $/ = "\n"; $Global::ignore_empty = 0; $Global::interactive = 0; @@ -1109,6 +1107,24 @@ sub init_globals { sub parse_replacement_string_options { # Deal with --rpl + # Uses: + # %Global::rpl + # $Global::parensleft + # $Global::parensright + # $opt::parens + # $Global::parensleft + # $Global::parensright + # $opt::plus + # %Global::plus + # $opt::I + # $opt::U + # $opt::i + # $opt::basenamereplace + # $opt::dirnamereplace + # $opt::seqreplace + # $opt::slotreplace + # $opt::basenameextensionreplace + sub rpl { # Modify %Global::rpl # Replace $old with $new @@ -1118,10 +1134,11 @@ sub parse_replacement_string_options { delete $Global::rpl{$old}; } } - if(defined $opt::parens) { $Global::parens = $opt::parens; } - my $parenslen = 0.5*length $Global::parens; - $Global::parensleft = substr($Global::parens,0,$parenslen); - $Global::parensright = substr($Global::parens,$parenslen); + my $parens = "{==}"; + if(defined $opt::parens) { $parens = $opt::parens; } + my $parenslen = 0.5*length $parens; + $Global::parensleft = substr($parens,0,$parenslen); + $Global::parensright = substr($parens,$parenslen); if(defined $opt::plus) { %Global::rpl = (%Global::plus,%Global::rpl); } if(defined $opt::I) { rpl('{}',$opt::I); } if(defined $opt::U) { rpl('{.}',$opt::U); } @@ -1145,6 +1162,23 @@ sub parse_semaphore { # Semaphore defaults # Must be done before computing number of processes and max_line_length # because when running as a semaphore GNU Parallel does not read args + # Uses: + # $opt::semaphore + # $Global::semaphore + # $opt::semaphoretimeout + # $Semaphore::timeout + # $opt::semaphorename + # $Semaphore::name + # $opt::fg + # $Semaphore::fg + # $opt::wait + # $Semaphore::wait + # $opt::bg + # @opt::a + # @Global::unget_argv + # $Global::default_simultaneous_sshlogins + # $opt::jobs + # $Global::interactive $Global::semaphore ||= ($0 =~ m:(^|/)sem$:); # called as 'sem' if(defined $opt::semaphore) { $Global::semaphore = 1; } if(defined $opt::semaphoretimeout) { $Global::semaphore = 1; } @@ -1177,18 +1211,6 @@ sub parse_semaphore { } } -sub env_quote { - # Input: - # $v = value to quote - # Returns: - # $v = value quoted as environment variable - my $v = $_[0]; - $v =~ s/([\\])/\\$1/g; - $v =~ s/([\[\] \#\'\&\<\>\(\)\;\{\}\t\"\$\`\*\174\!\?\~])/\\$1/g; - $v =~ s/\n/"\n"/g; - return $v; -} - sub record_env { # Record current %ENV-keys in ~/.parallel/ignored_vars # Returns: N/A @@ -1495,8 +1517,8 @@ sub read_args_from_command_line { sub cleanup { # Returns: N/A - if(@opt::basefile) { cleanup_basefile(); } unlink keys %Global::unlink; + if(@opt::basefile) { cleanup_basefile(); } } sub __QUOTING_ARGUMENTS_FOR_SHELL__ {} @@ -2798,14 +2820,14 @@ sub onall { close $fh; return $tmpfile; } - my @command = @_; + my ($input_source_fh_ref,@command) = @_; if($Global::quoting) { @command = shell_quote_empty(@command); } - # Copy all @fhlist (-a and :::) into tempfiles + # Copy all @input_source_fh (-a and :::) into tempfiles my @argfiles = (); - for my $fh (@fhlist) { + for my $fh (@$input_source_fh_ref) { my ($outfh, $name) = ::tmpfile(SUFFIX => ".all", UNLINK => 1); print $outfh (<$fh>); close $outfh; @@ -2893,10 +2915,14 @@ sub save_original_signal_handler { # Uses: # %Global::original_sig # Returns: N/A - $SIG{INT} = sub { if($opt::tmux) { qx { tmux kill-session -t p$$ }; } - unlink keys %Global::unlink; exit -1 }; - $SIG{TERM} = sub { if($opt::tmux) { qx { tmux kill-session -t p$$ }; } - unlink keys %Global::unlink; exit -1 }; + $SIG{INT} = sub { + if($opt::tmux) { qx { tmux kill-session -t p$$ }; } + wait_and_exit(255); + }; + $SIG{TERM} = sub { + if($opt::tmux) { qx { tmux kill-session -t p$$ }; } + wait_and_exit(255); + }; %Global::original_sig = %SIG; $SIG{TERM} = sub {}; # Dummy until jobs really start $SIG{ALRM} = 'IGNORE'; @@ -3009,11 +3035,11 @@ sub wait_and_exit { # If we do not wait, we sometimes get segfault # Returns: N/A my $error = shift; + unlink keys %Global::unlink; if($error) { # Kill all without printing for my $job (values %Global::running) { - $job->kill("TERM"); - $job->kill("TERM"); + $job->kill(); } } for (keys %Global::unkilled_children) { @@ -3193,10 +3219,16 @@ sub bibtex { if(open (my $fh, ">", $ENV{'HOME'}."/.parallel/will-cite")) { close $fh; print "\nThank you for your support. It is much appreciated. The citation\n", - "notice is now silenced. You may also use '--will-cite'.\n"; + "notice is now silenced. You may also use '--will-cite'.\n", + "If you use '--will-cite' in scripts you are expected to pay\n", + "the 10000 EUR, because you are making it harder to see the\n", + "citation notice.\n\n"; } else { print "\nThank you for your support. It is much appreciated. The citation\n", - "cannot permanently be disabled. Use --will-cite instead.\n"; + "cannot permanently be silenced. Use '--will-cite' instead.\n", + "If you use '--will-cite' in scripts you are expected to pay\n", + "the 10000 EUR, because you are making it harder to see the\n", + "citation notice.\n\n"; last; } } @@ -5637,12 +5669,13 @@ sub empty_input_detector { # If some input: Pass input as input to pipe # This avoids starting the $read command if there is no input. # Returns: - # $cmd = script to prepend to '($real command)' + # $cmd = script to prepend to '| ($real command)' # The $tmpfile might exist if run on a remote system - we accept that risk my ($dummy_fh, $tmpfile) = ::tmpfile(SUFFIX => ".chr"); # Unlink to avoid leaving files if --dry-run or --sshlogin unlink $tmpfile; + $Global::unlink{$tmpfile} = 1; my $cmd = # Exit value: # empty input = true @@ -5650,7 +5683,7 @@ sub empty_input_detector { # sh -c needed as csh cannot hide stderr qq{ sh -c 'dd bs=1 count=1 of=$tmpfile 2>/dev/null'; }. qq{ test \! -s "$tmpfile" && rm -f "$tmpfile" && exec true; }. - qq{ (cat $tmpfile; rm $tmpfile; cat - ) | }; + qq{ (cat $tmpfile; rm $tmpfile; cat - ) }; return $cmd; } @@ -5663,7 +5696,7 @@ sub filter_through_compress { for my $fdno (1,2) { my $wpid = open(my $fdw,"|-", empty_input_detector(). - "($opt::compress_program) >>". + "| ($opt::compress_program) >>". $self->fh($fdno,'name')) || die $?; $self->set_fh($fdno,'w',$fdw); $self->set_fh($fdno,'wpid',$wpid); @@ -5890,7 +5923,7 @@ sub kill { # Record this jobs as failed $self->set_exitstatus(-1); # Send two TERMs to give time to clean up - ::debug("run", "Kill seq ", $self->seq(), "\n"); + ::debug("run", "Kill seq ", $self->seq(), " signal '@signals'\n"); my @send_signals = @signals || ("TERM", "TERM", "KILL"); for my $signal (@send_signals) { my $alive = 0; @@ -5899,6 +5932,7 @@ sub kill { # The job still running kill $signal, $pid; $alive = 1; + ::debug("run","$pid is alive\n"); } } # If a signal was given as input, do not do the sleep below @@ -6119,7 +6153,7 @@ sub wrapped { $command = (shift @Global::cat_partials). " | ($command)"; } elsif($opt::pipe) { # Prepend EOF-detector to avoid starting $command if EOF. - $command = empty_input_detector(). "($command);"; + $command = empty_input_detector(). "| ($command);"; } if($opt::tmux) { # Wrap command with 'tmux' @@ -6761,7 +6795,7 @@ sub print_dryrun_and_verbose { my $actual_command = shift; # Temporary file name. Used for fifo to communicate exit val my ($fh, $tmpfile) = ::tmpfile(SUFFIX => ".tmx"); - $Global::unlink{$tmpfile}=1; + $Global::unlink{$tmpfile} = 1; close $fh; unlink $tmpfile; my $visual_command = $self->replaced(); @@ -8213,11 +8247,11 @@ sub tmux_length { for my $l (1, 2020, 16320, 100000, $len) { my ($fh, $tmpfile) = ::tmpfile(SUFFIX => ".tmb"); close $fh; - $Global::unlink{$tmpfile} = 1; unlink $tmpfile; my $tmuxcmd = "sh -c '".$ENV{'TMUX'}." -S $tmpfile new-session -d -n echo $l". - ("x"x$l). " 2>/dev/null' && echo $l"; + ("x"x$l). " 2>/dev/null' && echo $l; rm -f $tmpfile"; push @out, qx{ $tmuxcmd }; + unlink $tmpfile; } ::debug("tmux","tmux-length ",@out); chomp @out; diff --git a/src/parallel_design.pod b/src/parallel_design.pod index 7d451f0f..2f8a4b6c 100644 --- a/src/parallel_design.pod +++ b/src/parallel_design.pod @@ -32,7 +32,8 @@ is CentOS 3.9 and Perl 5.8.0. GNU B busy waits. This is because the reason why a job is not started may be due to load average, and thus it will not make sense to wait for a job to finish. Instead the load average must be -checked again. Load average is not the only reason. +checked again. Load average is not the only reason: --timeout has a +similar problem. To not burn up too up too much CPU GNU B sleeps exponentially longer and longer if nothing happens, maxing out at 1 @@ -282,17 +283,17 @@ too. Take B<--nice> as an example: will work just fine. But when run remotely, you need to move the nice command so it is being run on the server: - parallel --S server nice command ... + parallel -S server nice command ... And this will again work just fine, as long as you are running a single command. When you are running a composed command you need nice to apply to the whole command, and it gets harder still: - parallel --S server --q nice bash -c 'command1 ...; command2 | command3' + parallel -S server -q nice bash -c 'command1 ...; command2 | command3' It is not impossible, but by using B<--nice> GNU B will do the right thing for you. Similarly when transferring files: It starts -to get hard when the file names contain space, :, ‘, *, or other +to get hard when the file names contain space, :, `, *, or other special characters. To run the commands in a B session you basically just need to @@ -324,7 +325,7 @@ post-shellshock versions of B. =head2 Remote Ctrl-C and standard error (stderr) -If the user presses Ctrl-C the user expect jobs to stop. This works +If the user presses Ctrl-C the user expects jobs to stop. This works out of the box if the jobs are run locally. Unfortunately it is not so simple if the jobs are run remotely. @@ -366,7 +367,7 @@ stderr. The wrapper looks like this: =head2 Transferring of variables and functions -Transferring of variables and functions given by B<-env> is done by +Transferring of variables and functions given by B<--env> is done by running a Perl script remotely that calls the actual command. The Perl script sets $ENV{variable} to the correct value before exec'ing the a shell that runs the function definition followed by the actual @@ -378,7 +379,7 @@ B. This variable is picked up by GNU B and used to create the Perl script mentioned above. -=head2 Base64 encode bzip2 +=head2 Base64 encoded bzip2 B limits words of commands to 1024 chars. This is often too little when GNU B encodes environment variables and wraps the diff --git a/src/parallel_tutorial.html b/src/parallel_tutorial.html index f047cb25..a70f8698 100644 --- a/src/parallel_tutorial.html +++ b/src/parallel_tutorial.html @@ -553,27 +553,27 @@

When predefined replacement strings are not flexible enough a perl expression can be used instead. One example is to remove two extensions: foo.tar.gz -> foo

-
   parallel echo '{= s:\.[^.]+$::;s:\.[^.]+$::; =}' ::: foo.tar.gz
+
  parallel echo '{= s:\.[^.]+$::;s:\.[^.]+$::; =}' ::: foo.tar.gz

Output:

-
   foo
+
  foo

If the strings {= and =} cause problems they can be replaced with --parens:

-
   parallel --parens ,,,, echo ',, s:\.[^.]+$::;s:\.[^.]+$::; ,,' ::: foo.tar.gz
+
  parallel --parens ,,,, echo ',, s:\.[^.]+$::;s:\.[^.]+$::; ,,' ::: foo.tar.gz

Output: Same as above.

To define a short hand replacement string use --rpl:

-
   parallel --rpl '.. s:\.[^.]+$::;s:\.[^.]+$::;' echo '..' ::: foo.tar.gz
+
  parallel --rpl '.. s:\.[^.]+$::;s:\.[^.]+$::;' echo '..' ::: foo.tar.gz

Output: Same as above.

If the short hand starts with '{' it can be used as a positional replacement string, too:

-
   parallel --rpl '{..} s:\.[^.]+$::;s:\.[^.]+$::;' echo '{..}' ::: foo.tar.gz
+
  parallel --rpl '{..} s:\.[^.]+$::;s:\.[^.]+$::;' echo '{..}' ::: foo.tar.gz

Output: Same as above.

@@ -628,15 +628,15 @@

To use a perl expression as a positional replacement string simply prepend the perl expression with number and space:

-
   parallel echo '{=2 s:\.[^.]+$::;s:\.[^.]+$::; =} {1}' ::: bar ::: foo.tar.gz
+
  parallel echo '{=2 s:\.[^.]+$::;s:\.[^.]+$::; =} {1}' ::: bar ::: foo.tar.gz

Output:

-
   foo bar
+
  foo bar

If a defined short hand starts with '{' it can be used as a positional replacement string, too:

-
   parallel --rpl '{..} s:\.[^.]+$::;s:\.[^.]+$::;' echo '{2..} {1}' ::: bar ::: foo.tar.gz
+
  parallel --rpl '{..} s:\.[^.]+$::;s:\.[^.]+$::;' echo '{2..} {1}' ::: bar ::: foo.tar.gz

Output: Same as above.

@@ -918,7 +918,7 @@

To get the output immediately use --ungroup:

-
  parallel -j2 --ungroup 'printf "%s-start\n%s" {} {};sleep {};printf "%s\n" -middle;echo {}-end' ::: 4 2 1
+
  parallel -j2 --ungroup 'printf "%s-start\n%s" {} {};sleep {};printf "%s\n" -middle;echo {}-end' ::: 4 2 1

Output:

@@ -1117,14 +1117,14 @@ Starting 3 Thu Aug 15 16:24:38 CEST 2013 -

If jobs taking more than a certain amount of time are known to fail, they can be stopped with --timeout:

+

If jobs taking more than a certain amount of time are known to fail, they can be stopped with --timeout. The accuracy of --timeout is 2 seconds:

-
  parallel --timeout 2.1 sleep {}\; echo {} ::: 1 2 3 4
+
  parallel --timeout 4.1 sleep {}\; echo {} ::: 2 4 6 8

Output:

-
  1
-  2
+
  2
+  4

GNU Parallel can compute the median runtime for jobs and kill those that take more than 200% of the median runtime:

diff --git a/src/parallel_tutorial.pod b/src/parallel_tutorial.pod index 197d5abc..83796be6 100644 --- a/src/parallel_tutorial.pod +++ b/src/parallel_tutorial.pod @@ -452,28 +452,28 @@ When predefined replacement strings are not flexible enough a perl expression can be used instead. One example is to remove two extensions: foo.tar.gz -> foo - parallel echo '{= s:\.[^.]+$::;s:\.[^.]+$::; =}' ::: foo.tar.gz + parallel echo '{= s:\.[^.]+$::;s:\.[^.]+$::; =}' ::: foo.tar.gz Output: - foo + foo If the strings B<{=> and B<=}> cause problems they can be replaced with --parens: - parallel --parens ,,,, echo ',, s:\.[^.]+$::;s:\.[^.]+$::; ,,' ::: foo.tar.gz + parallel --parens ,,,, echo ',, s:\.[^.]+$::;s:\.[^.]+$::; ,,' ::: foo.tar.gz Output: Same as above. To define a short hand replacement string use B<--rpl>: - parallel --rpl '.. s:\.[^.]+$::;s:\.[^.]+$::;' echo '..' ::: foo.tar.gz + parallel --rpl '.. s:\.[^.]+$::;s:\.[^.]+$::;' echo '..' ::: foo.tar.gz Output: Same as above. If the short hand starts with '{' it can be used as a positional replacement string, too: - parallel --rpl '{..} s:\.[^.]+$::;s:\.[^.]+$::;' echo '{..}' ::: foo.tar.gz + parallel --rpl '{..} s:\.[^.]+$::;s:\.[^.]+$::;' echo '{..}' ::: foo.tar.gz Output: Same as above. @@ -532,16 +532,16 @@ Output (the order may be different): To use a perl expression as a positional replacement string simply prepend the perl expression with number and space: - parallel echo '{=2 s:\.[^.]+$::;s:\.[^.]+$::; =} {1}' ::: bar ::: foo.tar.gz + parallel echo '{=2 s:\.[^.]+$::;s:\.[^.]+$::; =} {1}' ::: bar ::: foo.tar.gz Output: - foo bar + foo bar If a defined short hand starts with '{' it can be used as a positional replacement string, too: - parallel --rpl '{..} s:\.[^.]+$::;s:\.[^.]+$::;' echo '{2..} {1}' ::: bar ::: foo.tar.gz + parallel --rpl '{..} s:\.[^.]+$::;s:\.[^.]+$::;' echo '{2..} {1}' ::: bar ::: foo.tar.gz Output: Same as above. @@ -838,10 +838,10 @@ Output: To get the output immediately use --ungroup: - parallel -j2 --ungroup 'printf "%s-start\n%s" {} {};sleep {};printf "%s\n" -middle;echo {}-end' ::: 4 2 1 + parallel -j2 --ungroup 'printf "%s-start\n%s" {} {};sleep {};printf "%s\n" -middle;echo {}-end' ::: 4 2 1 Output: - + 4-start 42-start 2-middle @@ -1052,14 +1052,15 @@ Output: If jobs taking more than a certain amount of time are known to fail, -they can be stopped with --timeout: +they can be stopped with --timeout. The accuracy of --timeout is 2 +seconds: - parallel --timeout 2.1 sleep {}\; echo {} ::: 1 2 3 4 + parallel --timeout 4.1 sleep {}\; echo {} ::: 2 4 6 8 Output: - 1 2 + 4 GNU Parallel can compute the median runtime for jobs and kill those that take more than 200% of the median runtime: @@ -1484,7 +1485,7 @@ arguments. --nonall is a no argument --onall: parallel --nonall -S $SERVER1,$SERVER2 echo foo bar Output: - + foo bar foo bar diff --git a/testsuite/Makefile b/testsuite/Makefile index 7f3aba20..8308af20 100644 --- a/testsuite/Makefile +++ b/testsuite/Makefile @@ -2,28 +2,28 @@ testsuite: 3 true 3: ../src/parallel tests-to-run/* wanted-results/* startdb prereqlocal prereqremote - TRIES=3 time sh Start.sh - mem || true + TRIES=3 time bash Start.sh - mem || true touch ~/.parallel/will-cite make stopvm 1: ../src/parallel tests-to-run/* wanted-results/* prereqlocal startdb prereqremote - time sh Start.sh - mem || true + time bash Start.sh - mem || true touch ~/.parallel/will-cite make stopvm mem: ../src/parallel tests-to-run/*mem* wanted-results/*mem* prereqlocal - time sh Start.sh mem NONE || true + time bash Start.sh mem NONE || true touch ~/.parallel/will-cite make stopvm testdb: ../src/parallel tests-to-run/*sql* wanted-results/*sql* prereqdb - time sh Start.sh sql NONE + time bash Start.sh sql NONE local: testlocal true testlocal: ../src/parallel tests-to-run/*local* wanted-results/*local* prereqlocal installparallel - time sh Start.sh local NONE + time bash Start.sh local NONE prereqlocal: installparallel tcsh -c echo tcsh installed || (echo tcsh is required for testsuite; /bin/false) @@ -40,7 +40,7 @@ prereqlocal: installparallel which timeout || (echo timeout is required for testsuite; /bin/false) prereqremote: installparallel startvm - parallel --tag ssh parallel@parallel-server{} parallel --minversion 20121021 ::: 1 2 3 || (echo parallel on remote required for testsuite; /bin/true) + parallel --timeout 5 --tag ssh parallel@parallel-server{} parallel --minversion 20121021 ::: 1 2 3 || (echo parallel on remote required for testsuite; /bin/true) startvm: # Make sure we can reach the virtual machines diff --git a/testsuite/Start.sh b/testsuite/Start.sh index fe7dceb5..15f2ef70 100644 --- a/testsuite/Start.sh +++ b/testsuite/Start.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/bash -x # Argument can be substring of tests (such as 'local') @@ -7,26 +7,37 @@ SHFILE=/tmp/unittest-parallel.sh MAX_SEC_PER_TEST=900 export TIMEOUT=$MAX_SEC_PER_TEST -if [ "$TRIES" = "3" ] ; then - # Try a failing test thrice - echo Retrying 3 times - ls -t tests-to-run/*${1}*.sh | - grep -v ${2} | - perl -pe 's:(.*/(.*)).sh:bash $1.sh > actual-results/$2; diff -Naur wanted-results/$2 actual-results/$2 >/dev/null || bash $1.sh > actual-results/$2; diff -Naur wanted-results/$2 actual-results/$2 >/dev/null || bash $1.sh > actual-results/$2; diff -Naur wanted-results/$2 actual-results/$2 || touch $1.sh: ' \ - >$SHFILE -else - # Run a failing test once - echo Not retrying - ls -t tests-to-run/*${1}*.sh | - grep -v ${2} | - perl -pe 's:(.*/(.*)).sh:bash $1.sh > actual-results/$2; diff -Naur wanted-results/$2 actual-results/$2 || touch $1.sh:' \ - >$SHFILE -fi +run_test() { + script="$1" + base=`basename "$script" .sh` + if [ "$TRIES" = "3" ] ; then + # Try 3 times + bash $script > actual-results/$base + diff -Naur wanted-results/$base actual-results/$base >/dev/null || + bash $script > actual-results/$base + diff -Naur wanted-results/$base actual-results/$base >/dev/null || + bash $script > actual-results/$base + diff -Naur wanted-results/$base actual-results/$base || + (touch $script && echo touch $script) + else + # Run only once + bash $script > actual-results/$base + diff -Naur wanted-results/$base actual-results/$base || + (touch $script && echo touch $script) + fi + + # Check if it was cleaned up + find /tmp -maxdepth 1 | + perl -ne '/\.(tmb|chr|tms|par)$/ and ++$a and print "TMP NOT CLEAN. FOUND: $_".`touch '$script'`;' + # May be owned by other users + sudo rm -f /tmp/*.{tmb,chr,tms,par} +} +export -f run_test date mkdir -p actual-results -stdout sh -x $SHFILE | tee testsuite.log -rm $SHFILE +ls -t tests-to-run/*${1}*.sh | grep -v ${2} | + stdout parallel --tty -tj1 run_test | tee testsuite.log # If testsuite.log contains @@ then there is a diff if grep -q '@@' testsuite.log ; then false diff --git a/testsuite/tests-to-run/parallel-local-0.3s.sh b/testsuite/tests-to-run/parallel-local-0.3s.sh index 1c3007d5..60f0b83a 100644 --- a/testsuite/tests-to-run/parallel-local-0.3s.sh +++ b/testsuite/tests-to-run/parallel-local-0.3s.sh @@ -94,3 +94,6 @@ echo '### TMUX not found' TMUX=not-existing parallel --tmux echo ::: 1 EOF +echo '### 1 .par file from --files expected' +ls /tmp/par*.par /var/tmp/par*.par /tmp/*.tms /tmp/*.tmx 2>/dev/null | wc -l +find /tmp/par*.par /var/tmp/par*.par /tmp/*.tms /tmp/*.tmx -mmin -10 2>/dev/null | parallel rm diff --git a/testsuite/tests-to-run/parallel-local-3s.sh b/testsuite/tests-to-run/parallel-local-3s.sh index f3862a27..e0e1013e 100644 --- a/testsuite/tests-to-run/parallel-local-3s.sh +++ b/testsuite/tests-to-run/parallel-local-3s.sh @@ -3,7 +3,7 @@ # Simple jobs that never fails # Each should be taking 3-10s and be possible to run in parallel # I.e.: No race conditions, no logins -cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj4 -k --joblog /tmp/jl-`basename $0` -L1 +cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj0 -k --joblog /tmp/jl-`basename $0` -L1 echo '### bug #42089: --results with arg > 256 chars (should be 1 char shorter)' parallel --results parallel_test_dir echo ::: 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456; ls parallel_test_dir/1/ @@ -72,4 +72,31 @@ echo '**' echo '### Test slow arguments generation - https://savannah.gnu.org/bugs/?32834'; seq 1 3 | parallel -j1 "sleep 2; echo {}" | parallel -kj2 echo +echo '**' + +echo '### Are children killed if GNU Parallel receives TERM twice? There should be no sleep at the end' + + parallel -q bash -c 'sleep 120 & pid=$!; wait $pid' ::: 1 & + T=$!; + sleep 1; + pstree $$; + kill -TERM $T; + sleep 1; + pstree $$; + kill -TERM $T; + sleep 1; + pstree $$; + +echo '**' + +echo '### Are children killed if GNU Parallel receives INT twice? There should be no sleep at the end' + + parallel -q bash -c 'sleep 120 & pid=$!; wait $pid' ::: 1 & + T=$!; + sleep 1; + pstree $$; + kill -INT $T; + sleep 1; + pstree $$; + EOF diff --git a/testsuite/tests-to-run/parallel-local-ssh4.sh b/testsuite/tests-to-run/parallel-local-ssh4.sh index ea33869d..cf42faf2 100644 --- a/testsuite/tests-to-run/parallel-local-ssh4.sh +++ b/testsuite/tests-to-run/parallel-local-ssh4.sh @@ -2,7 +2,7 @@ # SSH only allowed to localhost/lo # --retries if ssh dies -cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | parallel -vj7 --retries 2 -k --joblog /tmp/jl-`basename $0` -L1 +cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | parallel -vj1 --retries 2 -k --joblog /tmp/jl-`basename $0` -L1 echo '### zsh' ssh zsh@lo 'fun="() { echo function from zsh to zsh \$*; }"; export fun; diff --git a/testsuite/tests-to-run/parallel-local7.sh b/testsuite/tests-to-run/parallel-local7.sh index 17fd5fc8..1ba72e8d 100644 --- a/testsuite/tests-to-run/parallel-local7.sh +++ b/testsuite/tests-to-run/parallel-local7.sh @@ -9,8 +9,8 @@ par_tmux() { (stdout parallel --timeout 3 --tmux --delay .3 echo '{}{=$_="\\"x$_=}'; echo $?) | par_tmux_filter } export -f par_tmux - -cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj8 --retries 2 -k --joblog /tmp/jl-`basename $0` -L1 +#j8 +cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj0 --retries 2 -k --joblog /tmp/jl-`basename $0` -L1 echo '### tmux1.9' seq 000 100 | TMUX=tmux1.9 par_tmux seq 100 200 | TMUX=tmux1.9 par_tmux @@ -67,3 +67,5 @@ echo '### Test critical lengths. Must not block' seq 280 425 | TMUX=tmux1.9 stdout parallel --tmux echo '{}{=$_="a"x$_=}' | par_tmux_filter EOF + +rm -f /tmp/paralocal7*; \ No newline at end of file diff --git a/testsuite/tests-to-run/parallel-tutorial.sh b/testsuite/tests-to-run/parallel-tutorial.sh index af0fff34..48d991e5 100644 --- a/testsuite/tests-to-run/parallel-tutorial.sh +++ b/testsuite/tests-to-run/parallel-tutorial.sh @@ -1,7 +1,8 @@ #!/bin/bash cd testsuite 2>/dev/null -mkdir -p tmp +rm -rf tmp +mkdir tmp cd tmp echo '### test parallel_tutorial' rm -f /tmp/runs @@ -14,8 +15,10 @@ perl -ne '$/="\n\n"; /^Output/../^[^O]\S/ and next; /^ / and print;' ../../src/ s/zenity/zenity --timeout=12/; s:/usr/bin/time:/usr/bin/time -f %e:; s:ignored_vars:ignored_vars|sort:; + # When parallelized: Sleep to make sure the abc-files are made + /%head1/ and $_.="sleep .3\n\n"x10; ' | - stdout bash -x | + stdout parallel -j7 -vd'\n\n' | perl -pe '$|=1; # --files and --tmux s:/tmp/par......(...):/tmp/parXXXXX.$1:; @@ -42,4 +45,6 @@ perl -ne '$/="\n\n"; /^Output/../^[^O]\S/ and next; /^ / and print;' ../../src/ # + command_X | (Bash outputs these in random order) s/.*command_[ABC].*\n//; ' -# parallel -d'\n\n' +# 3+3 .par files (from --files), 1 .tms-file from tmux attach +ls /tmp/par*.par /var/tmp/par*.par /tmp/*.tms /tmp/*.tmx 2>/dev/null | wc -l +find /tmp/par*.par /var/tmp/par*.par /tmp/*.tms /tmp/*.tmx -mmin -10 2>/dev/null | parallel rm diff --git a/testsuite/wanted-results/parallel-local-0.3s b/testsuite/wanted-results/parallel-local-0.3s index 2134edd5..a318ee1b 100644 --- a/testsuite/wanted-results/parallel-local-0.3s +++ b/testsuite/wanted-results/parallel-local-0.3s @@ -161,3 +161,5 @@ echo '### TMUX not found' ### TMUX not found TMUX=not-existing parallel --tmux echo ::: 1 parallel: Error: not-existing not found in $PATH. +### 1 .par file from --files expected +1 diff --git a/testsuite/wanted-results/parallel-local-3s b/testsuite/wanted-results/parallel-local-3s index f42a26c3..02242a1f 100644 --- a/testsuite/wanted-results/parallel-local-3s +++ b/testsuite/wanted-results/parallel-local-3s @@ -142,3 +142,23 @@ echo '### Test slow arguments generation - https://savannah.gnu.org/bugs/?32834' 1 2 3 +echo '**' +** +echo '### Are children killed if GNU Parallel receives TERM twice? There should be no sleep at the end' +### Are children killed if GNU Parallel receives TERM twice? There should be no sleep at the end + parallel -q bash -c 'sleep 120 & pid=$!; wait $pid' ::: 1 & T=$!; sleep 1; pstree $$; kill -TERM $T; sleep 1; pstree $$; kill -TERM $T; sleep 1; pstree $$; echo '**' +bash-+-perl---bash---sleep + `-pstree +bash-+-perl---bash---sleep + `-pstree +bash---pstree +** +parallel: SIGTERM received. No new jobs will be started. +parallel: Waiting for these 1 jobs to finish. Send SIGTERM again to stop now. +parallel: bash -c sleep\ 120\ \&\ pid=\$\!\;\ wait\ \$pid 1 +echo '### Are children killed if GNU Parallel receives INT twice? There should be no sleep at the end' +### Are children killed if GNU Parallel receives INT twice? There should be no sleep at the end + parallel -q bash -c 'sleep 120 & pid=$!; wait $pid' ::: 1 & T=$!; sleep 1; pstree $$; kill -INT $T; sleep 1; pstree $$; +bash-+-perl---bash---sleep + `-pstree +bash---pstree diff --git a/testsuite/wanted-results/parallel-tutorial b/testsuite/wanted-results/parallel-tutorial index 68936fe5..9d51f862 100644 --- a/testsuite/wanted-results/parallel-tutorial +++ b/testsuite/wanted-results/parallel-tutorial @@ -1,27 +1,38 @@ ### test parallel_tutorial -+ parallel -k echo ::: A B C -+ parallel -k echo ::: D E F -+ perl -e 'printf "A\0B\0C\0"' -+ perl -e 'printf "A_B_C_"' -+ perl -e 'printf "f1\tf2\nA\tB\nC\tD\n"' -+ perl -e 'for(1..8){print "$_\n"}' -+ perl -e 'for(1..128){print "$_\n"}' -+ perl -e 'for(1..30000){print "$_\n"}' -+ perl -e 'for(1..1000000){print "$_\n"}' -+ perl -e 'for(1..10){print "$_\n"}' -+ parallel echo ::: A B C + parallel -k echo ::: A B C > abc-file + parallel -k echo ::: D E F > def-file + perl -e 'printf "A\0B\0C\0"' > abc0-file + perl -e 'printf "A_B_C_"' > abc_-file + perl -e 'printf "f1\tf2\nA\tB\nC\tD\n"' > tsv-file.tsv + perl -e 'for(1..8){print "$_\n"}' > num8 + perl -e 'for(1..128){print "$_\n"}' > num128 + perl -e 'for(1..30000){print "$_\n"}' > num30000 + perl -e 'for(1..1000000){print "$_\n"}' > num1000000 + (echo %head1; echo %head2; perl -e 'for(1..10){print "$_\n"}') > num_%header +sleep .3 +sleep .3 +sleep .3 +sleep .3 +sleep .3 +sleep .3 +sleep .3 +sleep .3 +sleep .3 +sleep .3 + + parallel echo ::: A B C A B C -+ parallel -a abc-file echo + parallel -a abc-file echo A B C -+ parallel echo + cat abc-file | parallel echo A B C -+ parallel echo ::: A B C ::: D E F + parallel echo ::: A B C ::: D E F A D A E A F @@ -31,7 +42,7 @@ B F C D C E C F -+ parallel -a abc-file -a def-file echo + parallel -a abc-file -a def-file echo A D A E A F @@ -41,7 +52,7 @@ B F C D C E C F -+ parallel -a - -a def-file echo + cat abc-file | parallel -a - -a def-file echo A D A E A F @@ -51,7 +62,7 @@ B F C D C E C F -+ parallel echo :::: - def-file + cat abc-file | parallel echo :::: - def-file A D A E A F @@ -61,7 +72,7 @@ B F C D C E C F -+ parallel echo ::: A B C :::: def-file + parallel echo ::: A B C :::: def-file A D A E A F @@ -71,17 +82,17 @@ B F C D C E C F -+ parallel --xapply echo ::: A B C ::: D E F + parallel --xapply echo ::: A B C ::: D E F A D B E C F -+ parallel --xapply echo ::: A B C D E ::: F G + parallel --xapply echo ::: A B C D E ::: F G A F B G C F D G E F -+ parallel --arg-sep ,, echo ,, A B C :::: def-file + parallel --arg-sep ,, echo ,, A B C :::: def-file A D A E A F @@ -91,7 +102,7 @@ B F C D C E C F -+ parallel --arg-file-sep // echo ::: A B C // def-file + parallel --arg-file-sep // echo ::: A B C // def-file A D A E A F @@ -101,32 +112,24 @@ B F C D C E C F -+ parallel -d _ echo :::: abc_-file + parallel -d _ echo :::: abc_-file A B C -+ parallel -d '\0' echo :::: abc0-file + parallel -d '\0' echo :::: abc0-file A B C -+ parallel -E stop echo ::: A B stop C D + parallel -E stop echo ::: A B stop C D A B -+ parallel --no-run-if-empty echo + (echo 1; echo; echo 2) | parallel --no-run-if-empty echo 1 2 -+ parallel ::: ls 'echo foo' pwd -1-col -1-col.diff -1-col.txt -2-col -2-col.diff -2-col.txt -a + parallel ::: ls 'echo foo' pwd abc-file abc0-file abc_-file -b def-file num1000000 num128 @@ -134,83 +137,85 @@ num30000 num8 num_%header tsv-file.tsv -中国 (Zhōngguó) foo /home/tange/privat/parallel/testsuite/tmp -+ export -f my_func -+ parallel my_func ::: 1 2 3 + my_func() { + echo in my_func $1 + } + export -f my_func + parallel my_func ::: 1 2 3 in my_func 1 in my_func 2 in my_func 3 -+ parallel echo ::: A/B.C + parallel echo ::: A/B.C A/B.C -+ parallel echo '{}' ::: A/B.C + parallel echo {} ::: A/B.C A/B.C -+ parallel echo '{.}' ::: A/B.C + parallel echo {.} ::: A/B.C A/B -+ parallel echo '{/}' ::: A/B.C + parallel echo {/} ::: A/B.C B.C -+ parallel echo '{//}' ::: A/B.C + parallel echo {//} ::: A/B.C A -+ parallel echo '{/.}' ::: A/B.C + parallel echo {/.} ::: A/B.C B -+ parallel echo '{#}' ::: A B C + parallel echo {#} ::: A B C 1 2 3 -+ parallel -j 2 echo '{%}' ::: A B C + parallel -j 2 echo {%} ::: A B C 1 2 1 -+ parallel -I ,, echo ,, ::: A/B.C + parallel -I ,, echo ,, ::: A/B.C A/B.C -+ parallel --extensionreplace ,, echo ,, ::: A/B.C + parallel --extensionreplace ,, echo ,, ::: A/B.C A/B -+ parallel --basenamereplace ,, echo ,, ::: A/B.C + parallel --basenamereplace ,, echo ,, ::: A/B.C B.C -+ parallel --dirnamereplace ,, echo ,, ::: A/B.C + parallel --dirnamereplace ,, echo ,, ::: A/B.C A -+ parallel --basenameextensionreplace ,, echo ,, ::: A/B.C + parallel --basenameextensionreplace ,, echo ,, ::: A/B.C B -+ parallel --seqreplace ,, echo ,, ::: A B C + parallel --seqreplace ,, echo ,, ::: A B C 1 2 3 -+ parallel -j2 --slotreplace ,, echo ,, ::: A B C + parallel -j2 --slotreplace ,, echo ,, ::: A B C 1 2 1 -+ parallel echo '{= s:\.[^.]+$::;s:\.[^.]+$::; =}' ::: foo.tar.gz + parallel echo '{= s:\.[^.]+$::;s:\.[^.]+$::; =}' ::: foo.tar.gz foo -+ parallel --parens ,,,, echo ',, s:\.[^.]+$::;s:\.[^.]+$::; ,,' ::: foo.tar.gz + parallel --parens ,,,, echo ',, s:\.[^.]+$::;s:\.[^.]+$::; ,,' ::: foo.tar.gz foo -+ parallel --rpl '.. s:\.[^.]+$::;s:\.[^.]+$::;' echo .. ::: foo.tar.gz + parallel --rpl '.. s:\.[^.]+$::;s:\.[^.]+$::;' echo '..' ::: foo.tar.gz foo -+ parallel --rpl '{..} s:\.[^.]+$::;s:\.[^.]+$::;' echo '{..}' ::: foo.tar.gz + parallel --rpl '{..} s:\.[^.]+$::;s:\.[^.]+$::;' echo '{..}' ::: foo.tar.gz foo -+ --rpl '{} ' -bash: line 103: --rpl: command not found -+ --rpl '{#} $_=$job->seq()' -bash: line 104: --rpl: command not found -+ --rpl '{%} $_=$job->slot()' -bash: line 105: --rpl: command not found -+ --rpl '{/} s:.*/::' -bash: line 106: --rpl: command not found -+ --rpl '{//} $Global::use{"File::Basename"} ||= eval "use File::Basename; 1;"; $_ = dirname($_);' -bash: line 107: --rpl: command not found -+ --rpl '{/.} s:.*/::; s:\.[^/.]+$::;' -bash: line 108: --rpl: command not found -+ --rpl '{.} s:\.[^/.]+$::' -bash: line 109: --rpl: command not found -+ parallel echo '{1}' and '{2}' ::: A B ::: C D + --rpl '{} ' + --rpl '{#} $_=$job->seq()' + --rpl '{%} $_=$job->slot()' + --rpl '{/} s:.*/::' + --rpl '{//} $Global::use{"File::Basename"} ||= eval "use File::Basename; 1;"; $_ = dirname($_);' + --rpl '{/.} s:.*/::; s:\.[^/.]+$::;' + --rpl '{.} s:\.[^/.]+$::' +/bin/bash: --rpl: command not found +/bin/bash: line 1: --rpl: command not found +/bin/bash: line 2: --rpl: command not found +/bin/bash: line 3: --rpl: command not found +/bin/bash: line 4: --rpl: command not found +/bin/bash: line 5: --rpl: command not found +/bin/bash: line 6: --rpl: command not found + parallel echo {1} and {2} ::: A B ::: C D A and C A and D B and C B and D -+ parallel echo '/={1/}' '//={1//}' '/.={1/.}' '.={1.}' ::: A/B.C D/E.F + parallel echo /={1/} //={1//} /.={1/.} .={1.} ::: A/B.C D/E.F /=B.C //=A /.=B .=A/B /=E.F //=D /.=E .=D/E -+ parallel echo '1={1}' '2={2}' '3={3}' '-1={-1}' '-2={-2}' '-3={-3}' ::: A B ::: C D ::: E F + parallel echo 1={1} 2={2} 3={3} -1={-1} -2={-2} -3={-3} ::: A B ::: C D ::: E F 1=A 2=C 3=E -1=E -2=C -3=A 1=A 2=C 3=F -1=F -2=C -3=A 1=A 2=D 3=E -1=E -2=D -3=A @@ -219,85 +224,87 @@ B and D 1=B 2=C 3=F -1=F -2=C -3=B 1=B 2=D 3=E -1=E -2=D -3=B 1=B 2=D 3=F -1=F -2=D -3=B -+ parallel echo '{=2 s:\.[^.]+$::;s:\.[^.]+$::; =} {1}' ::: bar ::: foo.tar.gz + parallel echo '{=2 s:\.[^.]+$::;s:\.[^.]+$::; =} {1}' ::: bar ::: foo.tar.gz foo bar -+ parallel --rpl '{..} s:\.[^.]+$::;s:\.[^.]+$::;' echo '{2..} {1}' ::: bar ::: foo.tar.gz + parallel --rpl '{..} s:\.[^.]+$::;s:\.[^.]+$::;' echo '{2..} {1}' ::: bar ::: foo.tar.gz foo bar -+ parallel --colsep '\t' echo '1={1}' '2={2}' :::: tsv-file.tsv + parallel --colsep '\t' echo 1={1} 2={2} :::: tsv-file.tsv 1=f1 2=f2 1=A 2=B 1=C 2=D -+ parallel --header : echo 'f1={f1}' 'f2={f2}' ::: f1 A B ::: f2 C D + parallel --header : echo f1={f1} f2={f2} ::: f1 A B ::: f2 C D f1=A f2=C f1=A f2=D f1=B f2=C f1=B f2=D -+ parallel --header : --colsep '\t' echo 'f1={f1}' 'f2={f2}' :::: tsv-file.tsv + parallel --header : --colsep '\t' echo f1={f1} f2={f2} :::: tsv-file.tsv f1=A f2=B f1=C f2=D -+ parallel --xargs echo + cat num30000 | parallel --xargs echo | wc -l 2 -+ parallel --xargs -s 10000 echo + cat num30000 | parallel --xargs -s 10000 echo | wc -l 17 -+ parallel --jobs 4 -m echo + cat num30000 | parallel --jobs 4 -m echo | wc -l 5 -+ parallel --jobs 4 -m echo ::: 1 2 3 4 5 6 7 8 9 10 + parallel --jobs 4 -m echo ::: 1 2 3 4 5 6 7 8 9 10 1 2 3 4 5 6 7 8 9 10 -+ parallel --jobs 4 -X echo 'pre-{}-post' ::: A B C D E F G + parallel --jobs 4 -X echo pre-{}-post ::: A B C D E F G pre-A-post pre-B-post pre-C-post pre-D-post pre-E-post pre-F-post pre-G-post -+ parallel -N3 echo ::: A B C D E F G H + parallel -N3 echo ::: A B C D E F G H A B C D E F G H -+ parallel -N3 echo '1={1}' '2={2}' '3={3}' ::: A B C D E F G H + parallel -N3 echo 1={1} 2={2} 3={3} ::: A B C D E F G H 1=A 2=B 3=C 1=D 2=E 3=F 1=G 2=H 3= -+ parallel -N0 echo foo ::: 1 2 3 + parallel -N0 echo foo ::: 1 2 3 foo foo foo -+ perl -e 'print "@ARGV\n"' A + perl -e 'print "@ARGV\n"' A A -+ parallel perl -e 'print "@ARGV\n"' ::: This wont work -+ parallel: Warning: Input is read from the terminal. Only experts do this on purpose. Press CTRL-D to exit. -bash: line 148: parallel:: command not found -+ perl -e 'print "@ARGV\n"' + parallel perl -e 'print "@ARGV\n"' ::: This wont work -+ '[CTRL-D]' -bash: line 150: [CTRL-D]: command not found -+ parallel --trim r echo 'pre-{}-post' ::: ' A ' + parallel: Warning: Input is read from the terminal. Only experts do this on purpose. Press CTRL-D to exit. + perl -e 'print "@ARGV\n"' + [CTRL-D] + +/bin/bash: line 1: parallel:: command not found +/bin/bash: line 3: [CTRL-D]: command not found + + parallel --trim r echo pre-{}-post ::: ' A ' pre- A-post -+ parallel --trim l echo 'pre-{}-post' ::: ' A ' + parallel --trim l echo pre-{}-post ::: ' A ' pre-A -post -+ parallel --trim lr echo 'pre-{}-post' ::: ' A ' + parallel --trim lr echo pre-{}-post ::: ' A ' pre-A-post -+ parallel --tag echo 'foo-{}' ::: A B C + parallel --tag echo foo-{} ::: A B C A foo-A B foo-B C foo-C -+ parallel --tagstring '{}-bar' echo 'foo-{}' ::: A B C + parallel --tagstring {}-bar echo foo-{} ::: A B C A-bar foo-A B-bar foo-B C-bar foo-C -+ parallel --dryrun echo '{}' ::: A B C -echo A -echo B -echo C -+ parallel --verbose echo '{}' ::: A B C + parallel --dryrun echo {} ::: A B C echo A echo B echo C + parallel --verbose echo {} ::: A B C A B C -+ parallel -j2 'printf "%s-start\n%s" {} {};sleep {};printf "%s\n" -middle;echo {}-end' ::: 4 2 1 +echo A +echo B +echo C + parallel -j2 'printf "%s-start\n%s" {} {};sleep {};printf "%s\n" -middle;echo {}-end' ::: 4 2 1 4-start 4-middle 4-end @@ -307,7 +314,7 @@ C 1-start 1-middle 1-end -+ parallel -j2 --ungroup 'printf "%s-start\n%s" {} {};sleep {};printf "%s\n" -middle;echo {}-end' ::: 4 2 1 + parallel -j2 --ungroup 'printf "%s-start\n%s" {} {};sleep {};printf "%s\n" -middle;echo {}-end' ::: 4 2 1 4-start 42-start 2-middle @@ -317,7 +324,7 @@ C 1-end -middle 4-end -+ parallel -j2 --linebuffer 'printf "%s-start\n%s" {} {};sleep {};printf "%s\n" -middle;echo {}-end' ::: 4 2 1 + parallel -j2 --linebuffer 'printf "%s-start\n%s" {} {};sleep {};printf "%s\n" -middle;echo {}-end' ::: 4 2 1 4-start 2-start 2-middle @@ -327,7 +334,7 @@ C 1-end 4-middle 4-end -+ parallel -j2 -k 'printf "%s-start\n%s" {} {};sleep {};printf "%s\n" -middle;echo {}-end' ::: 4 2 1 + parallel -j2 -k 'printf "%s-start\n%s" {} {};sleep {};printf "%s\n" -middle;echo {}-end' ::: 4 2 1 4-start 4-middle 4-end @@ -337,114 +344,117 @@ C 1-start 1-middle 1-end -+ parallel --files echo ::: A B C + parallel --files echo ::: A B C /tmp/parXXXXX.par /tmp/parXXXXX.par /tmp/parXXXXX.par -+ parallel --tmpdir /var/tmp --files echo ::: A B C + parallel --tmpdir /var/tmp --files echo ::: A B C /var/tmp/parXXXXX.par /var/tmp/parXXXXX.par /var/tmp/parXXXXX.par -+ parallel --results outdir echo ::: A B C + parallel --results outdir echo ::: A B C A B C -+ outdir/1/A/stderr -bash: line 181: outdir/1/A/stderr: Permission denied -+ outdir/1/A/stdout -bash: line 182: outdir/1/A/stdout: Permission denied -+ outdir/1/B/stderr -bash: line 183: outdir/1/B/stderr: Permission denied -+ outdir/1/B/stdout -bash: line 184: outdir/1/B/stdout: Permission denied -+ outdir/1/C/stderr -bash: line 185: outdir/1/C/stderr: Permission denied -+ outdir/1/C/stdout -bash: line 186: outdir/1/C/stdout: Permission denied -+ parallel --header : --results outdir echo ::: f1 A B ::: f2 C D + outdir/1/A/stderr + outdir/1/A/stdout + outdir/1/B/stderr + outdir/1/B/stdout + outdir/1/C/stderr + outdir/1/C/stdout +/bin/bash: outdir/1/A/stderr: No such file or directory +/bin/bash: line 1: outdir/1/A/stdout: No such file or directory +/bin/bash: line 2: outdir/1/B/stderr: No such file or directory +/bin/bash: line 3: outdir/1/B/stdout: No such file or directory +/bin/bash: line 4: outdir/1/C/stderr: No such file or directory +/bin/bash: line 5: outdir/1/C/stdout: No such file or directory + parallel --header : --results outdir echo ::: f1 A B ::: f2 C D A C A D B C B D -+ outdir/f1/A/f2/C/stderr -bash: line 190: outdir/f1/A/f2/C/stderr: Permission denied -+ outdir/f1/A/f2/C/stdout -bash: line 191: outdir/f1/A/f2/C/stdout: Permission denied -+ outdir/f1/A/f2/D/stderr -bash: line 192: outdir/f1/A/f2/D/stderr: Permission denied -+ outdir/f1/A/f2/D/stdout -bash: line 193: outdir/f1/A/f2/D/stdout: Permission denied -+ outdir/f1/B/f2/C/stderr -bash: line 194: outdir/f1/B/f2/C/stderr: Permission denied -+ outdir/f1/B/f2/C/stdout -bash: line 195: outdir/f1/B/f2/C/stdout: Permission denied -+ outdir/f1/B/f2/D/stderr -bash: line 196: outdir/f1/B/f2/D/stderr: Permission denied -+ outdir/f1/B/f2/D/stdout -bash: line 197: outdir/f1/B/f2/D/stdout: Permission denied -+ /usr/bin/time -f %e parallel -N0 -j64 sleep 1 :::: num128 + outdir/f1/A/f2/C/stderr + outdir/f1/A/f2/C/stdout + outdir/f1/A/f2/D/stderr + outdir/f1/A/f2/D/stdout + outdir/f1/B/f2/C/stderr + outdir/f1/B/f2/C/stdout + outdir/f1/B/f2/D/stderr + outdir/f1/B/f2/D/stdout +/bin/bash: outdir/f1/A/f2/C/stderr: No such file or directory +/bin/bash: line 1: outdir/f1/A/f2/C/stdout: No such file or directory +/bin/bash: line 2: outdir/f1/A/f2/D/stderr: No such file or directory +/bin/bash: line 3: outdir/f1/A/f2/D/stdout: No such file or directory +/bin/bash: line 4: outdir/f1/B/f2/C/stderr: No such file or directory +/bin/bash: line 5: outdir/f1/B/f2/C/stdout: No such file or directory +/bin/bash: line 6: outdir/f1/B/f2/D/stderr: No such file or directory +/bin/bash: line 7: outdir/f1/B/f2/D/stdout: No such file or directory + /usr/bin/time -f %e parallel -N0 -j64 sleep 1 :::: num128 2 -+ /usr/bin/time -f %e parallel -N0 sleep 1 :::: num128 + /usr/bin/time -f %e parallel -N0 sleep 1 :::: num128 16 -+ /usr/bin/time -f %e parallel -N0 --jobs 200% sleep 1 :::: num128 + /usr/bin/time -f %e parallel -N0 --jobs 200% sleep 1 :::: num128 8 -+ /usr/bin/time -f %e parallel -N0 --jobs 0 sleep 1 :::: num128 + /usr/bin/time -f %e parallel -N0 --jobs 0 sleep 1 :::: num128 1 -+ sleep 1 -+ /usr/bin/time -f %e parallel -N0 --jobs my_jobs sleep 1 :::: num128 -+ wait + echo 50% > my_jobs + /usr/bin/time -f %e parallel -N0 --jobs my_jobs sleep 1 :::: num128 & + sleep 1 + echo 0 > my_jobs + wait 3 -+ parallel --use-cpus-instead-of-cores -N0 sleep 1 :::: num8 -+ seq 10 20 -+ parallel --tmux 'echo start {}; sleep {}; echo done {}' + parallel --use-cpus-instead-of-cores -N0 sleep 1 :::: num8 + + seq 10 20 | parallel --tmux 'echo start {}; sleep {}; echo done {}' See output with: tmux -S /tmp/parXXXXX.tms attach -+ tmux -S /tmp/parXXXXX.tms attach + tmux -S /tmp/parXXXXX.tms attach no sessions -+ parallel --delay 2.5 echo Starting '{};date' ::: 1 2 3 + parallel --delay 2.5 echo Starting {}\;date ::: 1 2 3 Starting 1 DATE OUTPUT Starting 2 DATE OUTPUT Starting 3 DATE OUTPUT -+ parallel --timeout 2.1 sleep '{};' echo '{}' ::: 1 2 3 4 -1 + parallel --timeout 4.1 sleep {}\; echo {} ::: 2 4 6 8 2 -+ parallel --timeout 200% sleep '{};' echo '{}' ::: 2.1 2.2 3 7 2.3 +4 + parallel --timeout 200% sleep {}\; echo {} ::: 2.1 2.2 3 7 2.3 2 2 3 2 -+ parallel --eta sleep ::: 1 3 2 2 1 3 3 2 1 + parallel --eta sleep ::: 1 3 2 2 1 3 3 2 1 Computers / CPU cores / Max jobs to run 1:local / 8 / 8 Computer:jobs running/jobs completed/%of started jobs/Average seconds to complete -+ parallel --progress sleep ::: 1 3 2 2 1 3 3 2 1 + parallel --progress sleep ::: 1 3 2 2 1 3 3 2 1 Computers / CPU cores / Max jobs to run 1:local / 8 / 8 Computer:jobs running/jobs completed/%of started jobs/Average seconds to complete -+ seq 1000 -+ parallel -j10 --bar '(echo -n {};sleep 0.1)' -++ zenity --timeout=12 --progress --auto-kill -BASE64 parallel --joblog /tmp/log exit ::: 1 2 3 0 + seq 1000 | parallel -j10 --bar '(echo -n {};sleep 0.1)' 2> >(zenity --timeout=12 --progress --auto-kill) +BASE64 parallel --joblog /tmp/log exit ::: 1 2 3 0 + cat /tmp/log Seq Host Starttime JobRuntime Send Receive Exitval Signal Command 1 : TIMESTAMP 9.999 0 0 1 0 exit 1 2 : TIMESTAMP 9.999 0 0 2 0 exit 2 3 : TIMESTAMP 9.999 0 0 3 0 exit 3 4 : TIMESTAMP 9.999 0 0 0 0 exit 0 -+ parallel --joblog /tmp/log exit ::: 1 2 3 0 + parallel --joblog /tmp/log exit ::: 1 2 3 0 + cat /tmp/log + parallel --resume --joblog /tmp/log exit ::: 1 2 3 0 0 0 + cat /tmp/log Seq Host Starttime JobRuntime Send Receive Exitval Signal Command 1 : TIMESTAMP 9.999 0 0 1 0 exit 1 2 : TIMESTAMP 9.999 0 0 2 0 exit 2 3 : TIMESTAMP 9.999 0 0 3 0 exit 3 4 : TIMESTAMP 9.999 0 0 0 0 exit 0 -+ parallel --resume --joblog /tmp/log exit ::: 1 2 3 0 0 0 Seq Host Starttime JobRuntime Send Receive Exitval Signal Command 1 : TIMESTAMP 9.999 0 0 1 0 exit 1 2 : TIMESTAMP 9.999 0 0 2 0 exit 2 @@ -452,7 +462,8 @@ Seq Host Starttime JobRuntime Send Receive Exitval Signal Command 4 : TIMESTAMP 9.999 0 0 0 0 exit 0 5 : TIMESTAMP 9.999 0 0 0 0 exit 0 6 : TIMESTAMP 9.999 0 0 0 0 exit 0 -+ parallel --resume-failed --joblog /tmp/log exit ::: 1 2 3 0 0 0 + parallel --resume-failed --joblog /tmp/log exit ::: 1 2 3 0 0 0 + cat /tmp/log Seq Host Starttime JobRuntime Send Receive Exitval Signal Command 1 : TIMESTAMP 9.999 0 0 1 0 exit 1 2 : TIMESTAMP 9.999 0 0 2 0 exit 2 @@ -463,34 +474,35 @@ Seq Host Starttime JobRuntime Send Receive Exitval Signal Command 1 : TIMESTAMP 9.999 0 0 1 0 exit 1 2 : TIMESTAMP 9.999 0 0 2 0 exit 2 3 : TIMESTAMP 9.999 0 0 3 0 exit 3 -+ parallel -j2 --halt 1 echo '{};' exit '{}' ::: 0 0 1 2 3 + parallel -j2 --halt 1 echo {}\; exit {} ::: 0 0 1 2 3 0 0 1 +2 parallel: Starting no more jobs. Waiting for 2 jobs to finish. This job failed: echo 1; exit 1 -2 parallel: Starting no more jobs. Waiting for 1 jobs to finish. This job failed: echo 2; exit 2 -+ parallel -j2 --halt 2 echo '{};' exit '{}' ::: 0 0 1 2 3 + parallel -j2 --halt 2 echo {}\; exit {} ::: 0 0 1 2 3 0 0 1 parallel: This job failed: echo 1; exit 1 -+ parallel -j2 --halt 20% echo '{};' exit '{}' ::: 0 0 1 2 3 4 5 6 7 + parallel -j2 --halt 20% echo {}\; exit {} ::: 0 0 1 2 3 4 5 6 7 0 0 1 2 3 4 +5 parallel: Starting no more jobs. Waiting for 2 jobs to finish. This job failed: echo 4; exit 4 -5 parallel: Starting no more jobs. Waiting for 1 jobs to finish. This job failed: echo 5; exit 5 -+ parallel -k --retries 3 'echo tried {} >>/tmp/runs; echo completed {}; exit {}' ::: 1 2 0 + parallel -k --retries 3 'echo tried {} >>/tmp/runs; echo completed {}; exit {}' ::: 1 2 0 + cat /tmp/runs completed 1 completed 2 completed 0 @@ -501,78 +513,92 @@ tried 1 tried 2 tried 1 tried 2 -+ parallel --load 100% echo load is less than '{}' job per cpu ::: 1 + parallel --load 100% echo load is less than {} job per cpu ::: 1 load is less than 1 job per cpu -+ parallel --noswap echo the system is not swapping ::: now + parallel --noswap echo the system is not swapping ::: now the system is not swapping now -+ parallel --nice 17 echo this is being run with nice -n ::: 17 + parallel --nice 17 echo this is being run with nice -n ::: 17 this is being run with nice -n 17 -+ parallel -S parallel@lo echo running on ::: parallel@lo + parallel -S $SERVER1 echo running on ::: $SERVER1 running on parallel@lo -+ parallel -S parallel@lo echo running on ::: username@parallel@lo + parallel -S $SERVER1 echo running on ::: username@$SERVER1 running on username@parallel@lo -+ parallel -S : echo running on ::: the_local_machine + parallel -S : echo running on ::: the_local_machine running on the_local_machine -+ parallel -S '/usr/bin/ssh parallel@lo' echo custom ::: ssh + parallel -S '/usr/bin/ssh '$SERVER1 echo custom ::: ssh custom ssh -+ parallel -S parallel@lo -S csh@lo echo ::: running on more hosts + parallel -S $SERVER1 -S $SERVER2 echo ::: running on more hosts running on more hosts -+ parallel --sshloginfile nodefile echo ::: running on more hosts + echo $SERVER1 > nodefile + # Force 4 cores, special ssh-command, username + echo 4//usr/bin/ssh $SERVER2 >> nodefile + parallel --sshloginfile nodefile echo ::: running on more hosts running on more hosts -+ parallel -S 4/parallel@lo echo force '{}' cpus on server ::: 4 + parallel -S 4/$SERVER1 echo force {} cpus on server ::: 4 force 4 cpus on server -+ parallel -S parallel@lo --transfer cat ::: input_file + echo This is input_file > input_file + parallel -S $SERVER1 --transfer cat ::: input_file This is input_file -+ parallel -S parallel@lo --transfer --return '{}.out' cat '{}' '>{}.out' ::: input_file + echo This is input_file > input_file + parallel -S $SERVER1 --transfer --return {}.out cat {} ">"{}.out ::: input_file + cat input_file.out This is input_file -+ parallel -S parallel@lo --transfer --return '{}.out' --cleanup cat '{}' '>{}.out' ::: input_file + echo This is input_file > input_file + parallel -S $SERVER1 --transfer --return {}.out --cleanup cat {} ">"{}.out ::: input_file + cat input_file.out This is input_file -+ parallel -S parallel@lo --trc '{}.out' cat '{}' '>{}.out' ::: input_file + echo This is input_file > input_file + parallel -S $SERVER1 --trc {}.out cat {} ">"{}.out ::: input_file + cat input_file.out This is input_file -+ parallel --basefile common_file -S parallel@lo cat 'common_file;' echo '{}' ::: foo + echo common data > common_file + parallel --basefile common_file -S $SERVER1 cat common_file\; echo {} ::: foo common data foo -+ parallel -S parallel@lo pwd ::: '' + parallel -S $SERVER1 pwd ::: "" + parallel --workdir . -S $SERVER1 pwd ::: "" + parallel --workdir ... -S $SERVER1 pwd ::: "" /home/parallel -+ parallel --workdir . -S parallel@lo pwd ::: '' /home/parallel/privat/parallel/testsuite/tmp -+ parallel --workdir ... -S parallel@lo pwd ::: '' /home/parallel/.TMPWORKDIR -+ parallel -S parallel@lo --sshdelay 0.2 echo ::: 1 2 3 + parallel -S $SERVER1 --sshdelay 0.2 echo ::: 1 2 3 1 2 3 -+ parallel --controlmaster -S parallel@lo echo ::: 1 2 3 + parallel --controlmaster -S $SERVER1 echo ::: 1 2 3 1 2 3 -+ parallel --filter-hosts -S 173.194.32.46,parallel@lo echo ::: bar + parallel --filter-hosts -S 173.194.32.46,$SERVER1 echo ::: bar +bar parallel: Warning: Removed 173.194.32.46 -bar -+ parallel --onall -S parallel@lo,csh@lo echo ::: foo bar + parallel --onall -S $SERVER1,$SERVER2 echo ::: foo bar foo bar foo bar -+ parallel --nonall --tag -S parallel@lo,csh@lo echo foo bar + parallel --nonall --tag -S $SERVER1,$SERVER2 echo foo bar csh@lo foo bar parallel@lo foo bar -+ MYVAR='foo bar' -+ export MYVAR -+ parallel --env MYVAR -S parallel@lo echo '$MYVAR' ::: baz + MYVAR='foo bar' + export MYVAR + parallel --env MYVAR -S $SERVER1 echo '$MYVAR' ::: baz foo bar baz -+ export -f my_func -+ parallel --env my_func -S parallel@lo my_func ::: baz + my_func() { + echo in my_func $1 + } + export -f my_func + parallel --env my_func -S $SERVER1 my_func ::: baz in my_func baz -+ parallel --record-env -+ sort -BASH_FUNC_my_func%% + parallel --record-env + cat ~/.parallel/ignored_vars|sort +BASH_FUNC_run_test%% CLUTTER_IM_MODULE COLORFGBG DBUS_SESSION_BUS_ADDRESS @@ -610,10 +636,11 @@ MAKELEVEL MANDATORY_PATH MFLAGS MOZ_NO_REMOTE -MYVAR ORACLE_HOME ORACLE_SID PARALLEL +PARALLEL_PID +PARALLEL_SEQ PATH PERL_MB_OPT PERL_MM_OPT @@ -656,19 +683,26 @@ XDG_VTNR XDM_MANAGED XMODIFIERS _ -+ export -f my_func2 -+ VAR=foo -+ export VAR -+ parallel --env _ -S parallel@lo 'echo $VAR; my_func2' ::: bar -foo -in my_func2 foo bar -+ parallel -vv -S parallel@lo echo ::: bar + # The function is only copied if using Bash + my_func2() { + echo in my_func2 $VAR $1 + } + export -f my_func2 + VAR=foo + export VAR + parallel --env _ -S $SERVER1 'echo $VAR; my_func2' ::: bar + +/bin/bash: my_func2: command not found + parallel -vv -S $SERVER1 echo ::: bar ssh parallel@lo exec perl\ -e\ \\\$ENV\\\{\\\"PARALLEL_PID\\\"\\\}=\\\"000000\\\"\\\;\\\$ENV\\\{\\\"PARALLEL_SEQ\\\"\\\}=\\\"1\\\"\\\;\\\$bashfunc\\\ =\\\ \\\"\\\"\\\;@ARGV=\\\"echo\\\ bar\\\"\\\;\\\$shell=\\\"\\\$ENV\\\{SHELL\\\}\\\"\\\;\\\$SIG\\\{CHLD\\\}=sub\\\{\\\$done=1\\\;\\\}\\\;\\\$pid=fork\\\;unless\\\(\\\$pid\\\)\\\{setpgrp\\\;exec\\\$shell,\\\"-c\\\",\\\(\\\$bashfunc.\\\"@ARGV\\\"\\\)\\\;die\\\"exec:\\\$\\\!\\\\n\\\"\\\;\\\}do\\\{\\\$s=\\\$s\\\<1\\\?0.001+\\\$s\\\*1.03:\\\$s\\\;select\\\(undef,undef,undef,\\\$s\\\)\\\;\\\}until\\\(\\\$done\\\|\\\|getppid==1\\\)\\\;kill\\\(SIGHUP,-\\\$\\\{pid\\\}\\\)unless\\\$done\\\;wait\\\;exit\\\(\\\$\\\?\\\&127\\\?128+\\\(\\\$\\\?\\\&127\\\):1+\\\$\\\?\\\>\\\>8\\\); bar -+ export -f my_func3 -+ parallel -vv --workdir ... --nice 17 --env _ --trc '{}.out' -S parallel@lo my_func3 '{}' ::: abc-file + my_func3() { + echo in my_func $1 > $1.out + } + export -f my_func3 + parallel -vv --workdir ... --nice 17 --env _ --trc {}.out -S $SERVER1 my_func3 {} ::: abc-file ( ssh parallel@lo mkdir -p ./.TMPWORKDIR;rsync --protocol 30 -rlDzR -essh ./abc-file parallel@lo:./.TMPWORKDIR );ssh parallel@lo exec perl -e \''@GNU_Parallel=("use","IPC::Open3;","use","MIME::Base64");eval"@GNU_Parallel";$SIG{CHLD}="IGNORE";my$zip=(grep{-x$_}"/usr/local/bin/bzip2")[0]||"bzip2";my($in,$out,$eval);open3($in,$out,">&STDERR",$zip,"-dc");if(my$perlpid=fork){close$in;$eval=join"",<$out>;close$out;}else{close$out;print$in(decode_base64(join"",@ARGV));close$in;exit;}wait;eval$eval;'\' BASE64;_EXIT_status=$?; mkdir -p ./.; rsync --protocol 30 --rsync-path=cd\ ./.TMPWORKDIR/./.\;\ rsync -rlDzR -essh parallel@lo:./abc-file.out ./.;ssh parallel@lo \(rm\ -f\ ./.TMPWORKDIR/abc-file\;\ sh\ -c\ \'rmdir\ ./.TMPWORKDIR/\ ./.parallel/tmp/\ ./.parallel/\ 2\>/dev/null\'\;rm\ -rf\ ./.TMPWORKDIR\;\);ssh parallel@lo \(rm\ -f\ ./.TMPWORKDIR/abc-file.out\;\ sh\ -c\ \'rmdir\ ./.TMPWORKDIR/\ ./.parallel/tmp/\ ./.parallel/\ 2\>/dev/null\'\;rm\ -rf\ ./.TMPWORKDIR\;\);ssh parallel@lo rm -rf .TMPWORKDIR; exit $_EXIT_status; -+ parallel --pipe wc + cat num1000000 | parallel --pipe wc 165668 165668 1048571 149797 149797 1048579 149796 149796 1048572 @@ -676,17 +710,17 @@ bar 149797 149797 1048579 149796 149796 1048572 85349 85349 597444 -+ parallel --pipe --block 2M wc + cat num1000000 | parallel --pipe --block 2M wc 315465 315465 2097150 299593 299593 2097151 299593 299593 2097151 85349 85349 597444 -+ parallel --pipe -j4 --round-robin wc + cat num1000000 | parallel --pipe -j4 --round-robin wc 315465 315465 2097150 299593 299593 2097151 235145 235145 1646016 149797 149797 1048579 -+ parallel --pipe -N140000 wc + cat num1000000 | parallel --pipe -N140000 wc 140000 140000 868895 140000 140000 980000 140000 140000 980000 @@ -695,7 +729,7 @@ bar 140000 140000 980000 140000 140000 980000 20000 20000 140001 -+ parallel --pipe -L75 wc + cat num1000000 | parallel --pipe -L75 wc 165600 165600 1048095 149850 149850 1048950 149775 149775 1048425 @@ -704,7 +738,7 @@ bar 149775 149775 1048425 85350 85350 597450 25 25 176 -+ parallel -kN1 --recend ', ' --pipe echo 'JOB{#};cat;echo' END + echo /foo, bar/, /baz, qux/, | parallel -kN1 --recend ', ' --pipe echo JOB{#}\;cat\;echo END JOB1 /foo, END JOB2 @@ -714,7 +748,7 @@ JOB3 JOB4 qux/, END -+ parallel -kN1 --recstart / --pipe echo 'JOB{#};cat;echo' END + echo /foo, bar/, /baz, qux/, | parallel -kN1 --recstart '/' --pipe echo JOB{#}\;cat\;echo END JOB1 /foo, barEND JOB2 @@ -724,13 +758,13 @@ JOB3 JOB4 /, END -+ parallel -kN1 --recend ', ' --recstart / --pipe echo 'JOB{#};cat;echo' END + echo /foo, bar/, /baz, qux/, | parallel -kN1 --recend ', ' --recstart '/' --pipe echo JOB{#}\;cat\;echo END JOB1 /foo, bar/, END JOB2 /baz, qux/, END -+ parallel -kN1 --regexp --recend ,_+ --pipe echo 'JOB{#};cat;echo' END + echo foo,bar,_baz,__qux, | parallel -kN1 --regexp --recend ',_+' --pipe echo JOB{#}\;cat\;echo END JOB1 foo,bar,_END JOB2 @@ -738,7 +772,7 @@ baz,__END JOB3 qux, END -+ parallel -kN1 --rrs --regexp --recend ,_+ --pipe echo 'JOB{#};cat;echo' END + echo foo,bar,_baz,__qux, | parallel -kN1 --rrs --regexp --recend ',_+' --pipe echo JOB{#}\;cat\;echo END JOB1 foo,barEND JOB2 @@ -746,7 +780,7 @@ bazEND JOB3 qux, END -+ parallel --header '(%.*\n)*' --pipe -N3 echo 'JOB{#};cat' + cat num_%header | parallel --header '(%.*\n)*' --pipe -N3 echo JOB{#}\;cat JOB1 %head1 %head2 @@ -769,7 +803,7 @@ JOB4 %head1 %head2 10 -+ parallel --header 2 --pipe -N3 echo 'JOB{#};cat' + cat num_%header | parallel --header 2 --pipe -N3 echo JOB{#}\;cat JOB1 %head1 %head2 @@ -792,72 +826,78 @@ JOB4 %head1 %head2 10 -+ parallel --pipepart -a num1000000 --block 3m wc + parallel --pipepart -a num1000000 --block 3m wc 444444 444444 3000003 428572 428572 3000004 126984 126984 888889 -+ parallel command + cat input_file | parallel command /bin/bash: This is input_file: command not found -+ foo -bash: line 374: foo: command not found -+ bar -bash: line 375: bar: command not found -+ baz -bash: line 376: baz: command not found -+ parallel command + #!/bin/bash + #!/usr/bin/parallel --shebang -r echo + foo + bar + baz +/bin/bash: foo: command not found +/bin/bash: line 1: bar: command not found +/bin/bash: line 2: baz: command not found + cat input_file | parallel command + parallel command ::: foo bar /bin/bash: This is input_file: command not found -+ parallel command ::: foo bar /bin/bash: foo: command not found /bin/bash: bar: command not found -+ command -+ command foo bar -+ foo bar -bash: line 382: foo: command not found -+ print '@ARGV\n' + cat input_file | command + command foo bar +/bin/bash: line 1: foo: command not found + #!/usr/bin/perl + print "@ARGV\n" Warning: unknown mime-type for "@ARGV\n" -- using "application/octet-stream" Error: no such file "@ARGV\n" -+ parallel perl_echo ::: foo bar + parallel perl_echo ::: foo bar /bin/bash: perl_echo: command not found /bin/bash: perl_echo: command not found -+ print '@ARGV\n' + #!/usr/bin/parallel --shebang-wrap /usr/bin/perl + print "@ARGV\n" Warning: unknown mime-type for "@ARGV\n" -- using "application/octet-stream" Error: no such file "@ARGV\n" -+ perl_echo foo bar -bash: line 394: perl_echo: command not found -+ sem 'sleep 1; echo The first finished' + perl_echo foo bar +/bin/bash: perl_echo: command not found + sem 'sleep 1; echo The first finished' && + echo The first is now running in the background && + sem 'sleep 1; echo The second finished' && + echo The second is now running in the background + sem --wait The first is now running in the background -+ sem 'sleep 1; echo The second finished' The first finished The second is now running in the background -+ sem --wait The second finished -+ sem --fg 'sleep 1; echo The first finished' + sem --fg 'sleep 1; echo The first finished' && + echo The first finished running in the foreground && + sem --fg 'sleep 1; echo The second finished' && + echo The second finished running in the foreground + sem --wait The first finished The first finished running in the foreground -+ sem --fg 'sleep 1; echo The second finished' The second finished The second finished running in the foreground -+ sem --wait -+ sem --id my_id -u 'echo First started; sleep 10; echo The first finished' -+ sem --id my_id -u 'echo Second started; sleep 10; echo The second finished' + sem --id my_id -u 'echo First started; sleep 10; echo The first finished' First started The first finished -+ sem --jobs 3 --id my_id -u 'echo First started; sleep 5; echo The first finished' + sem --id my_id -u 'echo Second started; sleep 10; echo The second finished' Second started -+ sem --jobs 3 --id my_id -u 'echo Second started; sleep 6; echo The second finished' + sem --jobs 3 --id my_id -u 'echo First started; sleep 5; echo The first finished' && + sem --jobs 3 --id my_id -u 'echo Second started; sleep 6; echo The second finished' && + sem --jobs 3 --id my_id -u 'echo Third started; sleep 7; echo The third finished' && + sem --jobs 3 --id my_id -u 'echo Fourth started; sleep 8; echo The fourth finished' && + sem --wait --id my_id First started -+ sem --jobs 3 --id my_id -u 'echo Third started; sleep 7; echo The third finished' Second started The first finished -+ sem --jobs 3 --id my_id -u 'echo Fourth started; sleep 8; echo The fourth finished' Third started The second finished -+ sem --wait --id my_id Fourth started -The second finished The third finished The fourth finished -+ parallel --help + parallel --help Usage: parallel [options] [command [arguments]] < list_of_arguments @@ -895,8 +935,8 @@ please cite: This helps funding further development; and it won't cost you a cent. If you pay 10000 EUR you should feel free to use GNU Parallel without citing. -+ parallel --version -GNU parallel 20150329 + parallel --version +GNU parallel 20150403 Copyright (C) 2007,2008,2009,2010,2011,2012,2013,2014,2015 Ole Tange and Free Software Foundation, Inc. License GPLv3+: GNU GPL version 3 or later @@ -907,10 +947,10 @@ Web site: http://www.gnu.org/software/parallel When using programs that use GNU Parallel to process data for publication please cite as described in 'parallel --bibtex'. -+ parallel --minversion 20130722 -20150329 + parallel --minversion 20130722 && echo Your version is at least 20130722. +20150403 Your version is at least 20130722. -+ parallel --bibtex + parallel --bibtex Academic tradition requires you to cite works you base your article on. When using programs that use GNU Parallel to process data for publication please cite: @@ -937,17 +977,21 @@ If you pay 10000 EUR you should feel free to use GNU Parallel without citing. If you send a copy of your published article to tange@gnu.org, it will be mentioned in the release notes of next version of GNU Parallel. -+ parallel --max-line-length-allowed + parallel --max-line-length-allowed 131071 -+ parallel --number-of-cpus + parallel --number-of-cpus + parallel --number-of-cores 1 -+ parallel --number-of-cores 8 -+ parallel --profile nicetimeout echo ::: A B C + echo '--nice 17' > ~/.parallel/nicetimeout + echo '--timeout 300%' >> ~/.parallel/nicetimeout + parallel --profile nicetimeout echo ::: A B C A B C -+ parallel --profile dryverbose --profile nicetimeout echo ::: A B C + echo '-vv --dry-run' > ~/.parallel/dryverbose + parallel --profile dryverbose --profile nicetimeout echo ::: A B C \nice -n17 /bin/bash -c echo\ A \nice -n17 /bin/bash -c echo\ B \nice -n17 /bin/bash -c echo\ C +7