From fc0c6cee08b10f2a331247bb912e7eb1c3613ee7 Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Sun, 3 May 2015 01:22:34 +0200 Subject: [PATCH] parallel: Fixed bug #44995: parallel echo {#} ::: 1 2 ::: 1 2. Passes testsuite. --- doc/release_new_version | 7 +- src/parallel | 82 +++++++++++---------- src/sem.pod | 13 +++- testsuite/tests-to-run/parallel-local-3s.sh | 5 ++ testsuite/wanted-results/parallel-local-3s | 44 +++++------ testsuite/wanted-results/parallel-tutorial | 44 ++++++----- 6 files changed, 114 insertions(+), 81 deletions(-) diff --git a/doc/release_new_version b/doc/release_new_version index 9e38d778..b6774557 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -226,13 +226,14 @@ New in this release: * <> CIDER: a pipeline for detecting waves of coordinated transcriptional regulation in gene expression time-course data http://biorxiv.org/content/biorxiv/early/2015/03/17/012518.full.pdf - * <> GNU Parallel was used (unfortunately without citation) in: MUGBAS: a species free gene-based programme suite for post-GWAS analysis http://www.ncbi.nlm.nih.gov/pubmed/25765345 taxator-tk http://algbio.cs.uni-duesseldorf.de/webapps/wa-download/ (check it) * << afventer svar fra Rachel >> GNU Parallel was used in: SISRS: Site Identification from Short Read Sequences https://github.com/rachelss/SISRS/ +* GNU Parallel was cited in: Toward Enhanced Metadata Quality of Large-Scale Digital Libraries: Estimating Volume Time Range https://www.ideals.illinois.edu/bitstream/handle/2142/73656/186_ready.pdf + * GNU Parallel was cited in: Sequencing the cap-snatching repertoire of H1N1 influenza provides insight into the mechanism of viral transcription initiation http://nar.oxfordjournals.org/content/early/2015/04/20/nar.gkv333.full.pdf * GNU Parallel was cited in: Genome assemblyusing Nanopore-guided long and error-free DNA reads http://www.biomedcentral.com/content/pdf/s12864-015-1519-z.pdf @@ -247,6 +248,10 @@ taxator-tk http://algbio.cs.uni-duesseldorf.de/webapps/wa-download/ (check it) * Functions and GNU parallel for effective cluster load management http://genomespot.blogspot.dk/2015/04/functions-and-gnu-parallel-for.html +* Use parallel processing to save time importing databases http://drupalsun.com/node/41854 + +* Run multiple ssh commands in parallel with GNU Parallel http://www.ameir.net/blog/archives/380-run-multiple-ssh-commands-in-parallel-with-gnu-parallel.html + * Bug fixes and man page updates. GNU Parallel - For people who live life in the parallel lane. diff --git a/src/parallel b/src/parallel index 9a71f93c..b72b1edb 100755 --- a/src/parallel +++ b/src/parallel @@ -1053,7 +1053,7 @@ sub parse_options { sub init_globals { # Defaults: - $Global::version = 20150424; + $Global::version = 20150426; $Global::progname = 'parallel'; $Global::infinity = 2**31; $Global::debug = 0; @@ -3032,8 +3032,11 @@ sub reaper { next; } my $job = $Global::running{$stiff}; + # '-a <(seq 10)' will give us a pid not in %Global::running $job or next; + delete $Global::running{$stiff}; + $Global::total_running--; $job->set_exitstatus($? >> 8); $job->set_exitsignal($? & 127); debug("run", "seq ",$job->seq()," died (", $job->exitstatus(), ")"); @@ -3042,7 +3045,9 @@ sub reaper { # The process that died had the tty => release it $Global::tty_taken = 0; } - + my $sshlogin = $job->sshlogin(); + $sshlogin->dec_jobs_running(); + $sshlogin->inc_jobs_completed(); if(not $job->should_be_retried()) { # The job is done # Free the jobslot @@ -3057,15 +3062,10 @@ sub reaper { $job->print(); } if($job->should_we_halt() eq "now") { - ::killall(); - ::wait_and_exit($Global::exitstatus); + ::killall(); + ::wait_and_exit($Global::halt_exitstatus); } } - my $sshlogin = $job->sshlogin(); - $sshlogin->dec_jobs_running(); - $sshlogin->inc_jobs_completed(); - $Global::total_running--; - delete $Global::running{$stiff}; start_more_jobs(); if($opt::progress) { my %progress = progress(); @@ -7490,9 +7490,16 @@ sub set_exitsignal { $Global::halt_pct <= $Global::total_failed / $Global::total_started and $Global::total_failed > 3)) { - # More than N jobs or more than N% failed - $Global::halt_exitstatus = $job->exitstatus(); - if($Global::halt_when eq "soon") { + # At least N jobs had failed + # or at least N% had failed and more than 3 + if($Global::halt_count and $Global::halt_count == 1) { + $Global::halt_exitstatus = $job->exitstatus(); + } else { + $Global::halt_exitstatus = $Global::total_failed; + } + ::debug("halt","Pct: ",$Global::halt_pct,"<=",$Global::total_failed / $Global::total_started," count: ",$Global::halt_count,"\n"); + if($Global::halt_when eq "soon" + and scalar(keys %Global::running) > 0) { ::status ("$Global::progname: Starting no more jobs. ", "Waiting for ", scalar(keys %Global::running), @@ -7504,19 +7511,23 @@ sub set_exitsignal { } } else { if($Global::halt_success) { + $Global::halt_exitstatus = $Global::total_failed; ::status("$Global::progname: This job succeeded:\n", $job->replaced(),"\n"); if(($Global::halt_count and $Global::halt_count <= - 1+$Global::total_completed-$Global::total_failed) + $Global::total_completed-$Global::total_failed) or ($Global::halt_pct and $Global::halt_pct <= - (1+$Global::total_completed-$Global::total_failed) + ($Global::total_completed-$Global::total_failed) / $Global::total_completed and ($Global::total_completed-$Global::total_failed) > 3)) { - $Global::halt_exitstatus = $job->exitstatus(); - if($Global::halt_when eq "soon") { + # At least N jobs had success + # or at least N% had success and more than 3 + $Global::halt_exitstatus = 0; + if($Global::halt_when eq "soon" + and scalar(keys %Global::running) > 0) { ::status ("$Global::progname: Starting no more jobs. ", "Waiting for ", scalar(keys %Global::running), @@ -8845,18 +8856,20 @@ sub new { }, ref($class) || $class; } -sub replace { - # Calculates the corresponding value for a given perl expression - # Returns: - # The calculated string (quoted if asked for) - my $self = shift; - my $perlexpr = shift; # E.g. $_=$_ or s/.gz// - my $quote = (shift) ? 1 : 0; # should the string be quoted? - # This is actually a CommandLine-object, - # but it looks nice to be able to say {= $job->slot() =} - my $job = shift; - $perlexpr =~ s/^-?\d+ //; # Positional replace treated as normal replace - if(not defined $self->{"rpl",0,$perlexpr}) { +{ + my %perleval; + + sub replace { + # Calculates the corresponding value for a given perl expression + # Returns: + # The calculated string (quoted if asked for) + my $self = shift; + my $perlexpr = shift; # E.g. $_=$_ or s/.gz// + my $quote = (shift) ? 1 : 0; # should the string be quoted? + # This is actually a CommandLine-object, + # but it looks nice to be able to say {= $job->slot() =} + my $job = shift; + $perlexpr =~ s/^-?\d+ +//; # Positional replace treated as normal replace local $_; if($Global::trim eq "n") { $_ = $self->{'orig'}; @@ -8864,10 +8877,10 @@ sub replace { $_ = trim_of($self->{'orig'}); } ::debug("replace", "eval ", $perlexpr, " ", $_, "\n"); - if(not $Global::perleval{$perlexpr}) { + if(not $perleval{$perlexpr}) { # Make an anonymous function of the $perlexpr # And more importantly: Compile it only once - if($Global::perleval{$perlexpr} = + if($perleval{$perlexpr} = eval('sub { no strict; no warnings; my $job = shift; '. $perlexpr.' }')) { # All is good @@ -8878,14 +8891,9 @@ sub replace { } } # Execute the function - $Global::perleval{$perlexpr}->($job); - $self->{"rpl",0,$perlexpr} = $_; + $perleval{$perlexpr}->($job); + return $quote ? ::shell_quote_scalar($_) : $_; } - if(not defined $self->{"rpl",$quote,$perlexpr}) { - $self->{"rpl",1,$perlexpr} = - ::shell_quote_scalar($self->{"rpl",0,$perlexpr}); - } - return $self->{"rpl",$quote,$perlexpr}; } sub orig { diff --git a/src/sem.pod b/src/sem.pod index 389a858a..742536cf 100755 --- a/src/sem.pod +++ b/src/sem.pod @@ -161,10 +161,17 @@ available. pod2html creates two files: pod2htmd.tmp and pod2htmi.tmp which it does not clean up. It uses these two files for a short time. But if you run multiple pod2html in parallel (e.g. in a Makefile with make --j) you need to protect pod2html from running twice at the same -time. B running as a mutex will do just that: +-j) there is a risk that two different instances of pod2html will +write to the files at the same time: - sem --fg --id pod2html pod2html foo.pod > foo.html + # This may fail due to shared pod2htmd.tmp/pod2htmi.tmp files + pod2html foo.pod --outfile foo.html & pod2html bar.pod --outfile bar.html + +You need to protect pod2html from running twice at the same time. +B running as a mutex will make sure only one runs: + + sem --id pod2html pod2html foo.pod --outfile foo.html + sem --id pod2html pod2html bar.pod --outfile bar.html sem --fg --id pod2html rm -f pod2htmd.tmp pod2htmi.tmp diff --git a/testsuite/tests-to-run/parallel-local-3s.sh b/testsuite/tests-to-run/parallel-local-3s.sh index e0e1013e..caa1f920 100644 --- a/testsuite/tests-to-run/parallel-local-3s.sh +++ b/testsuite/tests-to-run/parallel-local-3s.sh @@ -14,6 +14,7 @@ echo '**' echo '### Test --halt-on-error 0'; (echo "sleep 1;true"; echo "sleep 2;false";echo "sleep 3;true") | parallel -j10 --halt-on-error 0; echo $?; + (echo "sleep 1;true"; echo "sleep 2;false";echo "sleep 3;true";echo "sleep 4; non_exist") | parallel -j10 --halt 0; echo $? @@ -22,6 +23,7 @@ echo '**' echo '### Test --halt-on-error 1'; (echo "sleep 1;true"; echo "sleep 2;false";echo "sleep 3;true") | parallel -j10 --halt-on-error 1; echo $?; + (echo "sleep 1;true"; echo "sleep 2;false";echo "sleep 3;true";echo "sleep 4; non_exist") | parallel -j10 --halt 1; echo $? @@ -30,6 +32,7 @@ echo '**' echo '### Test --halt-on-error 2'; (echo "sleep 1;true"; echo "sleep 2;false";echo "sleep 3;true") | parallel -j10 --halt-on-error 2; echo $?; + (echo "sleep 1;true"; echo "sleep 2;false";echo "sleep 3;true";echo "sleep 4; non_exist") | parallel -j10 --halt 2; echo $? @@ -38,6 +41,7 @@ echo '**' echo '### Test --halt -1'; (echo "sleep 1;false"; echo "sleep 2;true";echo "sleep 3;false") | parallel -j10 --halt-on-error -1; echo $?; + (echo "sleep 1;false"; echo "sleep 2;true";echo "sleep 3;false";echo "sleep 4; non_exist") | parallel -j10 --halt -1; echo $? @@ -46,6 +50,7 @@ echo '**' echo '### Test --halt -2'; (echo "sleep 1;false"; echo "sleep 2;true";echo "sleep 3;false") | parallel -j10 --halt-on-error -2; echo $?; + (echo "sleep 1;false"; echo "sleep 2;true";echo "sleep 3;false";echo "sleep 4; non_exist") | parallel -j10 --halt -2; echo $? diff --git a/testsuite/wanted-results/parallel-local-3s b/testsuite/wanted-results/parallel-local-3s index 4a6ec714..50514ada 100644 --- a/testsuite/wanted-results/parallel-local-3s +++ b/testsuite/wanted-results/parallel-local-3s @@ -19,12 +19,13 @@ echo '### Test --halt-on-error 1'; (echo "sleep 1;true"; echo "sleep 2;false"; 127 parallel: This job failed: sleep 2;false -parallel: Starting no more jobs. Waiting for 3 jobs to finish. This job failed: +parallel: Starting no more jobs. Waiting for 1 jobs to finish. +parallel: This job failed: sleep 2;false +parallel: Starting no more jobs. Waiting for 2 jobs to finish. /bin/bash: non_exist: command not found parallel: This job failed: sleep 4; non_exist -parallel: Starting no more jobs. Waiting for 1 jobs to finish. echo '**' ** echo '### Test --halt-on-error 2'; (echo "sleep 1;true"; echo "sleep 2;false";echo "sleep 3;true") | parallel -j10 --halt-on-error 2; echo $?; (echo "sleep 1;true"; echo "sleep 2;false";echo "sleep 3;true";echo "sleep 4; non_exist") | parallel -j10 --halt 2; echo $? @@ -43,9 +44,10 @@ echo '### Test --halt -1'; (echo "sleep 1;false"; echo "sleep 2;true";echo "sl 0 parallel: This job succeeded: sleep 2;true -parallel: Starting no more jobs. Waiting for 3 jobs to finish. This job succeeded: +parallel: Starting no more jobs. Waiting for 1 jobs to finish. +parallel: This job succeeded: sleep 2;true -parallel: Starting no more jobs. Waiting for 3 jobs to finish. +parallel: Starting no more jobs. Waiting for 2 jobs to finish. /bin/bash: non_exist: command not found echo '**' ** @@ -66,40 +68,39 @@ exit code 9 1 parallel: This job failed: perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 1 -parallel: Starting no more jobs. Waiting for 9 jobs to finish. +parallel: Starting no more jobs. Waiting for 8 jobs to finish. 2 parallel: This job failed: perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 2 -parallel: Starting no more jobs. Waiting for 8 jobs to finish. +parallel: Starting no more jobs. Waiting for 7 jobs to finish. 3 parallel: This job failed: perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 3 -parallel: Starting no more jobs. Waiting for 7 jobs to finish. +parallel: Starting no more jobs. Waiting for 6 jobs to finish. 4 parallel: This job failed: perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 4 -parallel: Starting no more jobs. Waiting for 6 jobs to finish. +parallel: Starting no more jobs. Waiting for 5 jobs to finish. 5 parallel: This job failed: perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 5 -parallel: Starting no more jobs. Waiting for 5 jobs to finish. +parallel: Starting no more jobs. Waiting for 4 jobs to finish. 6 parallel: This job failed: perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 6 -parallel: Starting no more jobs. Waiting for 4 jobs to finish. +parallel: Starting no more jobs. Waiting for 3 jobs to finish. 7 parallel: This job failed: perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 7 -parallel: Starting no more jobs. Waiting for 3 jobs to finish. +parallel: Starting no more jobs. Waiting for 2 jobs to finish. 8 0 parallel: This job failed: perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 8 -parallel: Starting no more jobs. Waiting for 2 jobs to finish. +parallel: Starting no more jobs. Waiting for 1 jobs to finish. 9 parallel: This job failed: perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 9 -parallel: Starting no more jobs. Waiting for 1 jobs to finish. echo '### Test last dying print --halt-on-error 2'; (seq 0 8;echo 0; echo 9) | parallel -j10 -kq --halt 2 perl -e 'sleep $ARGV[0];print STDERR @ARGV,"\n"; exit shift'; echo exit code $? ### Test last dying print --halt-on-error 2 exit code 1 @@ -114,40 +115,39 @@ exit code 0 1 parallel: This job succeeded: perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ not\ shift 1 -parallel: Starting no more jobs. Waiting for 9 jobs to finish. +parallel: Starting no more jobs. Waiting for 8 jobs to finish. 2 parallel: This job succeeded: perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ not\ shift 2 -parallel: Starting no more jobs. Waiting for 8 jobs to finish. +parallel: Starting no more jobs. Waiting for 7 jobs to finish. 3 parallel: This job succeeded: perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ not\ shift 3 -parallel: Starting no more jobs. Waiting for 7 jobs to finish. +parallel: Starting no more jobs. Waiting for 6 jobs to finish. 4 parallel: This job succeeded: perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ not\ shift 4 -parallel: Starting no more jobs. Waiting for 6 jobs to finish. +parallel: Starting no more jobs. Waiting for 5 jobs to finish. 5 parallel: This job succeeded: perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ not\ shift 5 -parallel: Starting no more jobs. Waiting for 5 jobs to finish. +parallel: Starting no more jobs. Waiting for 4 jobs to finish. 6 parallel: This job succeeded: perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ not\ shift 6 -parallel: Starting no more jobs. Waiting for 4 jobs to finish. +parallel: Starting no more jobs. Waiting for 3 jobs to finish. 7 parallel: This job succeeded: perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ not\ shift 7 -parallel: Starting no more jobs. Waiting for 3 jobs to finish. +parallel: Starting no more jobs. Waiting for 2 jobs to finish. 8 0 parallel: This job succeeded: perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ not\ shift 8 -parallel: Starting no more jobs. Waiting for 2 jobs to finish. +parallel: Starting no more jobs. Waiting for 1 jobs to finish. 9 parallel: This job succeeded: perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ not\ shift 9 -parallel: Starting no more jobs. Waiting for 1 jobs to finish. echo '### Test last dying print --halt-on-error -2'; (seq 0 8;echo 0; echo 9) | parallel -j10 -kq --halt -2 perl -e 'sleep $ARGV[0];print STDERR @ARGV,"\n"; exit not shift'; echo exit code $? ### Test last dying print --halt-on-error -2 exit code 0 diff --git a/testsuite/wanted-results/parallel-tutorial b/testsuite/wanted-results/parallel-tutorial index d6c1c5b3..7845a371 100644 --- a/testsuite/wanted-results/parallel-tutorial +++ b/testsuite/wanted-results/parallel-tutorial @@ -478,9 +478,10 @@ Seq Host Starttime JobRuntime Send Receive Exitval Signal Command 0 1 2 -parallel: Starting no more jobs. Waiting for 2 jobs to finish. This job failed: +parallel: This job failed: echo 1; exit 1 -parallel: Starting no more jobs. Waiting for 1 jobs to finish. This job failed: +parallel: Starting no more jobs. Waiting for 1 jobs to finish. +parallel: This job failed: echo 2; exit 2 parallel -j2 --halt 2 echo {}\; exit {} ::: 0 0 1 2 3 0 @@ -496,9 +497,16 @@ echo 1; exit 1 3 4 5 -parallel: Starting no more jobs. Waiting for 2 jobs to finish. This job failed: +parallel: This job failed: +echo 1; exit 1 +parallel: This job failed: +echo 2; exit 2 +parallel: This job failed: +echo 3; exit 3 +parallel: This job failed: echo 4; exit 4 -parallel: Starting no more jobs. Waiting for 1 jobs to finish. This job failed: +parallel: Starting no more jobs. Waiting for 1 jobs to finish. +parallel: This job failed: echo 5; exit 5 parallel -k --retries 3 'echo tried {} >>/tmp/runs; echo completed {}; exit {}' ::: 1 2 0 cat /tmp/runs @@ -931,7 +939,7 @@ This helps funding further development; and it won't cost you a cent. If you pay 10000 EUR you should feel free to use GNU Parallel without citing. parallel --version -GNU parallel 20150422 +GNU parallel 20150426 Copyright (C) 2007,2008,2009,2010,2011,2012,2013,2014,2015 Ole Tange and Free Software Foundation, Inc. License GPLv3+: GNU GPL version 3 or later @@ -943,7 +951,7 @@ Web site: http://www.gnu.org/software/parallel When using programs that use GNU Parallel to process data for publication please cite as described in 'parallel --bibtex'. parallel --minversion 20130722 && echo Your version is at least 20130722. -20150422 +20150426 Your version is at least 20130722. parallel --bibtex Academic tradition requires you to cite works you base your article on. @@ -951,17 +959,17 @@ When using programs that use GNU Parallel to process data for publication please cite: @article{Tange2011a, - title = {GNU Parallel - The Command-Line Power Tool}, - author = {O. Tange}, - address = {Frederiksberg, Denmark}, - journal = {;login: The USENIX Magazine}, - month = {Feb}, - number = {1}, - volume = {36}, - url = {http://www.gnu.org/s/parallel}, - year = {2011}, - pages = {42-47} - doi = {10.5281/zenodo.16303} + title = {GNU Parallel - The Command-Line Power Tool}, + author = {O. Tange}, + address = {Frederiksberg, Denmark}, + journal = {;login: The USENIX Magazine}, + month = {Feb}, + number = {1}, + volume = {36}, + url = {http://www.gnu.org/s/parallel}, + year = {2011}, + pages = {42-47} + doi = {10.5281/zenodo.16303} } (Feel free to use \nocite{Tange2011a}) @@ -989,4 +997,4 @@ C \nice -n17 /bin/bash -c echo\ A \nice -n17 /bin/bash -c echo\ B \nice -n17 /bin/bash -c echo\ C -7 +6