tange/parallel
1
0
Fork 0
mirror of https://git.savannah.gnu.org/git/parallel.git synced 2024-11-22 14:07:55 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

parallel: Fixed bug #44995: parallel echo {#} ::: 1 2 ::: 1 2.

Browse source 
Passes testsuite.
This commit is contained in:
Ole Tange 2015-05-03 01:22:34 +02:00
parent 8110572719
commit fc0c6cee08
6 changed files with 114 additions and 81 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
doc/release_new_version
View file

@ -226,13 +226,14 @@ New in this release:
* <<afventer opdatering>> CIDER: a pipeline for detecting waves of coordinated transcriptional regulation in gene expression time-course data http://biorxiv.org/content/biorxiv/early/2015/03/17/012518.full.pdf
* <<afventer opdatering>> GNU Parallel was used (unfortunately without citation) in: MUGBAS: a species free gene-based programme suite for post-GWAS analysis http://www.ncbi.nlm.nih.gov/pubmed/25765345
taxator-tk http://algbio.cs.uni-duesseldorf.de/webapps/wa-download/ (check it)
* << afventer svar fra Rachel >> GNU Parallel was used in: SISRS: Site Identification from Short Read Sequences https://github.com/rachelss/SISRS/
* GNU Parallel was cited in: Toward Enhanced Metadata Quality of Large-Scale Digital Libraries: Estimating Volume Time Range https://www.ideals.illinois.edu/bitstream/handle/2142/73656/186_ready.pdf
* GNU Parallel was cited in: Sequencing the cap-snatching repertoire of H1N1 influenza provides insight into the mechanism of viral transcription initiation http://nar.oxfordjournals.org/content/early/2015/04/20/nar.gkv333.full.pdf
* GNU Parallel was cited in: Genome assemblyusing Nanopore-guided long and error-free DNA reads http://www.biomedcentral.com/content/pdf/s12864-015-1519-z.pdf
@ -247,6 +248,10 @@ taxator-tk http://algbio.cs.uni-duesseldorf.de/webapps/wa-download/ (check it)
* Functions and GNU parallel for effective cluster load management http://genomespot.blogspot.dk/2015/04/functions-and-gnu-parallel-for.html
* Use parallel processing to save time importing databases http://drupalsun.com/node/41854
* Run multiple ssh commands in parallel with GNU Parallel http://www.ameir.net/blog/archives/380-run-multiple-ssh-commands-in-parallel-with-gnu-parallel.html
* Bug fixes and man page updates.
GNU Parallel - For people who live life in the parallel lane.

82
src/parallel
View file

@ -1053,7 +1053,7 @@ sub parse_options {
sub init_globals {
# Defaults:
$Global::version = 20150424;
$Global::version = 20150426;
$Global::progname = 'parallel';
$Global::infinity = 2**31;
$Global::debug = 0;
@ -3032,8 +3032,11 @@ sub reaper {
next;
}
my $job = $Global::running{$stiff};
# '-a <(seq 10)' will give us a pid not in %Global::running
$job or next;
delete $Global::running{$stiff};
$Global::total_running--;
$job->set_exitstatus($? >> 8);
$job->set_exitsignal($? & 127);
debug("run", "seq ",$job->seq()," died (", $job->exitstatus(), ")");
@ -3042,7 +3045,9 @@ sub reaper {
# The process that died had the tty => release it
$Global::tty_taken = 0;
}
my $sshlogin = $job->sshlogin();
$sshlogin->dec_jobs_running();
$sshlogin->inc_jobs_completed();
if(not $job->should_be_retried()) {
# The job is done
# Free the jobslot
@ -3057,15 +3062,10 @@ sub reaper {
$job->print();
}
if($job->should_we_halt() eq "now") {
::killall();
::wait_and_exit($Global::exitstatus);
::killall();
::wait_and_exit($Global::halt_exitstatus);
}
}
my $sshlogin = $job->sshlogin();
$sshlogin->dec_jobs_running();
$sshlogin->inc_jobs_completed();
$Global::total_running--;
delete $Global::running{$stiff};
start_more_jobs();
if($opt::progress) {
my %progress = progress();
@ -7490,9 +7490,16 @@ sub set_exitsignal {
$Global::halt_pct <=
$Global::total_failed / $Global::total_started
and $Global::total_failed > 3)) {
# More than N jobs or more than N% failed
$Global::halt_exitstatus = $job->exitstatus();
if($Global::halt_when eq "soon") {
# At least N jobs had failed
# or at least N% had failed and more than 3
if($Global::halt_count and $Global::halt_count == 1) {
$Global::halt_exitstatus = $job->exitstatus();
} else {
$Global::halt_exitstatus = $Global::total_failed;
}
::debug("halt","Pct: ",$Global::halt_pct,"<=",$Global::total_failed / $Global::total_started," count: ",$Global::halt_count,"\n");
if($Global::halt_when eq "soon"
and scalar(keys %Global::running) > 0) {
::status
("$Global::progname: Starting no more jobs. ",
"Waiting for ", scalar(keys %Global::running),
@ -7504,19 +7511,23 @@ sub set_exitsignal {
}
} else {
if($Global::halt_success) {
$Global::halt_exitstatus = $Global::total_failed;
::status("$Global::progname: This job succeeded:\n",
$job->replaced(),"\n");
if(($Global::halt_count and
$Global::halt_count <=
1+$Global::total_completed-$Global::total_failed)
$Global::total_completed-$Global::total_failed)
or
($Global::halt_pct and
$Global::halt_pct <=
(1+$Global::total_completed-$Global::total_failed)
($Global::total_completed-$Global::total_failed)
/ $Global::total_completed
and ($Global::total_completed-$Global::total_failed) > 3)) {
$Global::halt_exitstatus = $job->exitstatus();
if($Global::halt_when eq "soon") {
# At least N jobs had success
# or at least N% had success and more than 3
$Global::halt_exitstatus = 0;
if($Global::halt_when eq "soon"
and scalar(keys %Global::running) > 0) {
::status
("$Global::progname: Starting no more jobs. ",
"Waiting for ", scalar(keys %Global::running),
@ -8845,18 +8856,20 @@ sub new {
}, ref($class) || $class;
}
sub replace {
# Calculates the corresponding value for a given perl expression
# Returns:
# The calculated string (quoted if asked for)
my $self = shift;
my $perlexpr = shift; # E.g. $_=$_ or s/.gz//
my $quote = (shift) ? 1 : 0; # should the string be quoted?
# This is actually a CommandLine-object,
# but it looks nice to be able to say {= $job->slot() =}
my $job = shift;
$perlexpr =~ s/^-?\d+ //; # Positional replace treated as normal replace
if(not defined $self->{"rpl",0,$perlexpr}) {
{
my %perleval;
sub replace {
# Calculates the corresponding value for a given perl expression
# Returns:
# The calculated string (quoted if asked for)
my $self = shift;
my $perlexpr = shift; # E.g. $_=$_ or s/.gz//
my $quote = (shift) ? 1 : 0; # should the string be quoted?
# This is actually a CommandLine-object,
# but it looks nice to be able to say {= $job->slot() =}
my $job = shift;
$perlexpr =~ s/^-?\d+ +//; # Positional replace treated as normal replace
local $_;
if($Global::trim eq "n") {
$_ = $self->{'orig'};
@ -8864,10 +8877,10 @@ sub replace {
$_ = trim_of($self->{'orig'});
}
::debug("replace", "eval ", $perlexpr, " ", $_, "\n");
if(not $Global::perleval{$perlexpr}) {
if(not $perleval{$perlexpr}) {
# Make an anonymous function of the $perlexpr
# And more importantly: Compile it only once
if($Global::perleval{$perlexpr} =
if($perleval{$perlexpr} =
eval('sub { no strict; no warnings; my $job = shift; '.
$perlexpr.' }')) {
# All is good
@ -8878,14 +8891,9 @@ sub replace {
}
}
# Execute the function
$Global::perleval{$perlexpr}->($job);
$self->{"rpl",0,$perlexpr} = $_;
$perleval{$perlexpr}->($job);
return $quote ? ::shell_quote_scalar($_) : $_;
}
if(not defined $self->{"rpl",$quote,$perlexpr}) {
$self->{"rpl",1,$perlexpr} =
::shell_quote_scalar($self->{"rpl",0,$perlexpr});
}
return $self->{"rpl",$quote,$perlexpr};
}
sub orig {

13
src/sem.pod
View file

@ -161,10 +161,17 @@ available.
pod2html creates two files: pod2htmd.tmp and pod2htmi.tmp which it
does not clean up. It uses these two files for a short time. But if
you run multiple pod2html in parallel (e.g. in a Makefile with make
-j) you need to protect pod2html from running twice at the same
time. B<sem> running as a mutex will do just that:
-j) there is a risk that two different instances of pod2html will
write to the files at the same time:
sem --fg --id pod2html pod2html foo.pod > foo.html
# This may fail due to shared pod2htmd.tmp/pod2htmi.tmp files
pod2html foo.pod --outfile foo.html & pod2html bar.pod --outfile bar.html
You need to protect pod2html from running twice at the same time.
B<sem> running as a mutex will make sure only one runs:
sem --id pod2html pod2html foo.pod --outfile foo.html
sem --id pod2html pod2html bar.pod --outfile bar.html
sem --fg --id pod2html rm -f pod2htmd.tmp pod2htmi.tmp

5
testsuite/tests-to-run/parallel-local-3s.sh
View file

@ -14,6 +14,7 @@ echo '**'
echo '### Test --halt-on-error 0';
(echo "sleep 1;true"; echo "sleep 2;false";echo "sleep 3;true") | parallel -j10 --halt-on-error 0;
echo $?;
(echo "sleep 1;true"; echo "sleep 2;false";echo "sleep 3;true";echo "sleep 4; non_exist") | parallel -j10 --halt 0;
echo $?
@ -22,6 +23,7 @@ echo '**'
echo '### Test --halt-on-error 1';
(echo "sleep 1;true"; echo "sleep 2;false";echo "sleep 3;true") | parallel -j10 --halt-on-error 1;
echo $?;
(echo "sleep 1;true"; echo "sleep 2;false";echo "sleep 3;true";echo "sleep 4; non_exist") | parallel -j10 --halt 1;
echo $?
@ -30,6 +32,7 @@ echo '**'
echo '### Test --halt-on-error 2';
(echo "sleep 1;true"; echo "sleep 2;false";echo "sleep 3;true") | parallel -j10 --halt-on-error 2;
echo $?;
(echo "sleep 1;true"; echo "sleep 2;false";echo "sleep 3;true";echo "sleep 4; non_exist") | parallel -j10 --halt 2;
echo $?
@ -38,6 +41,7 @@ echo '**'
echo '### Test --halt -1';
(echo "sleep 1;false"; echo "sleep 2;true";echo "sleep 3;false") | parallel -j10 --halt-on-error -1;
echo $?;
(echo "sleep 1;false"; echo "sleep 2;true";echo "sleep 3;false";echo "sleep 4; non_exist") | parallel -j10 --halt -1;
echo $?
@ -46,6 +50,7 @@ echo '**'
echo '### Test --halt -2';
(echo "sleep 1;false"; echo "sleep 2;true";echo "sleep 3;false") | parallel -j10 --halt-on-error -2;
echo $?;
(echo "sleep 1;false"; echo "sleep 2;true";echo "sleep 3;false";echo "sleep 4; non_exist") | parallel -j10 --halt -2;
echo $?

44
testsuite/wanted-results/parallel-local-3s
View file

@ -19,12 +19,13 @@ echo '### Test --halt-on-error 1'; (echo "sleep 1;true"; echo "sleep 2;false";
127
parallel: This job failed:
sleep 2;false
parallel: Starting no more jobs. Waiting for 3 jobs to finish. This job failed:
parallel: Starting no more jobs. Waiting for 1 jobs to finish.
parallel: This job failed:
sleep 2;false
parallel: Starting no more jobs. Waiting for 2 jobs to finish.
/bin/bash: non_exist: command not found
parallel: This job failed:
sleep 4; non_exist
parallel: Starting no more jobs. Waiting for 1 jobs to finish.
echo '**'
**
echo '### Test --halt-on-error 2'; (echo "sleep 1;true"; echo "sleep 2;false";echo "sleep 3;true") | parallel -j10 --halt-on-error 2; echo $?; (echo "sleep 1;true"; echo "sleep 2;false";echo "sleep 3;true";echo "sleep 4; non_exist") | parallel -j10 --halt 2; echo $?
@ -43,9 +44,10 @@ echo '### Test --halt -1'; (echo "sleep 1;false"; echo "sleep 2;true";echo "sl
0
parallel: This job succeeded:
sleep 2;true
parallel: Starting no more jobs. Waiting for 3 jobs to finish. This job succeeded:
parallel: Starting no more jobs. Waiting for 1 jobs to finish.
parallel: This job succeeded:
sleep 2;true
parallel: Starting no more jobs. Waiting for 3 jobs to finish.
parallel: Starting no more jobs. Waiting for 2 jobs to finish.
/bin/bash: non_exist: command not found
echo '**'
**
@ -66,40 +68,39 @@ exit code 9
1
parallel: This job failed:
perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 1
parallel: Starting no more jobs. Waiting for 9 jobs to finish.
parallel: Starting no more jobs. Waiting for 8 jobs to finish.
2
parallel: This job failed:
perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 2
parallel: Starting no more jobs. Waiting for 8 jobs to finish.
parallel: Starting no more jobs. Waiting for 7 jobs to finish.
3
parallel: This job failed:
perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 3
parallel: Starting no more jobs. Waiting for 7 jobs to finish.
parallel: Starting no more jobs. Waiting for 6 jobs to finish.
4
parallel: This job failed:
perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 4
parallel: Starting no more jobs. Waiting for 6 jobs to finish.
parallel: Starting no more jobs. Waiting for 5 jobs to finish.
5
parallel: This job failed:
perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 5
parallel: Starting no more jobs. Waiting for 5 jobs to finish.
parallel: Starting no more jobs. Waiting for 4 jobs to finish.
6
parallel: This job failed:
perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 6
parallel: Starting no more jobs. Waiting for 4 jobs to finish.
parallel: Starting no more jobs. Waiting for 3 jobs to finish.
7
parallel: This job failed:
perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 7
parallel: Starting no more jobs. Waiting for 3 jobs to finish.
parallel: Starting no more jobs. Waiting for 2 jobs to finish.
8
0
parallel: This job failed:
perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 8
parallel: Starting no more jobs. Waiting for 2 jobs to finish.
parallel: Starting no more jobs. Waiting for 1 jobs to finish.
9
parallel: This job failed:
perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 9
parallel: Starting no more jobs. Waiting for 1 jobs to finish.
echo '### Test last dying print --halt-on-error 2'; (seq 0 8;echo 0; echo 9) | parallel -j10 -kq --halt 2 perl -e 'sleep $ARGV[0];print STDERR @ARGV,"\n"; exit shift'; echo exit code $?
### Test last dying print --halt-on-error 2
exit code 1
@ -114,40 +115,39 @@ exit code 0
1
parallel: This job succeeded:
perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ not\ shift 1
parallel: Starting no more jobs. Waiting for 9 jobs to finish.
parallel: Starting no more jobs. Waiting for 8 jobs to finish.
2
parallel: This job succeeded:
perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ not\ shift 2
parallel: Starting no more jobs. Waiting for 8 jobs to finish.
parallel: Starting no more jobs. Waiting for 7 jobs to finish.
3
parallel: This job succeeded:
perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ not\ shift 3
parallel: Starting no more jobs. Waiting for 7 jobs to finish.
parallel: Starting no more jobs. Waiting for 6 jobs to finish.
4
parallel: This job succeeded:
perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ not\ shift 4
parallel: Starting no more jobs. Waiting for 6 jobs to finish.
parallel: Starting no more jobs. Waiting for 5 jobs to finish.
5
parallel: This job succeeded:
perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ not\ shift 5
parallel: Starting no more jobs. Waiting for 5 jobs to finish.
parallel: Starting no more jobs. Waiting for 4 jobs to finish.
6
parallel: This job succeeded:
perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ not\ shift 6
parallel: Starting no more jobs. Waiting for 4 jobs to finish.
parallel: Starting no more jobs. Waiting for 3 jobs to finish.
7
parallel: This job succeeded:
perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ not\ shift 7
parallel: Starting no more jobs. Waiting for 3 jobs to finish.
parallel: Starting no more jobs. Waiting for 2 jobs to finish.
8
0
parallel: This job succeeded:
perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ not\ shift 8
parallel: Starting no more jobs. Waiting for 2 jobs to finish.
parallel: Starting no more jobs. Waiting for 1 jobs to finish.
9
parallel: This job succeeded:
perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ not\ shift 9
parallel: Starting no more jobs. Waiting for 1 jobs to finish.
echo '### Test last dying print --halt-on-error -2'; (seq 0 8;echo 0; echo 9) | parallel -j10 -kq --halt -2 perl -e 'sleep $ARGV[0];print STDERR @ARGV,"\n"; exit not shift'; echo exit code $?
### Test last dying print --halt-on-error -2
exit code 0

44
testsuite/wanted-results/parallel-tutorial
View file

@ -478,9 +478,10 @@ Seq Host Starttime JobRuntime Send Receive Exitval Signal Command
0
1
2
parallel: Starting no more jobs. Waiting for 2 jobs to finish. This job failed:
parallel: This job failed:
echo 1; exit 1
parallel: Starting no more jobs. Waiting for 1 jobs to finish. This job failed:
parallel: Starting no more jobs. Waiting for 1 jobs to finish.
parallel: This job failed:
echo 2; exit 2
parallel -j2 --halt 2 echo {}\; exit {} ::: 0 0 1 2 3
0
@ -496,9 +497,16 @@ echo 1; exit 1
3
4
5
parallel: Starting no more jobs. Waiting for 2 jobs to finish. This job failed:
parallel: This job failed:
echo 1; exit 1
parallel: This job failed:
echo 2; exit 2
parallel: This job failed:
echo 3; exit 3
parallel: This job failed:
echo 4; exit 4
parallel: Starting no more jobs. Waiting for 1 jobs to finish. This job failed:
parallel: Starting no more jobs. Waiting for 1 jobs to finish.
parallel: This job failed:
echo 5; exit 5
parallel -k --retries 3 'echo tried {} >>/tmp/runs; echo completed {}; exit {}' ::: 1 2 0
cat /tmp/runs
@ -931,7 +939,7 @@ This helps funding further development; and it won't cost you a cent.
If you pay 10000 EUR you should feel free to use GNU Parallel without citing.
parallel --version
GNU parallel 20150422
GNU parallel 20150426
Copyright (C) 2007,2008,2009,2010,2011,2012,2013,2014,2015 Ole Tange
and Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
@ -943,7 +951,7 @@ Web site: http://www.gnu.org/software/parallel
When using programs that use GNU Parallel to process data for publication
please cite as described in 'parallel --bibtex'.
parallel --minversion 20130722 && echo Your version is at least 20130722.
20150422
20150426
Your version is at least 20130722.
parallel --bibtex
Academic tradition requires you to cite works you base your article on.
@ -951,17 +959,17 @@ When using programs that use GNU Parallel to process data for publication
please cite:
@article{Tange2011a,
title = {GNU Parallel - The Command-Line Power Tool},
author = {O. Tange},
address = {Frederiksberg, Denmark},
journal = {;login: The USENIX Magazine},
month = {Feb},
number = {1},
volume = {36},
url = {http://www.gnu.org/s/parallel},
year = {2011},
pages = {42-47}
doi = {10.5281/zenodo.16303}
title = {GNU Parallel - The Command-Line Power Tool},
author = {O. Tange},
address = {Frederiksberg, Denmark},
journal = {;login: The USENIX Magazine},
month = {Feb},
number = {1},
volume = {36},
url = {http://www.gnu.org/s/parallel},
year = {2011},
pages = {42-47}
doi = {10.5281/zenodo.16303}
}
(Feel free to use \nocite{Tange2011a})
@ -989,4 +997,4 @@ C
\nice -n17 /bin/bash -c echo\ A
\nice -n17 /bin/bash -c echo\ B
\nice -n17 /bin/bash -c echo\ C
7
6