tange/parallel
1
0
Fork 0
mirror of https://git.savannah.gnu.org/git/parallel.git synced 2024-11-22 14:07:55 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fixed (for --retries) bug #50139: Replacement strings in other options.

Browse source
This commit is contained in:
Ole Tange 2017-02-04 16:01:42 +01:00
parent dae4f4b7ca
commit 09cc44e209
6 changed files with 93 additions and 36 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
doc/release_new_version
View file

@ -208,9 +208,9 @@ file:///home/tange/privat/parallel/doc/release_new_version
from:tange@gnu.org
to:parallel@gnu.org, bug-parallel@gnu.org
Subject: GNU Parallel 20170222 ('<<>>') released <<[stable]>>
Subject: GNU Parallel 20170222 ('13769') released <<[stable]>>
GNU Parallel 20170222 ('<<>>') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/
GNU Parallel 20170222 ('13769') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/
<<No new functionality was introduced so this is a good candidate for a stable release.>>
@ -221,8 +221,15 @@ Haiku of the month:
New in this release:
https://www.linux.org.ru/forum/development/13190247
http://coreygoldberg.blogspot.dk/2017/02/batch-convert-images-from-png-to-jpeg.html
http://tomkimpson.com/howto/gnuparallel/
http://workshop2016.iwslt.org/downloads/IWSLT_2016_paper_27.pdf
http://aip.scitation.org/doi/full/10.1063/1.4974021
http://fowlerlab.org/2017/01/26/accelerating-oxford-nanopore-basecalling/
http://rockefeller.jobs/new-york-ny/devops-engineersenior-systems-administrator/2DB3408EAC9E4617AD238550F31C7641/job/
* <<Possibly http://link.springer.com/chapter/10.1007%2F978-3-319-22053-6_46>>

76
src/parallel
View file

@ -921,7 +921,7 @@ sub options_hash {
# Termination and retries
"halt-on-error|halt=s" => \$opt::halt,
"memfree=s" => \$opt::memfree,
"retries=i" => \$opt::retries,
"retries=s" => \$opt::retries,
"timeout=s" => \$opt::timeout,
"termseq|term-seq=s" => \$opt::termseq,
# xargs-compatibility - implemented, man, testsuite
@ -1023,7 +1023,9 @@ sub parse_options {
if(defined $opt::silent) { $Global::verbose = 0; }
if(defined $opt::0) { $/ = "\0"; }
if(defined $opt::d) { $/ = unquote_printf($opt::d) }
if(defined $opt::tagstring) { $opt::tagstring = unquote_printf($opt::tagstring); }
if(defined $opt::tagstring) {
$opt::tagstring = unquote_printf($opt::tagstring);
}
if(defined $opt::interactive) { $Global::interactive = $opt::interactive; }
if(defined $opt::q) { $Global::quoting = 1; }
if(defined $opt::r) { $Global::ignore_empty = 1; }
@ -1031,18 +1033,28 @@ sub parse_options {
parse_replacement_string_options();
if(defined $opt::eof) { $Global::end_of_file_string = $opt::eof; }
if(defined $opt::max_args) { $Global::max_number_of_args = $opt::max_args; }
if(defined $opt::timeout) { $Global::timeoutq = TimeoutQueue->new($opt::timeout); }
if(defined $opt::timeout) {
$Global::timeoutq = TimeoutQueue->new($opt::timeout);
}
if(defined $opt::tmpdir) { $ENV{'TMPDIR'} = $opt::tmpdir; }
$opt::nice ||= 0;
if(defined $opt::help) { usage(); exit(0); }
if(defined $opt::sqlandworker) { $opt::sqlmaster = $opt::sqlworker = $opt::sqlandworker; }
if(defined $opt::sqlandworker) {
$opt::sqlmaster = $opt::sqlworker = $opt::sqlandworker;
}
if(defined $opt::tmuxpane) { $opt::tmux = $opt::tmuxpane; }
if(defined $opt::colsep) { $Global::trim = 'lr'; }
if(defined $opt::header) { $opt::colsep = defined $opt::colsep ? $opt::colsep : "\t"; }
if(defined $opt::header) {
$opt::colsep = defined $opt::colsep ? $opt::colsep : "\t";
}
if(defined $opt::trim) { $Global::trim = $opt::trim; }
if(defined $opt::arg_sep) { $Global::arg_sep = $opt::arg_sep; }
if(defined $opt::arg_file_sep) { $Global::arg_file_sep = $opt::arg_file_sep; }
if(defined $opt::number_of_cpus) { print SSHLogin::no_of_cpus(),"\n"; wait_and_exit(0); }
if(defined $opt::arg_file_sep) {
$Global::arg_file_sep = $opt::arg_file_sep;
}
if(defined $opt::number_of_cpus) {
print SSHLogin::no_of_cpus(),"\n"; wait_and_exit(0);
}
if(defined $opt::number_of_cores) {
print SSHLogin::no_of_cores(),"\n"; wait_and_exit(0);
}
@ -1056,8 +1068,10 @@ sub parse_options {
if(@opt::sshlogin) { @Global::sshlogin = @opt::sshlogin; }
if(@opt::sshloginfile) { read_sshloginfiles(@opt::sshloginfile); }
if(@opt::return) { push @Global::ret_files, @opt::return; }
if($opt::transfer) { push @Global::transfer_files, $opt::i || $opt::I || "{}"; }
if(@opt::transfer_files) { push @Global::transfer_files, @opt::transfer_files; }
if($opt::transfer) {
push @Global::transfer_files, $opt::i || $opt::I || "{}";
}
push @Global::transfer_files, @opt::transfer_files;
if(not defined $opt::recstart and
not defined $opt::recend) { $opt::recend = "\n"; }
$Global::blocksize = multiply_binary_prefix($opt::blocksize || "1M");
@ -1185,12 +1199,8 @@ sub parse_options {
}
parse_semaphore();
if(defined $opt::eta) {
$opt::progress = $opt::eta;
}
if(defined $opt::bar) {
$opt::progress = $opt::bar;
}
if(defined $opt::eta) { $opt::progress = $opt::eta; }
if(defined $opt::bar) { $opt::progress = $opt::bar; }
citation_notice();
parse_halt();
@ -1203,17 +1213,13 @@ sub parse_options {
::warning("Using -X or -m with --sshlogin may fail.");
}
if(not defined $opt::jobs) {
$opt::jobs = "100%";
}
if(not defined $opt::jobs) { $opt::jobs = "100%"; }
open_joblog();
open_csv();
if($opt::sqlmaster or $opt::sqlworker) {
$Global::sql = SQL->new($opt::sqlmaster || $opt::sqlworker);
}
if($opt::sqlworker) {
$Global::membuffer ||= 1;
}
if($opt::sqlworker) { $Global::membuffer ||= 1; }
}
sub check_invalid_option_combinations {
@ -1249,7 +1255,8 @@ sub check_invalid_option_combinations {
::wait_and_exit(255);
}
if(defined $opt::tollef and not $opt::gnu) {
::error("--tollef has been retired.","Remove --tollef or use --gnu to override --tollef.");
::error("--tollef has been retired.",
"Remove --tollef or use --gnu to override --tollef.");
::wait_and_exit(255);
}
if(defined $opt::retired) {
@ -1283,7 +1290,9 @@ sub init_globals {
'{#}' => '1 $_=$job->seq()',
'{%}' => '1 $_=$job->slot()',
'{/}' => 's:.*/::',
'{//}' => '$Global::use{"File::Basename"} ||= eval "use File::Basename; 1;"; $_ = dirname($_);',
'{//}' =>
('$Global::use{"File::Basename"} ||= eval "use File::Basename; 1;"; '.
'$_ = dirname($_);'),
'{/.}' => 's:.*/::; s:\.[^/.]+$::;',
'{.}' => 's:\.[^/.]+$::',
);
@ -2321,7 +2330,9 @@ sub init_run_jobs {
debug("run", "Running jobs before on ", $sshlogin->string(), ": ",
$sshlogin->jobs_running(), "\n");
if ($sshlogin->jobs_running() < $sshlogin->max_jobs_running()) {
if($opt::delay and $opt::delay > ::now() - $Global::newest_starttime) {
if($opt::delay
and
$opt::delay > ::now() - $Global::newest_starttime) {
# It has been too short since last start
next;
}
@ -2342,12 +2353,14 @@ sub init_run_jobs {
# It has been too short since
next;
}
debug("run", $sshlogin->string(), " has ", $sshlogin->jobs_running(),
debug("run", $sshlogin->string(),
" has ", $sshlogin->jobs_running(),
" out of ", $sshlogin->max_jobs_running(),
" jobs running. Start another.\n");
if(start_another_job($sshlogin) == 0) {
# No more jobs to start on this $sshlogin
debug("run","No jobs started on ", $sshlogin->string(), "\n");
debug("run","No jobs started on ",
$sshlogin->string(), "\n");
next;
}
$sshlogin->inc_jobs_running();
@ -2768,7 +2781,8 @@ sub progress {
my($total, $completed, $left, $pctcomplete, $avgtime, $eta) =
compute_eta();
my $arg = $Global::newest_job ?
$Global::newest_job->{'commandline'}->replace_placeholders(["\257<\257>"],0,0) : "";
$Global::newest_job->{'commandline'}->
replace_placeholders(["\257<\257>"],0,0) : "";
# These chars mess up display in the terminal
$arg =~ tr/[\011-\016\033\302-\365]//d;
my $bar_text =
@ -8079,7 +8093,9 @@ sub should_be_retried {
} else {
# The job failed. Should it be retried?
$self->add_failed_here();
if($self->total_failed() == $opt::retries) {
my $retries = $self->{'commandline'}->
replace_placeholders([$opt::retries],0,0);
if($self->total_failed() == $retries) {
# This has been retried enough
return 0;
} else {
@ -9314,7 +9330,7 @@ sub new {
# @command, --transferfile, --return,
# --tagstring, --workdir, --results
for(@command, @$transfer_files, @$return_files,
$opt::tagstring, $opt::workdir, $opt::results) {
$opt::tagstring, $opt::workdir, $opt::results, $opt::retries) {
# Skip if undefined
$_ or next;
# Disallow \257 to avoid nested {= {= =} =}
@ -9458,7 +9474,7 @@ sub replacement_counts_and_lengths {
$noncontextlen += length $c;
}
for(@$transfer_files, @$return_files,
$opt::tagstring, $opt::workdir, $opt::results) {
$opt::tagstring, $opt::workdir, $opt::results, $opt::retries) {
# Options that can contain replacement strings
$_ or next;
my $t = $_;

17
src/parallel.pod
View file

@ -674,10 +674,15 @@ See also: B<--bar>, B<--progress>.
=item B<--fg>
Run command in foreground thus GNU B<parallel> will wait for
completion of the command before exiting.
Run command in foreground.
With B<--tmux> and B<--tmuxpane> GNU B<parallel> will start B<tmux> in
the foreground.
With B<--semaphore> GNU B<parallel> will run the command in the
foreground (opposite B<--bg>), and wait for completion of the command
before exiting.
Implies B<--semaphore>.
See also B<--bg>, B<man sem>.
@ -889,9 +894,13 @@ and output of bytes.
To convert the times into ISO-8601 strict do:
perl -a -F"\t" -ne \
cat joblog | perl -a -F"\t" -ne \
'chomp($F[2]=`date -d \@$F[2] +%FT%T`); print join("\t",@F)'
If the host is long, you can use B<column -t> to pretty print it:
cat joblog | column -t
See also B<--resume> B<--resume-failed>.

7
testsuite/tests-to-run/parallel-local-0.3s.sh
View file

@ -665,6 +665,13 @@ par_pipepart_block_bigger_2G() {
rm $tmp
}
par_retries_replacement_string() {
tmp=$(mktemp)
parallel --retries {//} "echo {/} >>$tmp;exit {/}" ::: 1/11 2/22 3/33
sort $tmp
rm $tmp
}
export -f $(compgen -A function | grep par_)
compgen -A function | grep par_ | sort |
parallel -j6 --tag -k --joblog +/tmp/jl-`basename $0` '{} 2>&1'

12
testsuite/tests-to-run/parallel-local7.sh
View file

@ -10,7 +10,19 @@ par_tmux() {
(stdout parallel --timeout 3 --tmux --delay 0.03 echo '{}{=$_="\\"x$_=}'; echo $?) | par_tmux_filter
}
export -f par_tmux
# Does not work
# cat >/tmp/parallel-local7-script <<EOF
# stdout /usr/bin/time -f %e
# parallel --tmux --fg sleep ::: 1 2 3
# parallel --tmuxpane --fg sleep ::: 1 2 3
# EOF
# chmod +x /tmp/parallel-local7-script
# echo '### bug #48841: --tmux(pane) --fg should start tmux in foreground'
# stdout /usr/bin/time -f %e script -q -f -c /tmp/parallel-local7-script /dev/null | perl -ne '$_ >= 26 and $_ <= 45 and print "OK\n"'
cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj3 --timeout 60 --retries 2 -k --joblog /tmp/jl-`basename $0` -L1
echo '### tmux1.9'
seq 000 100 | PARALLEL_TMUX=tmux1.9 par_tmux
seq 100 200 | PARALLEL_TMUX=tmux1.9 par_tmux

6
testsuite/wanted-results/parallel-local-0.3s
View file

@ -1624,3 +1624,9 @@ par_file_ending_in_newline '
par_pipepart_block_bigger_2G ### Test that --pipepart can have blocks > 2GB
par_pipepart_block_bigger_2G 1 1 4
par_python_children ### bug #49970: Python child process dies if --env is used
par_retries_replacement_string 11
par_retries_replacement_string 22
par_retries_replacement_string 22
par_retries_replacement_string 33
par_retries_replacement_string 33
par_retries_replacement_string 33