Fixed (for --retries) bug #50139: Replacement strings in other options.

doc/release_new_version

@@ -208,9 +208,9 @@ file:///home/tange/privat/parallel/doc/release_new_version
 from:tange@gnu.org
 to:parallel@gnu.org, bug-parallel@gnu.org
 
-Subject: GNU Parallel 20170222 ('<<>>') released <<[stable]>>
+Subject: GNU Parallel 20170222 ('13769') released <<[stable]>>
 
-GNU Parallel 20170222 ('<<>>') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/
+GNU Parallel 20170222 ('13769') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/
 
 <<No new functionality was introduced so this is a good candidate for a stable release.>>
 
@@ -221,8 +221,15 @@ Haiku of the month:
 
 New in this release:
 
+https://www.linux.org.ru/forum/development/13190247
+http://coreygoldberg.blogspot.dk/2017/02/batch-convert-images-from-png-to-jpeg.html
+
+http://tomkimpson.com/howto/gnuparallel/
 http://workshop2016.iwslt.org/downloads/IWSLT_2016_paper_27.pdf
 http://aip.scitation.org/doi/full/10.1063/1.4974021
+http://fowlerlab.org/2017/01/26/accelerating-oxford-nanopore-basecalling/
+http://rockefeller.jobs/new-york-ny/devops-engineersenior-systems-administrator/2DB3408EAC9E4617AD238550F31C7641/job/
+
 
 
 * <<Possibly http://link.springer.com/chapter/10.1007%2F978-3-319-22053-6_46>>

src/parallel

@@ -921,7 +921,7 @@ sub options_hash {
 	 # Termination and retries
 	 "halt-on-error|halt=s" => \$opt::halt,
 	 "memfree=s" => \$opt::memfree,
-	 "retries=i" => \$opt::retries,
+	 "retries=s" => \$opt::retries,
 	 "timeout=s" => \$opt::timeout,
 	 "termseq|term-seq=s" => \$opt::termseq,
 	 # xargs-compatibility - implemented, man, testsuite
@@ -1023,7 +1023,9 @@ sub parse_options {
     if(defined $opt::silent) { $Global::verbose = 0; }
     if(defined $opt::0) { $/ = "\0"; }
     if(defined $opt::d) { $/ = unquote_printf($opt::d) }
-    if(defined $opt::tagstring) { $opt::tagstring = unquote_printf($opt::tagstring); }
+    if(defined $opt::tagstring) {
+	$opt::tagstring = unquote_printf($opt::tagstring);
+    }
     if(defined $opt::interactive) { $Global::interactive = $opt::interactive; }
     if(defined $opt::q) { $Global::quoting = 1; }
     if(defined $opt::r) { $Global::ignore_empty = 1; }
@@ -1031,18 +1033,28 @@ sub parse_options {
     parse_replacement_string_options();
     if(defined $opt::eof) { $Global::end_of_file_string = $opt::eof; }
     if(defined $opt::max_args) { $Global::max_number_of_args = $opt::max_args; }
-    if(defined $opt::timeout) { $Global::timeoutq = TimeoutQueue->new($opt::timeout); }
+    if(defined $opt::timeout) {
+	$Global::timeoutq = TimeoutQueue->new($opt::timeout);
+    }
     if(defined $opt::tmpdir) { $ENV{'TMPDIR'} = $opt::tmpdir; }
     $opt::nice ||= 0;
     if(defined $opt::help) { usage(); exit(0); }
-    if(defined $opt::sqlandworker) { $opt::sqlmaster = $opt::sqlworker = $opt::sqlandworker; }
+    if(defined $opt::sqlandworker) {
+	$opt::sqlmaster = $opt::sqlworker = $opt::sqlandworker;
+    }
     if(defined $opt::tmuxpane) { $opt::tmux = $opt::tmuxpane; }
     if(defined $opt::colsep) { $Global::trim = 'lr'; }
-    if(defined $opt::header) { $opt::colsep = defined $opt::colsep ? $opt::colsep : "\t"; }
+    if(defined $opt::header) {
+	$opt::colsep = defined $opt::colsep ? $opt::colsep : "\t";
+    }
     if(defined $opt::trim) { $Global::trim = $opt::trim; }
     if(defined $opt::arg_sep) { $Global::arg_sep = $opt::arg_sep; }
-    if(defined $opt::arg_file_sep) { $Global::arg_file_sep = $opt::arg_file_sep; }
+    if(defined $opt::arg_file_sep) {
-    if(defined $opt::number_of_cpus) { print SSHLogin::no_of_cpus(),"\n"; wait_and_exit(0); }
+	$Global::arg_file_sep = $opt::arg_file_sep;
+    }
+    if(defined $opt::number_of_cpus) {
+	print SSHLogin::no_of_cpus(),"\n"; wait_and_exit(0);
+    }
     if(defined $opt::number_of_cores) {
         print SSHLogin::no_of_cores(),"\n"; wait_and_exit(0);
     }
@@ -1056,8 +1068,10 @@ sub parse_options {
     if(@opt::sshlogin) { @Global::sshlogin = @opt::sshlogin; }
     if(@opt::sshloginfile) { read_sshloginfiles(@opt::sshloginfile); }
     if(@opt::return) { push @Global::ret_files, @opt::return; }
-    if($opt::transfer) { push @Global::transfer_files, $opt::i || $opt::I || "{}"; }
+    if($opt::transfer) {
-    if(@opt::transfer_files) { push @Global::transfer_files, @opt::transfer_files; }
+	push @Global::transfer_files, $opt::i || $opt::I || "{}";
+    }
+    push @Global::transfer_files, @opt::transfer_files;
     if(not defined $opt::recstart and
        not defined $opt::recend) { $opt::recend = "\n"; }
     $Global::blocksize = multiply_binary_prefix($opt::blocksize || "1M");
@@ -1185,12 +1199,8 @@ sub parse_options {
     }
     parse_semaphore();
 
-    if(defined $opt::eta) {
+    if(defined $opt::eta) { $opt::progress = $opt::eta; }
-        $opt::progress = $opt::eta;
+    if(defined $opt::bar) { $opt::progress = $opt::bar; }
-    }
-    if(defined $opt::bar) {
-        $opt::progress = $opt::bar;
-    }
     citation_notice();
 
     parse_halt();
@@ -1203,17 +1213,13 @@ sub parse_options {
         ::warning("Using -X or -m with --sshlogin may fail.");
     }
 
-    if(not defined $opt::jobs) {
+    if(not defined $opt::jobs) { $opt::jobs = "100%"; }
-        $opt::jobs = "100%";
-    }
     open_joblog();
     open_csv();
     if($opt::sqlmaster or $opt::sqlworker) {
 	$Global::sql = SQL->new($opt::sqlmaster || $opt::sqlworker);
     }
-    if($opt::sqlworker) {
+    if($opt::sqlworker) { $Global::membuffer ||= 1; }
-	$Global::membuffer ||= 1;
-    }
 }
 
 sub check_invalid_option_combinations {
@@ -1249,7 +1255,8 @@ sub check_invalid_option_combinations {
 	::wait_and_exit(255);
     }
     if(defined $opt::tollef and not $opt::gnu) {
-       ::error("--tollef has been retired.","Remove --tollef or use --gnu to override --tollef.");
+       ::error("--tollef has been retired.",
+	       "Remove --tollef or use --gnu to override --tollef.");
        ::wait_and_exit(255);
     }
     if(defined $opt::retired) {
@@ -1283,7 +1290,9 @@ sub init_globals {
 	 '{#}'  => '1 $_=$job->seq()',
 	 '{%}'  => '1 $_=$job->slot()',
 	 '{/}'  => 's:.*/::',
-	 '{//}' => '$Global::use{"File::Basename"} ||= eval "use File::Basename; 1;"; $_ = dirname($_);',
+	 '{//}' =>
+	 ('$Global::use{"File::Basename"} ||= eval "use File::Basename; 1;"; '.
+	  '$_ = dirname($_);'),
 	 '{/.}' => 's:.*/::; s:\.[^/.]+$::;',
 	 '{.}'  => 's:\.[^/.]+$::',
 	);
@@ -2321,7 +2330,9 @@ sub init_run_jobs {
 		debug("run", "Running jobs before on ", $sshlogin->string(), ": ",
 		      $sshlogin->jobs_running(), "\n");
 		if ($sshlogin->jobs_running() < $sshlogin->max_jobs_running()) {
-		    if($opt::delay and $opt::delay > ::now() - $Global::newest_starttime) {
+		    if($opt::delay
+		       and
+		       $opt::delay > ::now() - $Global::newest_starttime) {
 			# It has been too short since last start
 			next;
 		    }
@@ -2342,12 +2353,14 @@ sub init_run_jobs {
 			# It has been too short since
 			next;
 		    }
-		    debug("run", $sshlogin->string(), " has ", $sshlogin->jobs_running(),
+		    debug("run", $sshlogin->string(),
+			  " has ", $sshlogin->jobs_running(),
 			  " out of ", $sshlogin->max_jobs_running(),
 			  " jobs running. Start another.\n");
 		    if(start_another_job($sshlogin) == 0) {
 			# No more jobs to start on this $sshlogin
-			debug("run","No jobs started on ", $sshlogin->string(), "\n");
+			debug("run","No jobs started on ",
+			      $sshlogin->string(), "\n");
 			next;
 		    }
 		    $sshlogin->inc_jobs_running();
@@ -2768,7 +2781,8 @@ sub progress {
 	my($total, $completed, $left, $pctcomplete, $avgtime, $eta) =
 	    compute_eta();
 	my $arg = $Global::newest_job ?
-	    $Global::newest_job->{'commandline'}->replace_placeholders(["\257<\257>"],0,0) : "";
+	    $Global::newest_job->{'commandline'}->
+	    replace_placeholders(["\257<\257>"],0,0) : "";
 	# These chars mess up display in the terminal
 	$arg =~ tr/[\011-\016\033\302-\365]//d;
 	my $bar_text =
@@ -8079,7 +8093,9 @@ sub should_be_retried {
     } else {
 	# The job failed. Should it be retried?
 	$self->add_failed_here();
-	if($self->total_failed() == $opt::retries) {
+	my $retries = $self->{'commandline'}->
+	    replace_placeholders([$opt::retries],0,0);
+	if($self->total_failed() == $retries) {
 	    # This has been retried enough
 	    return 0;
 	} else {
@@ -9314,7 +9330,7 @@ sub new {
     # @command, --transferfile, --return,
     # --tagstring, --workdir, --results
     for(@command, @$transfer_files, @$return_files,
-	$opt::tagstring, $opt::workdir, $opt::results) {
+	$opt::tagstring, $opt::workdir, $opt::results, $opt::retries) {
 	# Skip if undefined
 	$_ or next;
 	# Disallow \257 to avoid nested {= {= =} =}
@@ -9458,7 +9474,7 @@ sub replacement_counts_and_lengths {
 	    $noncontextlen += length $c;
 	}
 	for(@$transfer_files, @$return_files,
-	    $opt::tagstring, $opt::workdir, $opt::results) {
+	    $opt::tagstring, $opt::workdir, $opt::results, $opt::retries) {
 	    # Options that can contain replacement strings
 	    $_ or next;
 	    my $t = $_;

src/parallel.pod

@@ -674,10 +674,15 @@ See also: B<--bar>, B<--progress>.
 
 =item B<--fg>
 
-Run command in foreground thus GNU B<parallel> will wait for
+Run command in foreground.
-completion of the command before exiting.
+
+With B<--tmux> and B<--tmuxpane> GNU B<parallel> will start B<tmux> in
+the foreground.
+
+With B<--semaphore> GNU B<parallel> will run the command in the
+foreground (opposite B<--bg>), and wait for completion of the command
+before exiting.
 
-Implies B<--semaphore>.
 
 See also B<--bg>, B<man sem>.
 
@@ -889,9 +894,13 @@ and output of bytes.
 
 To convert the times into ISO-8601 strict do:
 
-  perl -a -F"\t" -ne \
+  cat joblog | perl -a -F"\t" -ne \
     'chomp($F[2]=`date -d \@$F[2] +%FT%T`); print join("\t",@F)'
 
+If the host is long, you can use B<column -t> to pretty print it:
+
+  cat joblog | column -t
+
 See also B<--resume> B<--resume-failed>.
 
 

testsuite/tests-to-run/parallel-local-0.3s.sh

@@ -665,6 +665,13 @@ par_pipepart_block_bigger_2G() {
     rm $tmp
 }
 
+par_retries_replacement_string() {
+    tmp=$(mktemp)
+    parallel --retries {//} "echo {/} >>$tmp;exit {/}" ::: 1/11 2/22 3/33
+    sort $tmp
+    rm $tmp
+}
+
 export -f $(compgen -A function | grep par_)
 compgen -A function | grep par_ | sort |
     parallel -j6 --tag -k --joblog +/tmp/jl-`basename $0` '{} 2>&1'

testsuite/tests-to-run/parallel-local7.sh

@@ -10,7 +10,19 @@ par_tmux() {
     (stdout parallel --timeout 3 --tmux --delay 0.03 echo '{}{=$_="\\"x$_=}'; echo $?) | par_tmux_filter
 }
 export -f par_tmux
+
+# Does not work
+# cat >/tmp/parallel-local7-script <<EOF
+# stdout /usr/bin/time -f %e 
+# parallel --tmux --fg sleep ::: 1 2 3
+# parallel --tmuxpane --fg sleep ::: 1 2 3
+# EOF
+# chmod +x /tmp/parallel-local7-script
+# echo '### bug #48841: --tmux(pane) --fg should start tmux in foreground'
+# stdout /usr/bin/time -f %e script -q -f -c /tmp/parallel-local7-script /dev/null |  perl -ne '$_ >= 26 and $_ <= 45 and print "OK\n"'
+
 cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj3 --timeout 60 --retries 2 -k --joblog /tmp/jl-`basename $0` -L1
+
 echo '### tmux1.9'
   seq 000   100 | PARALLEL_TMUX=tmux1.9 par_tmux
   seq 100   200 | PARALLEL_TMUX=tmux1.9 par_tmux

testsuite/wanted-results/parallel-local-0.3s

@@ -1624,3 +1624,9 @@ par_file_ending_in_newline	'
 par_pipepart_block_bigger_2G	### Test that --pipepart can have blocks > 2GB
 par_pipepart_block_bigger_2G	      1       1       4
 par_python_children	### bug #49970: Python child process dies if --env is used
+par_retries_replacement_string	11
+par_retries_replacement_string	22
+par_retries_replacement_string	22
+par_retries_replacement_string	33
+par_retries_replacement_string	33
+par_retries_replacement_string	33