Fixed bug #45841: Replacement string for total no of jobs.

src/parallel

@@ -1131,6 +1131,8 @@ sub init_globals {
 	 '{...}' => 's:\.[^/.]+$::; s:\.[^/.]+$::; s:\.[^/.]+$::',
 	 '{/..}' => 's:.*/::; s:\.[^/.]+$::; s:\.[^/.]+$::',
 	 '{/...}' => 's:.*/::; s:\.[^/.]+$::; s:\.[^/.]+$::; s:\.[^/.]+$::',
+	 # {##} = number of jobs
+	 '{##}' => '$_=$Global::JobQueue->total_jobs()',
 	);
     # Modifiable copy of %Global::replace
     %Global::rpl = %Global::replace;
@@ -5746,6 +5748,7 @@ sub new {
     return bless {
         'unget' => \@unget,
         'commandlinequeue' => $commandlinequeue,
+        'this_job_no' => 1,
         'total_jobs' => undef,
     }, ref($class) || $class;
 }
@@ -5754,11 +5757,13 @@ sub get {
     my $self = shift;
 
     if(@{$self->{'unget'}}) {
+	$self->{'this_job_no'}++;
         my $job = shift @{$self->{'unget'}};
         return ($job);
     } else {
         my $commandline = $self->{'commandlinequeue'}->get();
         if(defined $commandline) {
+	    $self->{'this_job_no'}++;
             my $job = Job->new($commandline);
             return $job;
         } else {
@@ -5770,6 +5775,7 @@ sub get {
 sub unget {
     my $self = shift;
     unshift @{$self->{'unget'}}, @_;
+    $self->{'this_job_no'} -= @_;
 }
 
 sub empty {
@@ -5793,7 +5799,8 @@ sub total_jobs {
 	my $start = time;
         while($record = $record_queue->get()) {
 	    if(time - $start > 10) {
-		::warning("Reading ".scalar(@arg_records)." arguments took longer than 10 seconds.");
+		::warning("Reading ".scalar(@arg_records).
+			  " arguments took longer than 10 seconds.");
 		$opt::eta && ::warning("Consider removing --eta.");
 		$opt::bar && ::warning("Consider removing --bar.");
 		$opt::shuf && ::warning("Consider removing --shuf.");
@@ -5812,7 +5819,7 @@ sub total_jobs {
 	    }
 	}
 	$record_queue->unget(@arg_records);
-        $self->{'total_jobs'} = $#arg_records+1;
+        $self->{'total_jobs'} = $#arg_records+1+$self->{'this_job_no'};
 	::debug("init","Total jobs: ".$self->{'total_jobs'}."\n");
     }
     return $self->{'total_jobs'};
@@ -6094,16 +6101,20 @@ sub filter_through_compress {
 	my ($fh, $comfile) = ::tmpfile(SUFFIX => ".pac");
 	close $fh;
 	# Compressor: (echo > $comfile; compress pipe) > output
-	# When the echo is written to $comfile, it is known that output file is opened,
+	# When the echo is written to $comfile,
+	# it is known that output file is opened,
 	# thus output file can then be removed by the decompressor.
-        my $wpid = open(my $fdw,"|-", "(echo > $comfile; ".empty_input_wrapper($opt::compress_program).") >".
+        my $wpid = open(my $fdw,"|-", "(echo > $comfile; ".
+			empty_input_wrapper($opt::compress_program).") >".
 			$self->fh($fdno,'name')) || die $?;
 	$self->set_fh($fdno,'w',$fdw);
 	$self->set_fh($fdno,'wpid',$wpid);
-	# Decompressor: open output; -s $comfile > 0: rm $comfile output; decompress output > stdout
+	# Decompressor: open output; -s $comfile > 0: rm $comfile output;
+	#               decompress output > stdout
 	my $rpid = open(my $fdr, "-|", "perl", "-e", $cattail, $comfile,
 			$opt::decompress_program, $wpid,
-			$self->fh($fdno,'name'),$self->fh($fdno,'unlink')) || die $?;
+			$self->fh($fdno,'name'),$self->fh($fdno,'unlink'))
+	    || die $?;
 	$self->set_fh($fdno,'r',$fdr);
 	$self->set_fh($fdno,'rpid',$rpid);
     }
@@ -6125,9 +6136,14 @@ sub max_file_name_length {
     # Figure out the max length of a subdir
     # TODO and the max total length
     # Ext4 = 255,130816
+    # Uses:
+    #   $Global::max_file_length is set
+    # Returns:
+    #   $Global::max_file_length
     my $testdir = shift;
 
     my $upper = 8_000_000;
+    # Dir length of 8 chars is supported everywhere
     my $len = 8;
     my $dir = "x"x$len;
     do {
@@ -7107,8 +7123,10 @@ sub start {
 	open ERR, '>&', $stderr_fh or ::die_bug("Can't dup STDERR: $!");
 
 	my $pid;
-	my @setpgrp_wrap = ('perl','-e',
+	my @setpgrp_wrap =
-			    "setpgrp\;eval\{setpriority\(0,0,$opt::nice\)\}\;exec '$Global::shell', '-c', \@ARGV");
+	    ('perl','-e',
+	     "setpgrp\;eval\{setpriority\(0,0,$opt::nice\)\}\;".
+	     "exec '$Global::shell', '-c', \@ARGV");
 	# The eval is needed to catch exception from open3
 	eval {
 	    $pid = ::open3($stdin_fh, ">&OUT", ">&ERR", @setpgrp_wrap, $command)
@@ -7169,7 +7187,7 @@ sub start {
 	# The eval is needed to catch exception from open3
 	eval {
 	    no warnings;
-	    local(*IN) = $devtty_fh;
+	    local (*IN) = $devtty_fh;
 	    local (*OUT,*ERR);
 	    open OUT, '>&', $stdout_fh or ::die_bug("Can't dup STDOUT: $!");
 	    open ERR, '>&', $stderr_fh or ::die_bug("Can't dup STDERR: $!");
@@ -7191,7 +7209,8 @@ sub start {
 	*IN = *STDIN;
 	# The eval is needed to catch exception from open3
 	my @setpgrp_wrap = ('perl','-e',
-			    "setpgrp\;eval\{setpriority\(0,0,$opt::nice\)\}\;exec '$Global::shell', '-c', \@ARGV");
+			    "setpgrp\;eval\{setpriority\(0,0,$opt::nice\)\}\;".
+			    "exec '$Global::shell', '-c', \@ARGV");
 	eval {
 	    $pid = ::open3("<&IN", ">&OUT", ">&ERR", @setpgrp_wrap, $command)
 		|| ::die_bug("open3-<IN");
@@ -7394,7 +7413,7 @@ sub should_be_retried {
     my (%print_later,$job_end_sequence);
 
     sub print_earlier_jobs {
-	# Print jobs completed earlier
+	# Print jobs whose output is postponed due to --keep-order
 	# Returns: N/A
 	my $job = shift;
 	$print_later{$job->seq()} = $job;
@@ -7903,12 +7922,9 @@ sub populate {
     my $next_arg;
     my $max_len = $Global::minimal_command_line_length || Limits::Command::max_length();
 
-    if($opt::cat) {
+    if($opt::cat or $opt::fifo) {
-	# $PARALLEL_TMP will point to a tempfile that will be used as {}
+	# Get the empty arg added by --pipepart (if any)
-	$Global::JobQueue->{'commandlinequeue'}->{'arg_queue'}->
+	$Global::JobQueue->{'commandlinequeue'}->{'arg_queue'}->get();
-	    unget([Arg->new('$PARALLEL_TMP')]);
-    }
-    if($opt::fifo) {
 	# $PARALLEL_TMP will point to a tempfile that will be used as {}
 	$Global::JobQueue->{'commandlinequeue'}->{'arg_queue'}->
 	    unget([Arg->new('$PARALLEL_TMP')]);
@@ -8162,11 +8178,6 @@ sub len {
 		$self->{'replacecount'}{$replstring};
 	}
     }
-    if($opt::nice) {
-	# Pessimistic length if --nice is set
-	# Worse than worst case: every char needs to be quoted with \
-	$len *= 2;
-    }
     if($Global::quoting) {
 	# Pessimistic length if -q is set
 	# Worse than worst case: every char needs to be quoted with \

testsuite/tests-to-run/parallel-freebsd.sh

@@ -55,7 +55,7 @@ bash -c 'echo bug \#43358: shellshock breaks exporting functions using --env _;
   parallel --env funky -S parallel@192.168.1.72 funky ::: shellshock-hardened'
 
 echo '### Test --load (must give 1=true)'
-  parallel -j0 -N0 --timeout 5 --nice 10 'bzip2 < /dev/zero >/dev/null' ::: 1 2 3 & 
+  parallel -j0 -N0 --timeout 5 --nice 10 'bzip2 < /dev/zero >/dev/null' ::: 1 2 3 4 5 6 & 
   parallel --argsep ,, --joblog - -N0 parallel --load 100% echo ::: 1 ,, 1 | 
     parallel --colsep '\t' --header :  echo '{=4 $_=$_>5=}'
 

testsuite/tests-to-run/parallel-local-0.3s.sh

@@ -247,6 +247,14 @@ echo 'bug #46231: {%} with --pipepart broken. Should give 1+2'
 
 echo '**'
 
+echo '{##} bug #45841: Replacement string for total no of jobs'
+
+  parallel --plus echo {##} ::: {a..j};
+  parallel -k 'echo {= $::G++ > 3 and ($_=$Global::JobQueue->total_jobs());=}' ::: {1..10}
+
+echo '**'
+
+
 EOF
 echo '### 1 .par file from --files expected'
 find /tmp{/*,}/*.{par,tms,tmx} 2>/dev/null -mmin -10 | wc -l

testsuite/tests-to-run/parallel-tutorial.sh

@@ -60,6 +60,8 @@ perl -ne '$/="\n\n"; /^Output/../^[^O]\S/ and next; /^  / and print;' ../../src/
             s/tried 2\n//;
             # Due to order is often mixed up
             s/echo \d; exit \d\n/echo X; exit X\n/;
+            # Race condition causes outdir to sometime exist
+            s/(std(out|err):) Permission denied/$1 No such file or directory/;
 '
 # 3+3 .par files (from --files), 1 .tms-file from tmux attach
 find {$TMPDIR,/var/tmp,/tmp}/{fif,tms,par[^a]}* -mmin -10 2>/dev/null | wc -l

testsuite/wanted-results/parallel-local-0.3s

@@ -519,5 +519,30 @@ bug #46231: {%} with --pipepart broken. Should give 1+2
 1
 echo '**'
 **
+echo '{##} bug #45841: Replacement string for total no of jobs'
+{##} bug #45841: Replacement string for total no of jobs
+  parallel --plus echo {##} ::: {a..j};   parallel -k 'echo {= $::G++ > 3 and ($_=$Global::JobQueue->total_jobs());=}' ::: {1..10}
+10
+10
+10
+10
+10
+10
+10
+10
+10
+10
+1
+2
+3
+4
+10
+10
+10
+10
+10
+10
+echo '**'
+**
 ### 1 .par file from --files expected
 1

testsuite/wanted-results/parallel-local4

@@ -1,11 +1,13 @@
 bug #46120: Suspend should suspend (at least local) children
 2048 0a:c0:70:5b:ec:f6:c2:de:67:c3:53:7f:29:81:65:54 tange@hk (RSA1)
+1024 93:d1:21:c5:ff:e3:c3:be:6d:73:ea:aa:1e:a2:dc:06 /home/tange/.ssh/id_dsa (DSA)
 8192 e1:95:e3:ff:99:a6:3a:b5:53:5a:54:59:d0:72:94:7f /home/tange/.ssh/id_rsa (RSA)
 4096 94:2a:e3:cb:6b:66:63:21:13:51:8d:e8:4e:09:49:b2 /home/tange/.ssh/id_rsa_openindiana (RSA)
 
 stdout /usr/bin/time -f CPUTIME=%U parallel --timeout 5 burnP6 ::: 1 | grep --colour=auto -q CPUTIME=1
 Zero=OK 0
 2048 0a:c0:70:5b:ec:f6:c2:de:67:c3:53:7f:29:81:65:54 tange@hk (RSA1)
+1024 93:d1:21:c5:ff:e3:c3:be:6d:73:ea:aa:1e:a2:dc:06 /home/tange/.ssh/id_dsa (DSA)
 8192 e1:95:e3:ff:99:a6:3a:b5:53:5a:54:59:d0:72:94:7f /home/tange/.ssh/id_rsa (RSA)
 4096 94:2a:e3:cb:6b:66:63:21:13:51:8d:e8:4e:09:49:b2 /home/tange/.ssh/id_rsa_openindiana (RSA)
 

testsuite/wanted-results/parallel-polarhome

@@ -30,7 +30,7 @@ aix.polarhome.com	Error in tempfile() using /XXXXXXXX.arg: Could not create temp
 copy_and_test redhat.polarhome.com
 redhat.polarhome.com	### Run the test on redhat.polarhome.com
 redhat.polarhome.com	Works on redhat.polarhome.com
-redhat.polarhome.com	Error in tempfile() using /XXXXXXXX.arg: Could not create temp file /XXXXXXXX.arg: Permission denied at bin/parallel line 0000
+redhat.polarhome.com	Error in tempfile() using template /XXXXXXXX.arg: Could not create temp file /XXXXXXXX.arg: Permission denied at bin/parallel line 0000.
 copy_and_test hpux.polarhome.com
 hpux.polarhome.com	### Run the test on hpux.polarhome.com
 hpux.polarhome.com	Works on hpux.polarhome.com
@@ -118,6 +118,7 @@ centos.polarhome.com
 debian
 dragonfly.polarhome.com
 freebsd.polarhome.com
+hpux-ia64
 hpux64
 hurd
 macosx
@@ -129,6 +130,7 @@ openindiana
 pidora
 qnx
 raspbian
+redhat.polarhome.com
 scosysv.polarhome.com
 solaris-x86
 solaris