Fixed bug #46232: {%} with --bar/--eta/--shuf or --halt xx% broken.

src/parallel

@@ -211,9 +211,6 @@ sub pipe_part_files {
     for(my $i=0; $i<$#pos; $i++) {
 	push @cat_partials, cat_partial($file, 0, length($header), $pos[$i], $pos[$i+1]);
     }
-    # Remote exec should look like:
-    #  ssh -oLogLevel=quiet lo  'eval `echo $SHELL | grep "/t\{0,1\}csh" > /dev/null  && echo setenv PARALLEL_SEQ '$PARALLEL_SEQ'\;  setenv PARALLEL_PID '$PARALLEL_PID'  || echo PARALLEL_SEQ='$PARALLEL_SEQ'\;export PARALLEL_SEQ\;  PARALLEL_PID='$PARALLEL_PID'\;export PARALLEL_PID` ;'  tty\ \>/dev/null\ \&\&\ stty\ isig\ -onlcr\ -echo\;echo\ \$SHELL\ \|\ grep\ \"/t\\\{0,1\\\}csh\"\ \>\ /dev/null\ \&\&\ setenv\ FOO\ /tmp/foo\ \|\|\ export\ FOO=/tmp/foo\; \(wc\ -\ \$FOO\)
-    # ssh -tt not allowed. Remote will die due to broken pipe anyway.
     return @cat_partials;
 }
 
@@ -5786,35 +5783,33 @@ sub empty {
 sub total_jobs {
     my $self = shift;
     if(not defined $self->{'total_jobs'}) {
-        my $job;
-        my @queue;
+	my $record;
+	my @arg_records;
+	my $record_queue = $self->{'commandlinequeue'}{'arg_queue'};
 	my $start = time;
-        while($job = $self->get()) {
+        while($record = $record_queue->get()) {
 	    if(time - $start > 10) {
-		::warning("Reading ".scalar(@queue)." arguments took longer than 10 seconds.");
+		::warning("Reading ".scalar(@arg_records)." arguments took longer than 10 seconds.");
 		$opt::eta && ::warning("Consider removing --eta.");
 		$opt::bar && ::warning("Consider removing --bar.");
 		$opt::shuf && ::warning("Consider removing --shuf.");
 		last;
 	    }
-            push @queue, $job;
+            push @arg_records, $record;
         }
-        while($job = $self->get()) {
-            push @queue, $job;
+        while($record = $record_queue->get()) {
+            push @arg_records, $record;
         }
 	if($opt::shuf) {
-	    my $i = @queue;
+	    my $i = @arg_records;
 	    while (--$i) {
 		my $j = int rand($i+1);
-		@queue[$i,$j] = @queue[$j,$i];
-	    }
-	    my $seq = 1;
-	    for my $job (@queue) {
-		$job->{'commandline'}->set_seq($seq++);
+		@arg_records[$i,$j] = @arg_records[$j,$i];
 	    }
 	}
-	$self->unget(@queue);
-        $self->{'total_jobs'} = $#queue+1;
+	$record_queue->unget(@arg_records);
+        $self->{'total_jobs'} = $#arg_records+1;
+	::debug("init","Total jobs: ".$self->{'total_jobs'}."\n");
     }
     return $self->{'total_jobs'};
 }
@@ -6480,7 +6475,7 @@ sub wrapped {
     if(not defined $self->{'wrapped'}) {
 	my $command = $self->replaced();
 	if($opt::shellquote) {
-	    # Prepend \echo (echo no-\ is wrong in csh)
+	    # Prepend /bin/echo (echo no-/bin is wrong in csh)
 	    # and quote twice
 	    $command = "/bin/echo " .
 		::shell_quote_scalar(::shell_quote_scalar($command));
@@ -8785,7 +8780,13 @@ sub get {
     my $self = shift;
     if(@{$self->{'unget'}}) {
 	$self->{'arg_number'}++;
-	return shift @{$self->{'unget'}};
+	# Flush cached computed replacements in Arg-objects
+	# To fix: parallel --bar echo {%} ::: a b c ::: d e f
+	my $ret = shift @{$self->{'unget'}};
+	if($ret) {
+	    map { $_->flush_cache() } map { @$_ } $ret;
+	}
+	return $ret;
     }
     my $ret = $self->{'arg_sub_queue'}->get();
     if(defined $Global::max_number_of_args
@@ -8794,6 +8795,11 @@ sub get {
 	# \0 => nothing (not the empty string)
 	return [Arg->new("\0")];
     } else {
+	# Flush cached computed replacements in Arg-objects
+	# To fix: parallel --bar echo {%} ::: a b c ::: d e f
+	if($ret) {
+	    map { $_->flush_cache() } map { @$_ } $ret;
+	}
 	return $ret;
     }
 }
@@ -8904,13 +8910,7 @@ sub get {
     if($opt::xapply) {
 	return $self->xapply_get();
     } else {
-	my $arglist = $self->nest_get();
-	# Flush cached computed values
-	# To fix: parallel echo {%} ::: 1 2 ::: 4 3
-	if($arglist) {
-	    map { $_->flush_cache() } @$arglist;
-	}
-	return $arglist;
+	return $self->nest_get();
     }
 }
 

testsuite/tests-to-run/parallel-local-0.3s.sh

@@ -16,6 +16,9 @@ export SMALLDISK
   sudo chmod 777 /mnt/ram
 ) >/dev/null 2>/dev/null
 
+
+
+
 cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj8 -k --joblog /tmp/jl-`basename $0` -L1
 echo '### Test bug #45619: "--halt" erroneous error exit code (should give 0)'; 
   seq 10 | parallel --halt now,fail=1 true; 
@@ -229,6 +232,20 @@ parallel --wd . 'pwd; echo $OLDPWD; echo' ::: OK
 
 echo '**'
 
+echo 'bug #46232: {%} with --bar/--eta/--shuf or --halt xx% broken'
+  parallel --bar -kj2 echo {%} ::: a b  ::: c d e 2>/dev/null
+  parallel --halt now,fail=10% -kj2 echo {%} ::: a b  ::: c d e
+  parallel --eta -kj2 echo {%} ::: a b  ::: c d e 2>/dev/null
+  parallel --shuf -kj2 echo {%} ::: a b  ::: c d e 2>/dev/null
+
+echo '**'
+
+echo 'bug #46231: {%} with --pipepart broken. Should give 1+2'
+  seq 10000 > /tmp/num10000; 
+  parallel --pipepart -ka /tmp/num10000 --block 10k -j2 echo {%}; 
+  rm /tmp/num10000
+
+echo '**'
 
 EOF
 echo '### 1 .par file from --files expected'

testsuite/wanted-results/parallel-local-0.3s

@@ -477,5 +477,45 @@ parallel --wd . 'pwd; echo $OLDPWD; echo' ::: OK
 OK
 echo '**'
 **
+echo 'bug #46232: {%} with --bar/--eta/--shuf or --halt xx% broken'
+bug #46232: {%} with --bar/--eta/--shuf or --halt xx% broken
+  parallel --bar -kj2 echo {%} ::: a b  ::: c d e 2>/dev/null
+1
+2
+1
+2
+1
+2
+  parallel --halt now,fail=10% -kj2 echo {%} ::: a b  ::: c d e
+1
+2
+1
+2
+1
+2
+  parallel --eta -kj2 echo {%} ::: a b  ::: c d e 2>/dev/null
+1
+2
+1
+2
+1
+2
+  parallel --shuf -kj2 echo {%} ::: a b  ::: c d e 2>/dev/null
+1
+2
+1
+2
+1
+2
+echo '**'
+**
+echo 'bug #46231: {%} with --pipepart broken. Should give 1+2'
+bug #46231: {%} with --pipepart broken. Should give 1+2
+  seq 10000 > /tmp/num10000;   parallel --pipepart -ka /tmp/num10000 --block 10k -j2 echo {%};   rm /tmp/num10000
+1
+2
+1
+2
+1
 ### 1 .par file from --files expected
 1