parallel: Support for input lines > max line length.

doc/release_new_version

@@ -254,7 +254,7 @@ from:tange@gnu.org
 to:parallel@gnu.org, bug-parallel@gnu.org
 stable-bcc: Jesse Alama <jessealama@fastmail.fm>
 
-Subject: GNU Parallel 20211022 ('<<>>Scholz') released <<[stable]>>
+Subject: GNU Parallel 20211022 ('<<>>Scholz/Vilks/FacebookDown') released <<[stable]>>
 
 GNU Parallel 20211022 ('<<>>') <<[stable]>> has been released. It is available for download at: lbry://@GnuParallel:4
 

src/parallel

@@ -11266,7 +11266,8 @@ sub slot($) {
 	}
 
 	if($opt::sqlmaster) {
-	    # Insert the V1..Vn for this $seq in SQL table instead of generating one
+	    # Insert the V1..Vn for this $seq in SQL table
+	    # instead of generating one
 	    $Global::sql->insert_records($self->seq(), $self->{'command'},
 					 $self->{'arg_list_flat_orig'});
 	}
@@ -11285,12 +11286,16 @@ sub push($) {
     *Arg::arg = $self->{'arg_list_flat_orig'};
 
     my $quote_arg = ($Global::quote_replace and not $Global::quoting);
+    my $col;
     for my $perlexpr (keys %{$self->{'replacecount'}}) {
-	if($perlexpr =~ /^(\d+) /) {
+	if($perlexpr =~ /^(-?\d+)(?:\D.*|)$/) {
-	    # Positional
+	    # Positional replacement string
-	    defined($record->[$1-1]) or next;
+	    # Deal with negative positional replacement string
+	    $col = ($1 < 0) ? $1 : $1-1;
+	    if(defined($record->[$col])) {
 		$self->{'len'}{$perlexpr} +=
-		length $record->[$1-1]->replace($perlexpr,$quote_arg,$self);
+		    length $record->[$col]->replace($perlexpr,$quote_arg,$self);
+	    }
 	} else {
 	    for my $arg (@$record) {
 		if(defined $arg) {

testsuite/tests-to-run/parallel-local-0.3s.sh

@@ -16,23 +16,6 @@ export -f stdsort
 # Test amount of parallelization
 # parallel --shuf --jl /tmp/myjl -j1 'export JOBS={1};'bash tests-to-run/parallel-local-0.3s.sh ::: {1..16} ::: {1..5}
 
-par_long_input() {
-    echo '### Long input lines should not fail if they are not used'
-    perl -e 'map { print "$_\t"."X"x100000 ."\t".uc($_)."\n" } (a..c)' |
-	parallel --colsep '\t' echo {1}
-    perl -e 'map { print "$_\t"."X"x100000 ."\t".uc($_)."\n" } (a..c)' |
-	parallel --colsep '\t' echo {3}
-    perl -e 'map { print "$_\t"."X"x100000 ."\t".uc($_)."\n" } (a..c)' |
-	parallel --colsep '\t' echo {1} {3}
-
-    perl -e 'map { print "$_\t"."X"x1000000 ."\t".uc($_)."\n" } (a..c)' |
-	parallel --colsep '\t' echo {1}
-    perl -e 'map { print "$_\t"."X"x1000000 ."\t".uc($_)."\n" } (a..c)' |
-	parallel --colsep '\t' echo {3}
-    perl -e 'map { print "$_\t"."X"x1000000 ."\t".uc($_)."\n" } (a..c)' |
-	parallel --colsep '\t' echo {1} {3}
-}
-
 par_ctagstring() {
     echo '### --ctag --ctagstring should be different from --tag --tagstring'
     parallel --tag echo ::: 1 ::: a| wc -c

testsuite/tests-to-run/parallel-local-1s.sh

@@ -8,6 +8,19 @@
 # Each should be taking 1-3s and be possible to run in parallel
 # I.e.: No race conditions, no logins
 
+par_long_input() {
+    echo '### Long input lines should not fail if they are not used'
+    longline_tsv() {
+	perl -e '$a = "X"x3000000;
+	  map { print join "\t", $_, $a, "$_/$a.$a", "$a/$_.$a", "$a/$a.$_\n" }
+          (a..c)'
+    }
+    longline_tsv |
+	parallel --colsep '\t' echo {1} {3//} {4/.} '{=5 s/.*\.// =}'
+    longline_tsv |
+	parallel --colsep '\t' echo {-5} {-3//} {-2/.} '{=-1 s/.*\.// =}'
+}
+
 par_plus_slot_replacement() {
     echo '### show {slot} {0%} {0#}'
     parallel -k --plus 'sleep 0.{%};echo {slot}=$PARALLEL_JOBSLOT={%}' ::: A B C

testsuite/wanted-results/parallel-local-0.3s

@@ -492,17 +492,6 @@ par_long_cmd_mem_use	perl -e '$r=rand(shift); for($f = 0; $f < $r; $f++){ $a = "
 par_long_cmd_mem_use	perl -e '$r=rand(shift); for($f = 0; $f < $r; $f++){ $a = "a"x100 } print shift,"\n"' 10000 97
 par_long_cmd_mem_use	perl -e '$r=rand(shift); for($f = 0; $f < $r; $f++){ $a = "a"x100 } print shift,"\n"' 10000 98
 par_long_cmd_mem_use	perl -e '$r=rand(shift); for($f = 0; $f < $r; $f++){ $a = "a"x100 } print shift,"\n"' 10000 99
-par_long_input	### Long input lines should not fail if they are not used
-par_long_input	a
-par_long_input	c
-par_long_input	b
-par_long_input	A
-par_long_input	C
-par_long_input	B
-par_long_input	parallel: Error: Command line too long (200014 >= 131063) at input 0: a XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX...
-par_long_input	parallel: Error: Command line too long (1000009 >= 131063) at input 0: a XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX...
-par_long_input	parallel: Error: Command line too long (1000009 >= 131063) at input 0: a XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX...
-par_long_input	parallel: Error: Command line too long (2000014 >= 131063) at input 0: a XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX...
 par_maxchar_s	### Test --max-chars and -s: Max number of chars in a line
 par_maxchar_s	line 1 line 1
 par_maxchar_s	line 2

testsuite/wanted-results/parallel-local-1s

@@ -178,6 +178,13 @@ par_interactive	sleep 0.1; echo opt-p 1 ?...y
 par_interactive	sleep 0.1; echo opt-p 2 ?...n
 par_interactive	sleep 0.1; echo opt-p 3 ?...y
 par_interactive	spawn /tmp/parallel-script-for-expect
+par_long_input	### Long input lines should not fail if they are not used
+par_long_input	a a a a
+par_long_input	b b b b
+par_long_input	c c c c
+par_long_input	a a a a
+par_long_input	b b b b
+par_long_input	c c c c
 par_mix_triple_colon_with_quad_colon	### Test :::: mixed with :::
 par_mix_triple_colon_with_quad_colon	### Test :::: < ::: :::
 par_mix_triple_colon_with_quad_colon	6 4 1