From 072897c5678608561963959cb88e8663805ef1a7 Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Tue, 5 Oct 2021 22:49:43 +0200 Subject: [PATCH] parallel: Support for input lines > max line length. --- doc/release_new_version | 2 +- src/parallel | 17 +++++++++++------ testsuite/tests-to-run/parallel-local-0.3s.sh | 17 ----------------- testsuite/tests-to-run/parallel-local-1s.sh | 13 +++++++++++++ testsuite/wanted-results/parallel-local-0.3s | 11 ----------- testsuite/wanted-results/parallel-local-1s | 7 +++++++ 6 files changed, 32 insertions(+), 35 deletions(-) diff --git a/doc/release_new_version b/doc/release_new_version index 1764b484..a1324efe 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -254,7 +254,7 @@ from:tange@gnu.org to:parallel@gnu.org, bug-parallel@gnu.org stable-bcc: Jesse Alama -Subject: GNU Parallel 20211022 ('<<>>Scholz') released <<[stable]>> +Subject: GNU Parallel 20211022 ('<<>>Scholz/Vilks/FacebookDown') released <<[stable]>> GNU Parallel 20211022 ('<<>>') <<[stable]>> has been released. It is available for download at: lbry://@GnuParallel:4 diff --git a/src/parallel b/src/parallel index 969cd934..d1696f89 100755 --- a/src/parallel +++ b/src/parallel @@ -11266,7 +11266,8 @@ sub slot($) { } if($opt::sqlmaster) { - # Insert the V1..Vn for this $seq in SQL table instead of generating one + # Insert the V1..Vn for this $seq in SQL table + # instead of generating one $Global::sql->insert_records($self->seq(), $self->{'command'}, $self->{'arg_list_flat_orig'}); } @@ -11285,12 +11286,16 @@ sub push($) { *Arg::arg = $self->{'arg_list_flat_orig'}; my $quote_arg = ($Global::quote_replace and not $Global::quoting); + my $col; for my $perlexpr (keys %{$self->{'replacecount'}}) { - if($perlexpr =~ /^(\d+) /) { - # Positional - defined($record->[$1-1]) or next; - $self->{'len'}{$perlexpr} += - length $record->[$1-1]->replace($perlexpr,$quote_arg,$self); + if($perlexpr =~ /^(-?\d+)(?:\D.*|)$/) { + # Positional replacement string + # Deal with negative positional replacement string + $col = ($1 < 0) ? $1 : $1-1; + if(defined($record->[$col])) { + $self->{'len'}{$perlexpr} += + length $record->[$col]->replace($perlexpr,$quote_arg,$self); + } } else { for my $arg (@$record) { if(defined $arg) { diff --git a/testsuite/tests-to-run/parallel-local-0.3s.sh b/testsuite/tests-to-run/parallel-local-0.3s.sh index 16a0ce53..cc10866c 100644 --- a/testsuite/tests-to-run/parallel-local-0.3s.sh +++ b/testsuite/tests-to-run/parallel-local-0.3s.sh @@ -16,23 +16,6 @@ export -f stdsort # Test amount of parallelization # parallel --shuf --jl /tmp/myjl -j1 'export JOBS={1};'bash tests-to-run/parallel-local-0.3s.sh ::: {1..16} ::: {1..5} -par_long_input() { - echo '### Long input lines should not fail if they are not used' - perl -e 'map { print "$_\t"."X"x100000 ."\t".uc($_)."\n" } (a..c)' | - parallel --colsep '\t' echo {1} - perl -e 'map { print "$_\t"."X"x100000 ."\t".uc($_)."\n" } (a..c)' | - parallel --colsep '\t' echo {3} - perl -e 'map { print "$_\t"."X"x100000 ."\t".uc($_)."\n" } (a..c)' | - parallel --colsep '\t' echo {1} {3} - - perl -e 'map { print "$_\t"."X"x1000000 ."\t".uc($_)."\n" } (a..c)' | - parallel --colsep '\t' echo {1} - perl -e 'map { print "$_\t"."X"x1000000 ."\t".uc($_)."\n" } (a..c)' | - parallel --colsep '\t' echo {3} - perl -e 'map { print "$_\t"."X"x1000000 ."\t".uc($_)."\n" } (a..c)' | - parallel --colsep '\t' echo {1} {3} -} - par_ctagstring() { echo '### --ctag --ctagstring should be different from --tag --tagstring' parallel --tag echo ::: 1 ::: a| wc -c diff --git a/testsuite/tests-to-run/parallel-local-1s.sh b/testsuite/tests-to-run/parallel-local-1s.sh index 29891105..99895f62 100644 --- a/testsuite/tests-to-run/parallel-local-1s.sh +++ b/testsuite/tests-to-run/parallel-local-1s.sh @@ -8,6 +8,19 @@ # Each should be taking 1-3s and be possible to run in parallel # I.e.: No race conditions, no logins +par_long_input() { + echo '### Long input lines should not fail if they are not used' + longline_tsv() { + perl -e '$a = "X"x3000000; + map { print join "\t", $_, $a, "$_/$a.$a", "$a/$_.$a", "$a/$a.$_\n" } + (a..c)' + } + longline_tsv | + parallel --colsep '\t' echo {1} {3//} {4/.} '{=5 s/.*\.// =}' + longline_tsv | + parallel --colsep '\t' echo {-5} {-3//} {-2/.} '{=-1 s/.*\.// =}' +} + par_plus_slot_replacement() { echo '### show {slot} {0%} {0#}' parallel -k --plus 'sleep 0.{%};echo {slot}=$PARALLEL_JOBSLOT={%}' ::: A B C diff --git a/testsuite/wanted-results/parallel-local-0.3s b/testsuite/wanted-results/parallel-local-0.3s index c3f2a238..d8ea49bf 100644 --- a/testsuite/wanted-results/parallel-local-0.3s +++ b/testsuite/wanted-results/parallel-local-0.3s @@ -492,17 +492,6 @@ par_long_cmd_mem_use perl -e '$r=rand(shift); for($f = 0; $f < $r; $f++){ $a = " par_long_cmd_mem_use perl -e '$r=rand(shift); for($f = 0; $f < $r; $f++){ $a = "a"x100 } print shift,"\n"' 10000 97 par_long_cmd_mem_use perl -e '$r=rand(shift); for($f = 0; $f < $r; $f++){ $a = "a"x100 } print shift,"\n"' 10000 98 par_long_cmd_mem_use perl -e '$r=rand(shift); for($f = 0; $f < $r; $f++){ $a = "a"x100 } print shift,"\n"' 10000 99 -par_long_input ### Long input lines should not fail if they are not used -par_long_input a -par_long_input c -par_long_input b -par_long_input A -par_long_input C -par_long_input B -par_long_input parallel: Error: Command line too long (200014 >= 131063) at input 0: a XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX... -par_long_input parallel: Error: Command line too long (1000009 >= 131063) at input 0: a XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX... -par_long_input parallel: Error: Command line too long (1000009 >= 131063) at input 0: a XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX... -par_long_input parallel: Error: Command line too long (2000014 >= 131063) at input 0: a XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX... par_maxchar_s ### Test --max-chars and -s: Max number of chars in a line par_maxchar_s line 1 line 1 par_maxchar_s line 2 diff --git a/testsuite/wanted-results/parallel-local-1s b/testsuite/wanted-results/parallel-local-1s index 79d1f388..ee2f51f7 100644 --- a/testsuite/wanted-results/parallel-local-1s +++ b/testsuite/wanted-results/parallel-local-1s @@ -178,6 +178,13 @@ par_interactive sleep 0.1; echo opt-p 1 ?...y par_interactive sleep 0.1; echo opt-p 2 ?...n par_interactive sleep 0.1; echo opt-p 3 ?...y par_interactive spawn /tmp/parallel-script-for-expect +par_long_input ### Long input lines should not fail if they are not used +par_long_input a a a a +par_long_input b b b b +par_long_input c c c c +par_long_input a a a a +par_long_input b b b b +par_long_input c c c c par_mix_triple_colon_with_quad_colon ### Test :::: mixed with ::: par_mix_triple_colon_with_quad_colon ### Test :::: < ::: ::: par_mix_triple_colon_with_quad_colon 6 4 1