tange/parallel
1
0
Fork 0
mirror of https://git.savannah.gnu.org/git/parallel.git synced 2024-12-22 12:47:54 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

parallel: Support for input lines > max line length.

Browse source
This commit is contained in:
Ole Tange 2021-10-05 22:49:43 +02:00
parent 917c1795f0
commit 072897c567
6 changed files with 32 additions and 35 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
doc/release_new_version
View file

@ -254,7 +254,7 @@ from:tange@gnu.org
to:parallel@gnu.org, bug-parallel@gnu.org
stable-bcc: Jesse Alama <jessealama@fastmail.fm>
Subject: GNU Parallel 20211022 ('<<>>Scholz') released <<[stable]>>
Subject: GNU Parallel 20211022 ('<<>>Scholz/Vilks/FacebookDown') released <<[stable]>>
GNU Parallel 20211022 ('<<>>') <<[stable]>> has been released. It is available for download at: lbry://@GnuParallel:4

17
src/parallel
View file

@ -11266,7 +11266,8 @@ sub slot($) {
}
if($opt::sqlmaster) {
# Insert the V1..Vn for this $seq in SQL table instead of generating one
# Insert the V1..Vn for this $seq in SQL table
# instead of generating one
$Global::sql->insert_records($self->seq(), $self->{'command'},
$self->{'arg_list_flat_orig'});
}
@ -11285,12 +11286,16 @@ sub push($) {
*Arg::arg = $self->{'arg_list_flat_orig'};
my $quote_arg = ($Global::quote_replace and not $Global::quoting);
my $col;
for my $perlexpr (keys %{$self->{'replacecount'}}) {
if($perlexpr =~ /^(\d+) /) {
# Positional
defined($record->[$1-1]) or next;
$self->{'len'}{$perlexpr} +=
length $record->[$1-1]->replace($perlexpr,$quote_arg,$self);
if($perlexpr =~ /^(-?\d+)(?:\D.*|)$/) {
# Positional replacement string
# Deal with negative positional replacement string
$col = ($1 < 0) ? $1 : $1-1;
if(defined($record->[$col])) {
$self->{'len'}{$perlexpr} +=
length $record->[$col]->replace($perlexpr,$quote_arg,$self);
}
} else {
for my $arg (@$record) {
if(defined $arg) {

17
testsuite/tests-to-run/parallel-local-0.3s.sh
View file

@ -16,23 +16,6 @@ export -f stdsort
# Test amount of parallelization
# parallel --shuf --jl /tmp/myjl -j1 'export JOBS={1};'bash tests-to-run/parallel-local-0.3s.sh ::: {1..16} ::: {1..5}
par_long_input() {
echo '### Long input lines should not fail if they are not used'
perl -e 'map { print "$_\t"."X"x100000 ."\t".uc($_)."\n" } (a..c)' |
parallel --colsep '\t' echo {1}
perl -e 'map { print "$_\t"."X"x100000 ."\t".uc($_)."\n" } (a..c)' |
parallel --colsep '\t' echo {3}
perl -e 'map { print "$_\t"."X"x100000 ."\t".uc($_)."\n" } (a..c)' |
parallel --colsep '\t' echo {1} {3}
perl -e 'map { print "$_\t"."X"x1000000 ."\t".uc($_)."\n" } (a..c)' |
parallel --colsep '\t' echo {1}
perl -e 'map { print "$_\t"."X"x1000000 ."\t".uc($_)."\n" } (a..c)' |
parallel --colsep '\t' echo {3}
perl -e 'map { print "$_\t"."X"x1000000 ."\t".uc($_)."\n" } (a..c)' |
parallel --colsep '\t' echo {1} {3}
}
par_ctagstring() {
echo '### --ctag --ctagstring should be different from --tag --tagstring'
parallel --tag echo ::: 1 ::: a| wc -c

13
testsuite/tests-to-run/parallel-local-1s.sh
View file

@ -8,6 +8,19 @@
# Each should be taking 1-3s and be possible to run in parallel
# I.e.: No race conditions, no logins
par_long_input() {
echo '### Long input lines should not fail if they are not used'
longline_tsv() {
perl -e '$a = "X"x3000000;
map { print join "\t", $_, $a, "$_/$a.$a", "$a/$_.$a", "$a/$a.$_\n" }
(a..c)'
}
longline_tsv |
parallel --colsep '\t' echo {1} {3//} {4/.} '{=5 s/.*\.// =}'
longline_tsv |
parallel --colsep '\t' echo {-5} {-3//} {-2/.} '{=-1 s/.*\.// =}'
}
par_plus_slot_replacement() {
echo '### show {slot} {0%} {0#}'
parallel -k --plus 'sleep 0.{%};echo {slot}=$PARALLEL_JOBSLOT={%}' ::: A B C

11
testsuite/wanted-results/parallel-local-0.3s
View file

@ -492,17 +492,6 @@ par_long_cmd_mem_use perl -e '$r=rand(shift); for($f = 0; $f < $r; $f++){ $a = "
par_long_cmd_mem_use perl -e '$r=rand(shift); for($f = 0; $f < $r; $f++){ $a = "a"x100 } print shift,"\n"' 10000 97
par_long_cmd_mem_use perl -e '$r=rand(shift); for($f = 0; $f < $r; $f++){ $a = "a"x100 } print shift,"\n"' 10000 98
par_long_cmd_mem_use perl -e '$r=rand(shift); for($f = 0; $f < $r; $f++){ $a = "a"x100 } print shift,"\n"' 10000 99
par_long_input ### Long input lines should not fail if they are not used
par_long_input a
par_long_input c
par_long_input b
par_long_input A
par_long_input C
par_long_input B
par_long_input parallel: Error: Command line too long (200014 >= 131063) at input 0: a XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX...
par_long_input parallel: Error: Command line too long (1000009 >= 131063) at input 0: a XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX...
par_long_input parallel: Error: Command line too long (1000009 >= 131063) at input 0: a XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX...
par_long_input parallel: Error: Command line too long (2000014 >= 131063) at input 0: a XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX...
par_maxchar_s ### Test --max-chars and -s: Max number of chars in a line
par_maxchar_s line 1 line 1
par_maxchar_s line 2

7
testsuite/wanted-results/parallel-local-1s
View file

@ -178,6 +178,13 @@ par_interactive sleep 0.1; echo opt-p 1 ?...y
par_interactive sleep 0.1; echo opt-p 2 ?...n
par_interactive sleep 0.1; echo opt-p 3 ?...y
par_interactive spawn /tmp/parallel-script-for-expect
par_long_input ### Long input lines should not fail if they are not used
par_long_input a a a a
par_long_input b b b b
par_long_input c c c c
par_long_input a a a a
par_long_input b b b b
par_long_input c c c c
par_mix_triple_colon_with_quad_colon ### Test :::: mixed with :::
par_mix_triple_colon_with_quad_colon ### Test :::: < ::: :::
par_mix_triple_colon_with_quad_colon 6 4 1