tange/parallel
1
0
Fork 0
mirror of https://git.savannah.gnu.org/git/parallel.git synced 2024-12-25 22:27:55 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

parallel: Attempt 2 at fixing https://lists.gnu.org/archive/html/parallel/2020-08/msg00003.html

Browse source
This commit is contained in:
Ole Tange 2020-08-16 10:12:42 +02:00
parent 72dd439f20
commit 2ed5279f6a
7 changed files with 220 additions and 117 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

78
src/parallel
View file

@ -2160,7 +2160,7 @@ sub check_invalid_option_combinations() {
sub init_globals() {
# Defaults:
$Global::version = 20200623;
$Global::version = 20200815;
$Global::progname = 'parallel';
$::name = "GNU Parallel";
$Global::infinity = 2**31;
@ -10499,25 +10499,6 @@ sub slot($) {
my $next_arg;
my $max_len = $Global::minimal_command_line_length
|| Limits::Command::max_length();
if($^O eq "darwin") {
# On Mac bash 3.2.48(1)-release the length also depends on
# number of functions, variables and the size of those.
# 21 is found by trial and error and seems to be a safe value.
# test_with() {
# gen500k() {
# seq -f %f 1000000000000000 1000000000050000 |
# head -c 131000;
# }
# for a in `seq 5100`; do eval "export a$a=1" ; done;
# for a in `seq 5100`; do eval "a$a() { 1; }" ; done;
# for a in `seq 5100`; do eval export -f a$a ; done;
# gen500k | stdout parallel -Xj1 'echo {} {} {} {} | wc' |
# perl -pe 's/\d{10,} //g'
# }
$env_size ||= (keys %ENV)*21
+ length(join'',keys %ENV) + length(join'',%ENV);
$max_len -= $env_size;
}
if($opt::cat or $opt::fifo) {
# Get the empty arg added by --pipepart (if any)
$Global::JobQueue->{'commandlinequeue'}->{'arg_queue'}->get();
@ -11517,10 +11498,40 @@ sub max_length($) {
return int($Limits::Command::line_max_len);
}
sub real_max_length($) {
sub real_max_length() {
# Find the max_length of a command line
# Returns:
# The maximal command line length
if($^O eq "darwin") {
# env $((-260932 + $envl *4 -1064 + $envc + $single))
# Darwin's limit is affected by:
# * number of environment names (variables+functions)
# * size of environment
# * the length of arguments:
# a one-char argument lowers the limit by 5
# To be safe assume all arguments are one-char
my $max_len = find_max("x");
my $env_size = (keys %ENV) * 4 + length(join'',%ENV);
$max_len -= $env_size;
# Small fudge constant I cannot explain
$max_len -= 1064;
# If all arguments are 1 char, the limit is 5x smaller
$max_len /= 5;
return $max_len;
} else {
return find_max("x");
}
}
sub real_max_args() {
# Find the max number of args allowed on a command line
# Returns:
# The maximal number of args allowed on a command line
return find_max(" x");
}
sub find_max($) {
my $string = shift;
# This is slow on Cygwin, so give Cygwin users a warning
if($^O eq "cygwin") {
::warning("Finding the maximal command line length. This may take up to 30 seconds.")
@ -11533,39 +11544,42 @@ sub real_max_length($) {
do {
if($len > $upper) { return $len };
$len *= 16;
} while (is_acceptable_command_line_length($len));
} while (is_acceptable_command_line_length($len,$string));
# Then search for the actual max length between 0 and upper bound
return binary_find_max_length(int($len/16),$len);
return binary_find_max(int($len/16),$len,$string);
}
# Prototype forwarding
sub binary_find_max_length($$);
sub binary_find_max_length($$) {
# Given a lower and upper bound find the max_length of a command line
sub binary_find_max($$$);
sub binary_find_max($$$) {
# Given a lower and upper bound find the max (length or args) of a command line
# Returns:
# number of chars on the longest command line allowed
my ($lower, $upper) = (@_);
my ($lower, $upper, $string) = (@_);
if($lower == $upper or $lower == $upper-1) { return $lower; }
my $middle = int (($upper-$lower)/2 + $lower);
::debug("init", "Maxlen: $lower<$middle<$upper: ");
if (is_acceptable_command_line_length($middle)) {
return binary_find_max_length($middle,$upper);
if (is_acceptable_command_line_length($middle,$string)) {
return binary_find_max($middle,$upper,$string);
} else {
return binary_find_max_length($lower,$middle);
return binary_find_max($lower,$middle,$string);
}
}
sub is_acceptable_command_line_length($) {
sub is_acceptable_command_line_length($$) {
# Test if a command line of this length can run
# in the current environment
# If the string is " x" it tests how many args are allowed
# Returns:
# 0 if the command line length is too long
# 1 otherwise
my $len = shift;
my $string = shift;
if($Global::parallel_env) {
$len += length $Global::parallel_env;
}
::qqx("true "."x"x$len);
# Force using non-built-in command
::qqx("/bin/echo ".${string}x(($len-length "/bin/echo ")/length $string));
::debug("init", "$len=$? ");
return not $?;
}

9
testsuite/tests-to-run/parallel-local-ssh8.sh
View file

@ -2,6 +2,15 @@
# Each test should at most run 1 ssh against parallel@lo or lo
par_transfer_special_char_names() {
echo '### Test --return of weirdly named file'
ssh parallel@lo rm 'aa*b'
rm -f 'aa<${#}" b'
stdout parallel --return {} -S parallel@lo echo '>'{} ::: 'aa<${#}" b'
ls 'aa<${#}" b'
rm -f 'aa<${#}" b'
}
par_path_remote_bash() {
echo 'bug #47695: How to set $PATH on remote? Bash'
rm -rf /tmp/parallel

49
testsuite/tests-to-run/parallel-macos.sh
View file

@ -3,64 +3,95 @@
. `which env_parallel.bash`
env_parallel --session
par_many_args() {
rm -f ~/.parallel/tmp/sshlogin/*/linelen
pecho() { perl -e 'print "@ARGV\n"' "$@"; }
export -f pecho
gen500k() { yes | head -c 131000; }
for a in `seq 6000`; do eval "export a$a=1" ; done
gen500k | stdout parallel --load 27 -Xkj1 'pecho {} {} {} {} | wc' |
perl -pe 's/\d{10,}.\d+ //g'
}
par_many_var() {
rm -f ~/.parallel/tmp/sshlogin/*/linelen
pecho() { perl -e 'print "@ARGV\n"' "$@"; }
export -f pecho
gen500k() { seq -f %f 1000000000000000 1000000000050000 | head -c 131000; }
for a in `seq 6000`; do eval "export a$a=1" ; done
gen500k | stdout parallel --load 5 -Xkj1 'echo {} {} {} {} | wc' |
gen500k | stdout parallel --load 4 -Xkj1 'pecho {} {} {} {} | wc' |
perl -pe 's/\d{10,}.\d+ //g'
}
par_many_var_func() {
rm -f ~/.parallel/tmp/sshlogin/*/linelen
gen500k() { seq -f %f 1000000000000000 1000000000050000 | head -c 131000; }
pecho() { perl -e 'print "@ARGV\n"' "$@"; }
export -f pecho
for a in `seq 5000`; do eval "export a$a=1" ; done
for a in `seq 5000`; do eval "a$a() { 1; }" ; done
for a in `seq 5000`; do eval export -f a$a ; done
gen500k | stdout parallel --load 21 -Xkj1 'echo {} {} {} {} | wc' |
gen500k | stdout parallel --load 20 -Xkj1 'pecho {} {} {} {} | wc' |
perl -pe 's/\d{10,}.\d+ //g'
}
par_many_func() {
rm -f ~/.parallel/tmp/sshlogin/*/linelen
gen500k() { seq -f %f 1000000000000000 1000000000050000 | head -c 131000; }
pecho() { perl -e 'print "@ARGV\n"' "$@"; }
export -f pecho
for a in `seq 5000`; do eval "a$a() { 1; }" ; done
for a in `seq 5000`; do eval export -f a$a ; done
gen500k | stdout parallel --load 6 -Xkj1 'echo {} {} {} {} | wc' |
gen500k | stdout parallel --load 5 -Xkj1 'pecho {} {} {} {} | wc' |
perl -pe 's/\d{10,}.\d+ //g'
}
par_big_func() {
rm -f ~/.parallel/tmp/sshlogin/*/linelen
gen500k() { seq -f %f 1000000000000000 1000000000050000 | head -c 131000; }
pecho() { perl -e 'print "@ARGV\n"' "$@"; }
export -f pecho
big=`seq 1000`
for a in `seq 50`; do eval "a$a() { '$big'; }" ; done
for a in `seq 50`; do eval export -f a$a ; done
gen500k | stdout parallel --load 3 -Xkj1 'echo {} {} {} {} | wc' |
for a in `seq 1`; do eval "a$a() { '$big'; }" ; done
for a in `seq 1`; do eval export -f a$a ; done
gen500k | stdout parallel --load 2 -Xkj1 'pecho {} {} {} {} | wc' |
perl -pe 's/\d{10,}.\d+ //g'
}
par_many_var_big_func() {
rm -f ~/.parallel/tmp/sshlogin/*/linelen
gen500k() { seq -f %f 1000000000000000 1000000000050000 | head -c 131000; }
pecho() { perl -e 'print "@ARGV\n"' "$@"; }
export -f pecho
big=`seq 1000`
for a in `seq 5000`; do eval "export a$a=1" ; done
for a in `seq 10`; do eval "a$a() { '$big'; }" ; done
for a in `seq 10`; do eval export -f a$a ; done
gen500k | stdout parallel --load 6 -Xkj1 'echo {} {} {} {} | wc' |
gen500k | stdout parallel --load 5 -Xkj1 'pecho {} {} {} {} | wc' |
perl -pe 's/\d{10,}.\d+ //g'
}
par_big_func_name() {
rm -f ~/.parallel/tmp/sshlogin/*/linelen
gen500k() { seq -f %f 1000000000000000 1000000000050000 | head -c 131000; }
pecho() { perl -e 'print "@ARGV\n"' "$@"; }
export -f pecho
big=`perl -e print\"x\"x10000`
for a in `seq 10`; do eval "export a$big$a=1" ; done
gen500k | stdout parallel --load 7 -Xkj1 'echo {} {} {} {} | wc' |
gen500k | stdout parallel --load 5 -Xkj1 'pecho {} {} {} {} | wc' |
perl -pe 's/\d{10,}.\d+ //g'
}
par_big_var_func_name() {
rm -f ~/.parallel/tmp/sshlogin/*/linelen
gen500k() { seq -f %f 1000000000000000 1000000000050000 | head -c 131000; }
pecho() { perl -e 'print "@ARGV\n"' "$@"; }
export -f pecho
big=`perl -e print\"x\"x10000`
for a in `seq 10`; do eval "export a$big$a=1" ; done
for a in `seq 10`; do eval "a$big$a() { 1; }" ; done
for a in `seq 10`; do eval export -f a$big$a ; done
gen500k | stdout parallel --load 5 -Xkj1 'echo {} {} {} {} | wc' |
gen500k | stdout parallel --load 4 -Xkj1 'pecho {} {} {} {} | wc' |
perl -pe 's/\d{10,}.\d+ //g'
}

9
testsuite/tests-to-run/test61.sh
View file

@ -8,13 +8,4 @@ SSHLOGIN1=$SSHUSER1@$SERVER1
SSHLOGIN2=$SSHUSER2@$SERVER2
cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | parallel -vj10 -k --joblog /tmp/jl-`basename $0` -L1 -r | perl -pe 's/(PARALLEL_PID....)\d+/$1XXXXX/g'
echo '### Test --return of weirdly named file'
stdout parallel --return {} -vv -S $SSHLOGIN1 echo '>'{} ::: 'aa<${#}" b' |
perl -pe 's/\S*parallel-server\S*/one-server/;s:[a-z+=/\\0-9]{500,}:base64:i;'; rm 'aa<${#}" b'
echo '### Test if remote login shell is csh'
stdout parallel -k -vv -S csh@localhost 'echo $PARALLEL_PID $PARALLEL_SEQ {}| wc -w' ::: a b c |
perl -pe 's/\S*parallel-server\S*/one-server/;s:[a-z+=/\\0-9]{500,}:base64:i;'
EOF

8
testsuite/wanted-results/parallel-freebsd
View file

@ -55,13 +55,13 @@ par_shebang bug #40134: FreeBSD: --shebang(-wrap) not working
par_shebang wrap
par_shebang works
par_shebang bug #40134: FreeBSD: --shebang(-wrap) with options not working
par_shebang /usr/bin/perl -w ./shebang-wrap-opt wrap
par_shebang /usr/local/bin/perl -w ./shebang-wrap-opt wrap
par_shebang wrap
par_shebang /usr/bin/perl -w ./shebang-wrap-opt works
par_shebang /usr/local/bin/perl -w ./shebang-wrap-opt works
par_shebang works
par_shebang /usr/bin/perl -w ./shebang-wrap-opt with
par_shebang /usr/local/bin/perl -w ./shebang-wrap-opt with
par_shebang with
par_shebang /usr/bin/perl -w ./shebang-wrap-opt options
par_shebang /usr/local/bin/perl -w ./shebang-wrap-opt options
par_shebang options
par_shellshock_bug par_shellshock_bug 2>&1
par_shellshock_bug bug #43358: shellshock breaks exporting functions using --env name

182
testsuite/wanted-results/parallel-macos
View file

@ -1,62 +1,120 @@
par_big_func 1 2592 62208
par_big_func 1 2592 62208
par_big_func 1 2592 62208
par_big_func 1 2592 62208
par_big_func 1 2592 62208
par_big_func 1 2592 62208
par_big_func 1 2592 62208
par_big_func 1 2592 62208
par_big_func 1 1100 26340
par_big_func_name 1 2432 58368
par_big_func_name 1 2432 58368
par_big_func_name 1 2432 58368
par_big_func_name 1 2432 58368
par_big_func_name 1 2432 58368
par_big_func_name 1 2432 58368
par_big_func_name 1 2432 58368
par_big_func_name 1 2432 58368
par_big_func_name 1 2380 57060
par_big_var_func_name 1 2428 58272
par_big_var_func_name 1 2428 58272
par_big_var_func_name 1 2428 58272
par_big_var_func_name 1 2428 58272
par_big_var_func_name 1 2428 58272
par_big_var_func_name 1 2428 58272
par_big_var_func_name 1 2428 58272
par_big_var_func_name 1 2428 58272
par_big_var_func_name 1 2412 57828
par_many_func 1 2532 60768
par_many_func 1 2532 60768
par_many_func 1 2532 60768
par_many_func 1 2532 60768
par_many_func 1 2532 60768
par_many_func 1 2532 60768
par_many_func 1 2532 60768
par_many_func 1 2532 60768
par_many_func 1 1580 37860
par_many_var 1 2868 68832
par_many_var 1 2868 68832
par_many_var 1 2868 68832
par_many_var 1 2868 68832
par_many_var 1 2868 68832
par_many_var 1 2868 68832
par_many_var 1 2868 68832
par_many_var 1 1760 42180
par_many_var_big_func 1 2576 61824
par_many_var_big_func 1 2576 61824
par_many_var_big_func 1 2576 61824
par_many_var_big_func 1 2576 61824
par_many_var_big_func 1 2576 61824
par_many_var_big_func 1 2576 61824
par_many_var_big_func 1 2576 61824
par_many_var_big_func 1 2576 61824
par_many_var_big_func 1 1228 29412
par_many_var_func 1 2532 60768
par_many_var_func 1 2532 60768
par_many_var_func 1 2532 60768
par_many_var_func 1 2532 60768
par_many_var_func 1 2532 60768
par_many_var_func 1 2532 60768
par_many_var_func 1 2532 60768
par_many_var_func 1 2532 60768
par_many_var_func 1 1580 37860
par_big_func 1 2168 52032
par_big_func 1 2168 52032
par_big_func 1 2168 52032
par_big_func 1 2168 52032
par_big_func 1 2168 52032
par_big_func 1 2168 52032
par_big_func 1 2168 52032
par_big_func 1 2168 52032
par_big_func 1 2168 52032
par_big_func 1 2168 52032
par_big_func 1 156 3684
par_big_func_name 1 2168 52032
par_big_func_name 1 2168 52032
par_big_func_name 1 2168 52032
par_big_func_name 1 2168 52032
par_big_func_name 1 2168 52032
par_big_func_name 1 2168 52032
par_big_func_name 1 2168 52032
par_big_func_name 1 2168 52032
par_big_func_name 1 2168 52032
par_big_func_name 1 2168 52032
par_big_func_name 1 156 3684
par_big_var_func_name 1 2168 52032
par_big_var_func_name 1 2168 52032
par_big_var_func_name 1 2168 52032
par_big_var_func_name 1 2168 52032
par_big_var_func_name 1 2168 52032
par_big_var_func_name 1 2168 52032
par_big_var_func_name 1 2168 52032
par_big_var_func_name 1 2168 52032
par_big_var_func_name 1 2168 52032
par_big_var_func_name 1 2168 52032
par_big_var_func_name 1 156 3684
par_many_args 1 16156 32312
par_many_args 1 16156 32312
par_many_args 1 16156 32312
par_many_args 1 16156 32312
par_many_args 1 16156 32312
par_many_args 1 16156 32312
par_many_args 1 16156 32312
par_many_args 1 16156 32312
par_many_args 1 16156 32312
par_many_args 1 16156 32312
par_many_args 1 16156 32312
par_many_args 1 16156 32312
par_many_args 1 16156 32312
par_many_args 1 16156 32312
par_many_args 1 16156 32312
par_many_args 1 16156 32312
par_many_args 1 3504 7008
par_many_func 1 1260 30240
par_many_func 1 1260 30240
par_many_func 1 1260 30240
par_many_func 1 1260 30240
par_many_func 1 1260 30240
par_many_func 1 1260 30240
par_many_func 1 1260 30240
par_many_func 1 1260 30240
par_many_func 1 1260 30240
par_many_func 1 1260 30240
par_many_func 1 1260 30240
par_many_func 1 1260 30240
par_many_func 1 1260 30240
par_many_func 1 1260 30240
par_many_func 1 1260 30240
par_many_func 1 1260 30240
par_many_func 1 1260 30240
par_many_func 1 416 9924
par_many_var 1 1344 32256
par_many_var 1 1344 32256
par_many_var 1 1344 32256
par_many_var 1 1344 32256
par_many_var 1 1344 32256
par_many_var 1 1344 32256
par_many_var 1 1344 32256
par_many_var 1 1344 32256
par_many_var 1 1344 32256
par_many_var 1 1344 32256
par_many_var 1 1344 32256
par_many_var 1 1344 32256
par_many_var 1 1344 32256
par_many_var 1 1344 32256
par_many_var 1 1344 32256
par_many_var 1 1344 32256
par_many_var 1 332 7908
par_many_var_big_func 1 1304 31296
par_many_var_big_func 1 1304 31296
par_many_var_big_func 1 1304 31296
par_many_var_big_func 1 1304 31296
par_many_var_big_func 1 1304 31296
par_many_var_big_func 1 1304 31296
par_many_var_big_func 1 1304 31296
par_many_var_big_func 1 1304 31296
par_many_var_big_func 1 1304 31296
par_many_var_big_func 1 1304 31296
par_many_var_big_func 1 1304 31296
par_many_var_big_func 1 1304 31296
par_many_var_big_func 1 1304 31296
par_many_var_big_func 1 1304 31296
par_many_var_big_func 1 1304 31296
par_many_var_big_func 1 1304 31296
par_many_var_big_func 1 972 23268
par_many_var_func 1 1260 30240
par_many_var_func 1 1260 30240
par_many_var_func 1 1260 30240
par_many_var_func 1 1260 30240
par_many_var_func 1 1260 30240
par_many_var_func 1 1260 30240
par_many_var_func 1 1260 30240
par_many_var_func 1 1260 30240
par_many_var_func 1 1260 30240
par_many_var_func 1 1260 30240
par_many_var_func 1 1260 30240
par_many_var_func 1 1260 30240
par_many_var_func 1 1260 30240
par_many_var_func 1 1260 30240
par_many_var_func 1 1260 30240
par_many_var_func 1 1260 30240
par_many_var_func 1 1260 30240
par_many_var_func 1 416 9924

2
testsuite/wanted-results/test30
View file

@ -22,7 +22,7 @@ Computers / CPU cores / Max jobs to run
0
echo '### --timeout --onall on remote machines: 2*slept 1, 2 jobs failed'
### --timeout --onall on remote machines: 2*slept 1, 2 jobs failed
parallel -j0 --timeout 6 --onall -S localhost,parallel@parallel-server1 'sleep {}; echo slept {}' ::: 1 8 9 ; echo jobs failed: $?
parallel -j0 --timeout 6 --onall -S localhost,$SSHLOGIN1 'sleep {}; echo slept {}' ::: 1 8 9 ; echo jobs failed: $?
slept 1
slept 1
jobs failed: 2