diff --git a/CREDITS b/CREDITS index b5ff9706..4ca0e4fd 100644 --- a/CREDITS +++ b/CREDITS @@ -1,5 +1,6 @@ People who have helped GNU Parallel different ways. +Morten Rønne: Donating his old Mac for testing. Renan Valieris: Maintaining GNU Parallel for Anaconda Cloud. Jakub Kulík: Maintaining GNU Parallel for Solaris-userland. Rich Burridge: Maintaining GNU Parallel for Solaris-userland. diff --git a/src/parallel b/src/parallel index 13a789ab..41ff77e5 100755 --- a/src/parallel +++ b/src/parallel @@ -2160,7 +2160,7 @@ sub check_invalid_option_combinations() { sub init_globals() { # Defaults: - $Global::version = 20200815; + $Global::version = 20200820; $Global::progname = 'parallel'; $::name = "GNU Parallel"; $Global::infinity = 2**31; @@ -5351,7 +5351,7 @@ sub qqx(@) { # CygWin does not respect 2>/dev/null # so we do that by hand # This trick does not work: - # https://stackoverflow.com/questions/13833088/why-doesnt-local-work-on-stderr-and-stdout + # https://stackoverflow.com/q/13833088/363028 # local *STDERR; # open(STDERR, ">", "/dev/null"); open(local *CHILD_STDIN, '<', '/dev/null') or die $!; @@ -5370,7 +5370,10 @@ sub qqx(@) { # Make sure $? is set waitpid($pid, 0); return wantarray ? @arr : join "",@arr; - } + } or do { + # If eval fails, force $?=false + `false`; + }; } } @@ -10481,7 +10484,7 @@ sub slot($) { { my $already_spread; - my $env_size; + my $darwin_max_len; sub populate($) { # Add arguments from arg_queue until the number of arguments or @@ -10499,6 +10502,28 @@ sub slot($) { my $next_arg; my $max_len = $Global::minimal_command_line_length || Limits::Command::max_length(); + if($^O eq "darwin") { + # env $((260932 - $single - $envc - $envl * 4 + 1064 )) + # Darwin's limit is affected by: + # * number of environment names (variables+functions) + # * size of environment + # * the length of arguments: + # a one-char argument lowers the limit by 5 + # To be safe assume all arguments are one-char + # The max_len is cached between runs, but if the size of + # the environment is different we need to recompute the + # usable max length for this run of GNU Parallel + # See https://unix.stackexchange.com/a/604943/2972 + if(not $darwin_max_len) { + my $envc = (keys %ENV); + my $envn = length join"",(keys %ENV); + my $envv = length join"",(values %ENV); + $darwin_max_len = 3+($max_len - $envn - $envv) / 5 - $envc*2; + ::debug("init", + "length: $darwin_max_len 3+($max_len - $envn - $envv) / 5 - $envc*2"); + } + $max_len = $darwin_max_len; + } if($opt::cat or $opt::fifo) { # Get the empty arg added by --pipepart (if any) $Global::JobQueue->{'commandlinequeue'}->{'arg_queue'}->get(); @@ -11501,33 +11526,9 @@ sub max_length($) { sub real_max_length() { # Find the max_length of a command line # Returns: - # The maximal command line length - if($^O eq "darwin") { - # env $((-260932 + $envl *4 -1064 + $envc + $single)) - # Darwin's limit is affected by: - # * number of environment names (variables+functions) - # * size of environment - # * the length of arguments: - # a one-char argument lowers the limit by 5 - # To be safe assume all arguments are one-char - my $max_len = find_max("x"); - my $env_size = (keys %ENV) * 4 + length(join'',%ENV); - $max_len -= $env_size; - # Small fudge constant I cannot explain - $max_len -= 1064; - # If all arguments are 1 char, the limit is 5x smaller - $max_len /= 5; - return $max_len; - } else { - return find_max("x"); - } -} - -sub real_max_args() { - # Find the max number of args allowed on a command line - # Returns: - # The maximal number of args allowed on a command line - return find_max(" x"); + # The maximal command line length with 1 byte arguments + # return find_max(" x"); + return find_max("x"); } sub find_max($) { diff --git a/src/parallel.pod b/src/parallel.pod index 5eb7a55a..99d97743 100644 --- a/src/parallel.pod +++ b/src/parallel.pod @@ -5192,7 +5192,7 @@ Your bug report should always include: The error message you get (if any). If the error message is not from GNU B you need to show why you think GNU B caused -these. +this. =item * @@ -5204,15 +5204,16 @@ version. =item * A minimal, complete, and verifiable example (See description on -http://stackoverflow.com/help/mcve). +https://stackoverflow.com/help/mcve). -It should be a complete example that others can run that shows the +It should be a complete example that others can run which shows the problem including all files needed to run the example. This should preferably be small and simple, so try to remove as many options as possible. A combination of B, B, B, B, B, and B can reproduce most errors. If your example requires large files, see if you can make them with something like B > -B or B > B. +B or B > B. If you need +multiple columns: B If your example requires remote execution, see if you can use B - maybe using another login. @@ -5392,9 +5393,9 @@ For remote usage it uses B with B. =head1 SEE ALSO -B(1), B(1), B(1), B(1), -B(1), B(1), B(1), B(1), B(1), -B(1), B(1), B(1), B(1), B(1), -B(1) +B(1), B(1), B(1), +B(1), B(1), B(7), +B(1), B(1), B(1), B(1), B(1), +B(1), B(1) =cut diff --git a/testsuite/tests-to-run/parallel-macos.sh b/testsuite/tests-to-run/parallel-macos.sh index d0444727..41ab299a 100644 --- a/testsuite/tests-to-run/parallel-macos.sh +++ b/testsuite/tests-to-run/parallel-macos.sh @@ -3,103 +3,160 @@ . `which env_parallel.bash` env_parallel --session +true <<'EOF' +#!/bin/bash + +# Find the command line limit formula +# macosx.p = 10.7.5 +# El capitan = 10.11.4 + +. `which binsearch` +doit() { + nfunc=$1 + lfunc=$2 + lfuncname=$3 + nvar=$4 + lvar=$5 + lvarname=$6 + varval="$(perl -e 'print "x "x('$lvar'/2)')" + varname=$(perl -e 'print "x"x'$lvarname) + funcval="$(perl -e 'print "x "x('$lfunc'/2)')" + funcname=$(perl -e 'print "x"x'$lfuncname) + for a in `seq $nvar`; do eval "export v$varname$a='$varval'" ; done + for a in `seq $nfunc`; do eval "f$funcname$a() { $funcval; }" ; done + for a in `seq $nfunc`; do eval "export -f f$funcname$a" ; done + myrun() { + /bin/echo $(perl -e 'print " x"x('$1'/2-5)') + } + export -f myrun + binlen=dummy + binlen=$(binsearch -q myrun) + perl -e ' + $envc=(keys %ENV); + $envn=length join"",(keys %ENV); + $envv=length join"",(values %ENV); + $maxlen=3+(262144 - $envn - $envv) / 5 - $envc*2; + print("Max len = $maxlen\n"); + $bin='$binlen'; + print("$bin=",$bin-$maxlen," $envc $envn $envv\n"); + ' +} +export -f doit + +val="$(seq 2 100 1000)" +val="10 20 50 100 200 500 1000" +val="11 23 57 101 207 503 1007" +parallel --shuf --tag -k doit ::: $val ::: $val ::: $val ::: $val ::: $val ::: $val +EOF + +# Each should generate at least 2 commands + par_many_args() { rm -f ~/.parallel/tmp/sshlogin/*/linelen pecho() { perl -e 'print "@ARGV\n"' "$@"; } export -f pecho - gen500k() { yes | head -c 131000; } + geny() { yes | head -c $1; } for a in `seq 6000`; do eval "export a$a=1" ; done - gen500k | stdout parallel --load 27 -Xkj1 'pecho {} {} {} {} | wc' | - perl -pe 's/\d{10,}.\d+ //g' + geny 10000 | stdout parallel -Xkj1 'pecho {} {} {} {} | wc' | + perl -pe 's/( y){10,}//g' } par_many_var() { + export LC_ALL=C rm -f ~/.parallel/tmp/sshlogin/*/linelen pecho() { perl -e 'print "@ARGV\n"' "$@"; } export -f pecho - gen500k() { seq -f %f 1000000000000000 1000000000050000 | head -c 131000; } + gen() { seq -f %f 1000000000000000 1000000000050000 | head -c $1; } for a in `seq 6000`; do eval "export a$a=1" ; done - gen500k | stdout parallel --load 4 -Xkj1 'pecho {} {} {} {} | wc' | + gen 10000 | stdout parallel -Xkj1 'pecho {} {} {} {} | wc' | perl -pe 's/\d{10,}.\d+ //g' } par_many_var_func() { + export LC_ALL=C rm -f ~/.parallel/tmp/sshlogin/*/linelen - gen500k() { seq -f %f 1000000000000000 1000000000050000 | head -c 131000; } + gen() { seq -f %f 1000000000000000 1000000000050000 | head -c $1; } pecho() { perl -e 'print "@ARGV\n"' "$@"; } export -f pecho - for a in `seq 5000`; do eval "export a$a=1" ; done - for a in `seq 5000`; do eval "a$a() { 1; }" ; done - for a in `seq 5000`; do eval export -f a$a ; done - gen500k | stdout parallel --load 20 -Xkj1 'pecho {} {} {} {} | wc' | + for a in `seq 2000`; do eval "export a$a=1" ; done + for a in `seq 2000`; do eval "a$a() { 1; }" ; done + for a in `seq 2000`; do eval export -f a$a ; done + gen 20000 | stdout parallel -Xkj1 'pecho {} {} {} {} | wc' | perl -pe 's/\d{10,}.\d+ //g' } par_many_func() { + export LC_ALL=C rm -f ~/.parallel/tmp/sshlogin/*/linelen - gen500k() { seq -f %f 1000000000000000 1000000000050000 | head -c 131000; } + gen() { seq -f %f 1000000000000000 1000000000050000 | head -c $1; } pecho() { perl -e 'print "@ARGV\n"' "$@"; } export -f pecho for a in `seq 5000`; do eval "a$a() { 1; }" ; done for a in `seq 5000`; do eval export -f a$a ; done - gen500k | stdout parallel --load 5 -Xkj1 'pecho {} {} {} {} | wc' | + gen 10000 | stdout parallel -Xkj1 'pecho {} {} {} {} | wc' | perl -pe 's/\d{10,}.\d+ //g' } par_big_func() { + export LC_ALL=C rm -f ~/.parallel/tmp/sshlogin/*/linelen - gen500k() { seq -f %f 1000000000000000 1000000000050000 | head -c 131000; } + gen() { seq -f %f 1000000000000000 1000000000050000 | head -c $1; } pecho() { perl -e 'print "@ARGV\n"' "$@"; } export -f pecho big=`seq 1000` for a in `seq 1`; do eval "a$a() { '$big'; }" ; done for a in `seq 1`; do eval export -f a$a ; done - gen500k | stdout parallel --load 2 -Xkj1 'pecho {} {} {} {} | wc' | + gen 20000 | stdout parallel --load 2 -Xkj1 'pecho {} {} {} {} | wc' | perl -pe 's/\d{10,}.\d+ //g' } par_many_var_big_func() { + export LC_ALL=C rm -f ~/.parallel/tmp/sshlogin/*/linelen - gen500k() { seq -f %f 1000000000000000 1000000000050000 | head -c 131000; } + gen() { seq -f %f 1000000000000000 1000000000050000 | head -c $1; } pecho() { perl -e 'print "@ARGV\n"' "$@"; } export -f pecho big=`seq 1000` for a in `seq 5000`; do eval "export a$a=1" ; done for a in `seq 10`; do eval "a$a() { '$big'; }" ; done for a in `seq 10`; do eval export -f a$a ; done - gen500k | stdout parallel --load 5 -Xkj1 'pecho {} {} {} {} | wc' | + gen 10000 | stdout parallel -Xkj1 'pecho {} {} {} {} | wc' | perl -pe 's/\d{10,}.\d+ //g' } par_big_func_name() { + export LC_ALL=C rm -f ~/.parallel/tmp/sshlogin/*/linelen - gen500k() { seq -f %f 1000000000000000 1000000000050000 | head -c 131000; } + gen() { seq -f %f 1000000000000000 1000000000050000 | head -c $1; } pecho() { perl -e 'print "@ARGV\n"' "$@"; } export -f pecho big=`perl -e print\"x\"x10000` for a in `seq 10`; do eval "export a$big$a=1" ; done - gen500k | stdout parallel --load 5 -Xkj1 'pecho {} {} {} {} | wc' | + gen 15000 | stdout parallel -Xkj1 'pecho {} {} {} {} | wc' | perl -pe 's/\d{10,}.\d+ //g' } par_big_var_func_name() { + export LC_ALL=C rm -f ~/.parallel/tmp/sshlogin/*/linelen - gen500k() { seq -f %f 1000000000000000 1000000000050000 | head -c 131000; } + gen() { seq -f %f 1000000000000000 1000000000050000 | head -c $1; } pecho() { perl -e 'print "@ARGV\n"' "$@"; } export -f pecho big=`perl -e print\"x\"x10000` for a in `seq 10`; do eval "export a$big$a=1" ; done for a in `seq 10`; do eval "a$big$a() { 1; }" ; done for a in `seq 10`; do eval export -f a$big$a ; done - gen500k | stdout parallel --load 4 -Xkj1 'pecho {} {} {} {} | wc' | + gen 10000 | stdout parallel --load 4 -Xkj1 'pecho {} {} {} {} | wc' | perl -pe 's/\d{10,}.\d+ //g' } -scp /usr/local/bin/parallel macosx.p:bin/ +#macsshlogin=ota@mac +macsshlogin=macosx.p +scp /usr/local/bin/parallel $macsshlogin:bin/ +export LC_ALL=C export -f $(compgen -A function | grep par_) -#compgen -A function | compgen -A function | grep par_ | LC_ALL=C sort | - env_parallel --timeout 3000% --tag -k -S macosx.p + env_parallel --timeout 1000% --tag -k -S 6/$macsshlogin 'PATH=$HOME/bin:$PATH; {}'