tange/parallel
1
0
Fork 0
mirror of https://git.savannah.gnu.org/git/parallel.git synced 2024-11-21 13:37:56 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

parallel: Fixed bug #59010: Max command line length for MacOS

Browse source
This commit is contained in:
Ole Tange 2020-09-12 23:33:55 +02:00
parent 555863c0b5
commit 31031e521d
15 changed files with 336 additions and 3684 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
doc/haikus
View file

@ -1,9 +1,14 @@
Quote of the month:
It's not a data migration party until GNU Parallel is involved...
involved
involved
-- rrees @rrees@twitter
Great tool, gets jobs done fast.
Great tool, gets jobs done fast.
Great tool, gets jobs done fast.
-- Paul F. De La Cruz @pdelacruzcc
-- Paul F. De La Cruz @pdelacruzcc@twitter
I get a weird sense of satisfaction every single time I see the
lovely logo of #GNU Parallel (plus, what an underrated piece of

18
src/parallel
View file

@ -7407,16 +7407,19 @@ sub sct_gnu_linux($) {
my %phy_seen;
my $physicalid;
for(@cpuinfo) {
# physical id : 0
if(/^physical id.*[:](.*)/) {
$physicalid = $1;
if(not $phy_seen{$1}++) {
$cpu->{'sockets'}++;
}
}
# core id : 3
if(/^core id.*[:](.*)/ and not $seen{$physicalid,$1}++) {
$cpu->{'cores'}++;
}
/^processor.*[:]/i and $cpu->{'threads'}++;
# processor : 2
/^processor.*[:]\s*\d/i and $cpu->{'threads'}++;
}
$cpu->{'cores'} ||= $cpu->{'threads'} || $cpu->{'sockets'};
}
@ -10519,7 +10522,7 @@ sub slot($) {
my $envc = (keys %ENV);
my $envn = length join"",(keys %ENV);
my $envv = length join"",(values %ENV);
$darwin_max_len = 3+($max_len - $envn - $envv)/5 - $envc*2;
$darwin_max_len = -146+($max_len - $envn - $envv) - $envc*10;
::debug("init",
"length: $darwin_max_len ".
"3+($max_len - $envn - $envv)/5 - $envc*2");
@ -10840,6 +10843,8 @@ sub len($) {
# Length of command w/ all replacement args removed
$len += $self->{'len'}{'noncontext'} + @{$self->{'command'}} -1;
::debug("length", "noncontext + command: $len\n");
# MacOS has an overhead of 8 bytes per argument
my $darwin = ($^O eq "darwin") ? 8 : 0;
my $recargs = $self->number_of_recargs();
if($self->{'context_replace'}) {
# Context is duplicated for each arg
@ -10858,16 +10863,23 @@ sub len($) {
" Groups: ", $self->{'len'}{'noncontextgroups'}, "\n");
# Add space between context groups
$len += ($recargs-1) * ($self->{'len'}{'contextgroups'});
if($darwin) {
$len += $recargs * $self->{'len'}{'contextgroups'} * $darwin;
}
} else {
# Each replacement string may occur several times
# Add the length for each time
$len += 1*$self->{'len'}{'context'};
::debug("length", "context+noncontext + command: $len\n");
for my $replstring (keys %{$self->{'replacecount'}}) {
# (space between regargs + length of replacement)
# (space between recargs + length of replacement)
# * number this replacement is used
$len += ($recargs -1 + $self->{'len'}{$replstring}) *
$self->{'replacecount'}{$replstring};
if($darwin) {
$len += ($recargs * $self->{'replacecount'}{$replstring}
* $darwin);
}
}
}
if(defined $Global::parallel_env) {

23
testsuite/REQUIREMENTS
View file

@ -18,7 +18,8 @@ install_packages() {
# SHEBANG TOOLS
shebang_pkgs="gnuplot octave ruby r-base-core"
# BUILD TOOLS
build_pkgs="bison"
# SQL TOOLS
sql_pkgs="libdbd-pg-perl libdbd-sqlite3-perl libdbd-csv-perl"
sql_pkgs="$sql_pkgs libdbd-mysql-perl rlwrap"
@ -315,6 +316,10 @@ bash_versions() {
echo You may have to kill conftest
(cd /tmp
git clone https://git.savannah.gnu.org/git/bash.git
listtags() {
(cd bash;
git tag | grep -v -- '-.*-' | grep -v bash-4.1.11)
}
make_one() {
rsync -a --delete bash/ $1/
cd $1
@ -326,10 +331,10 @@ bash_versions() {
}
export -f make_one
echo '# Building bash'
(cd bash; git tag | grep -v -- '-.*-') |
listtags |
stdout parallel --lb --tag \
'/usr/local/bin/{} --version || make_one {}'
(cd bash; git tag | grep -v -- '-.*-') |
listtags |
parallel -k -v --tag '/usr/local/bin/{} --version'
)
}
@ -337,21 +342,28 @@ bash_versions() {
rsync_versions() {
(cd /tmp
git clone https://github.com/WayneD/rsync
listtags() {
(cd rsync;
git tag | grep -v -- '-.*-' | grep -v pre)
}
make_one() {
rsync -a --delete rsync/ rsync-$1/
cd rsync-$1
git reset --hard
git checkout $1
autoreconf --install -W gnu
make proto
# Make "lib/addrinfo.h" ?
LDFLAGS=-static ./configure &&
make -j2 &&
sudo cp rsync /usr/local/bin/rsync-$1
}
export -f make_one
echo '# Building rsync'
(cd rsync; git tag | grep -v -- '-.*-' | grep -v pre) |
listtags |
stdout parallel --lb --tag \
'/usr/local/bin/rsync-{} --version || make_one {}'
(cd rsync; git tag | grep -v -- '-.*-' | grep -v pre) |
listtags |
parallel -k -v --tag '/usr/local/bin/rsync-{} --version'
)
}
@ -375,5 +387,6 @@ run() {
add_freebsd &&
tmux_versions &&
bash_versions &&
rsync_versions &&
misc
}

14
testsuite/tests-to-run/parallel-freebsd.sh
View file

@ -65,20 +65,6 @@ par_shebang() {
./shebang-wrap-opt wrap works with options
}
par_shellshock_bug() {
bash -c 'echo bug \#43358: shellshock breaks exporting functions using --env name;
echo Non-shellshock-hardened to non-shellshock-hardened;
funky() { echo Function $1; };
export -f funky;
PARALLEL_SHELL=bash parallel --env funky -S localhost funky ::: non-shellshock-hardened'
bash -c 'echo bug \#43358: shellshock breaks exporting functions using --env name;
echo Non-shellshock-hardened to shellshock-hardened;
funky() { echo Function $1; };
export -f funky;
PARALLEL_SHELL=bash parallel --env funky -S parallel@192.168.1.72 funky ::: shellshock-hardened'
}
par_load() {
echo '### Test --load (must give 1=true)'
parallel -j0 -N0 --timeout 5 --nice 10 'bzip2 < /dev/zero >/dev/null' ::: 1 2 3 4 5 6 &

3804
testsuite/tests-to-run/parallel-local-1s.sh
View file

File diff suppressed because it is too large Load diff

2
testsuite/tests-to-run/parallel-local-ssh8.sh
View file

@ -62,7 +62,7 @@ par_path_remote_csh() {
echo Done
_EOS
stdout ssh nopathcsh@lo -T |
perl -ne '/logged in/..0 and print' |
perl -ne '/Users logged in/ and next; /logged in/..0 and print' |
uniq
}

7
testsuite/tests-to-run/parallel-local19.sh
View file

@ -58,13 +58,6 @@ echo '### Test -m with 10000 args';
wait;
sleep 1
echo '### Test -X with 10000 args';
seq 10000 | perl -pe 's/$/.gif/' |
parallel -j1 -kX echo a{}b{.}c{.} |
tee >(wc; sleep 1) >(md5sum; sleep 1) >/dev/null;
wait;
sleep 1
echo '### Test -X with 10000 args and 5 expansions'
seq 10000 | perl -pe 's/$/.gif/' | parallel -j1 -kX echo a{}b{.}c{.}{.}{} | wc -l
seq 10000 | perl -pe 's/$/.gif/' | parallel -j1 -kX echo a{}b{.}c{.}{.} | wc -l

34
testsuite/tests-to-run/parallel-macos.sh
View file

@ -10,7 +10,7 @@ true <<'EOF'
# macosx.p = 10.7.5
# El capitan = 10.11.4
. `which binsearch`
. `which find-first-fail`
doit() {
nfunc=$1
lfunc=$2
@ -18,6 +18,7 @@ doit() {
nvar=$4
lvar=$5
lvarname=$6
onechar=$7
varval="$(perl -e 'print "x "x('$lvar'/2)')"
varname=$(perl -e 'print "x"x'$lvarname)
funcval="$(perl -e 'print "x "x('$lfunc'/2)')"
@ -26,27 +27,32 @@ doit() {
for a in `seq $nfunc`; do eval "f$funcname$a() { $funcval; }" ; done
for a in `seq $nfunc`; do eval "export -f f$funcname$a" ; done
myrun() {
/bin/echo $(perl -e 'print " x"x('$1'/2-5)')
/bin/echo $(perl -e 'print "a"x('$2')." x"x('$1'/2)')
}
export -f myrun
binlen=dummy
binlen=$(binsearch -q myrun)
binlen=$(find-first-fail -q myrun $onechar)
perl -e '
$envc=(keys %ENV);
$envn=length join"",(keys %ENV);
$envv=length join"",(values %ENV);
$maxlen=3+(262144 - $envn - $envv) / 5 - $envc*2;
$onechar='$onechar';
$maxlen=5-39+262144 - $envn - $envv - $onechar*5 - $envc*10;
print("Max len = $maxlen\n");
$bin='$binlen';
print("$bin=",$bin-$maxlen," $envc $envn $envv\n");
print("$bin=",$bin-$maxlen," $onechar $envc $envn $envv\n");
'
}
export -f doit
val="$(seq 2 100 1000)"
val="10 20 50 100 200 500 1000"
val="11 23 57 101 207 503 1007"
parallel --shuf --tag -k doit ::: $val ::: $val ::: $val ::: $val ::: $val ::: $val
val="12 103 304 506 1005"
parallel --timeout 20 --shuf --tag -k doit ::: $val ::: $val ::: $val ::: $val ::: $val ::: $val ::: $val
# Test with random data
(seq 10;seq 100;seq 100;seq 100;seq 100; seq 300 ;seq 1000) |
shuf | parallel -n 7 --timeout 20 --tag -k doit
EOF
# Each should generate at least 2 commands
@ -68,7 +74,7 @@ par_many_var() {
export -f pecho
gen() { seq -f %f 1000000000000000 1000000000050000 | head -c $1; }
for a in `seq 6000`; do eval "export a$a=1" ; done
gen 10000 | stdout parallel -Xkj1 'pecho {} {} {} {} | wc' |
gen 40000 | stdout parallel -Xkj1 'pecho {} {} {} {} | wc' |
perl -pe 's/\d{10,}.\d+ //g'
}
@ -81,7 +87,7 @@ par_many_var_func() {
for a in `seq 2000`; do eval "export a$a=1" ; done
for a in `seq 2000`; do eval "a$a() { 1; }" ; done
for a in `seq 2000`; do eval export -f a$a ; done
gen 20000 | stdout parallel -Xkj1 'pecho {} {} {} {} | wc' |
gen 40000 | stdout parallel -Xkj1 'pecho {} {} {} {} | wc' |
perl -pe 's/\d{10,}.\d+ //g'
}
@ -93,7 +99,7 @@ par_many_func() {
export -f pecho
for a in `seq 5000`; do eval "a$a() { 1; }" ; done
for a in `seq 5000`; do eval export -f a$a ; done
gen 10000 | stdout parallel -Xkj1 'pecho {} {} {} {} | wc' |
gen 40000 | stdout parallel -Xkj1 'pecho {} {} {} {} | wc' |
perl -pe 's/\d{10,}.\d+ //g'
}
@ -106,7 +112,7 @@ par_big_func() {
big=`seq 1000`
for a in `seq 1`; do eval "a$a() { '$big'; }" ; done
for a in `seq 1`; do eval export -f a$a ; done
gen 20000 | stdout parallel --load 2 -Xkj1 'pecho {} {} {} {} | wc' |
gen 80000 | stdout parallel --load 2 -Xkj1 'pecho {} {} {} {} | wc' |
perl -pe 's/\d{10,}.\d+ //g'
}
@ -120,7 +126,7 @@ par_many_var_big_func() {
for a in `seq 5000`; do eval "export a$a=1" ; done
for a in `seq 10`; do eval "a$a() { '$big'; }" ; done
for a in `seq 10`; do eval export -f a$a ; done
gen 10000 | stdout parallel -Xkj1 'pecho {} {} {} {} | wc' |
gen 40000 | stdout parallel -Xkj1 'pecho {} {} {} {} | wc' |
perl -pe 's/\d{10,}.\d+ //g'
}
@ -132,7 +138,7 @@ par_big_func_name() {
export -f pecho
big=`perl -e print\"x\"x10000`
for a in `seq 10`; do eval "export a$big$a=1" ; done
gen 15000 | stdout parallel -Xkj1 'pecho {} {} {} {} | wc' |
gen 30000 | stdout parallel -Xkj1 'pecho {} {} {} {} | wc' |
perl -pe 's/\d{10,}.\d+ //g'
}
@ -146,7 +152,7 @@ par_big_var_func_name() {
for a in `seq 10`; do eval "export a$big$a=1" ; done
for a in `seq 10`; do eval "a$big$a() { 1; }" ; done
for a in `seq 10`; do eval export -f a$big$a ; done
gen 10000 | stdout parallel --load 4 -Xkj1 'pecho {} {} {} {} | wc' |
gen 80000 | stdout parallel --load 4 -Xkj1 'pecho {} {} {} {} | wc' |
perl -pe 's/\d{10,}.\d+ //g'
}

15
testsuite/tests-to-run/parallel-virtualbox1.sh
View file

@ -16,21 +16,6 @@ par_warning_on_centos3() {
::: /usr/local/bin/parallel-20120822 `which parallel`
}
par_shellshock() {
# Bash on centos3 is non-shellshock-hardened
echo '### bug #43358: shellshock breaks exporting functions using --env'
echo shellshock-hardened to shellshock-hardened
funky() { echo Function $1; }
export -f funky
parallel --env funky -S parallel@localhost funky ::: shellshock-hardened
echo '2bug #43358: shellshock breaks exporting functions using --env'
echo shellshock-hardened to non-shellshock-hardened
funky() { echo Function $1; }
export -f funky
parallel --env funky -S centos3 funky ::: non-shellshock-hardened
}
export -f $(compgen -A function | grep par_)
compgen -A function | grep par_ | LC_ALL=C sort |
parallel --timeout 1000% -j6 --tag -k --joblog /tmp/jl-`basename $0` '{} 2>&1' |

9
testsuite/wanted-results/parallel-centos3 Normal file
View file

@ -0,0 +1,9 @@
### These tests requires VirtualBox running with the following images
vagrant@centos3
par_shellshock_bug par_shellshock_bug 2>&1
par_shellshock_bug bug #43358: shellshock breaks exporting functions using --env name
par_shellshock_bug Non-shellshock-hardened to non-shellshock-hardened
par_shellshock_bug Function non-shellshock-hardened
par_shellshock_bug bug #43358: shellshock breaks exporting functions using --env name
par_shellshock_bug Non-shellshock-hardened to shellshock-hardened
par_shellshock_bug Function shellshock-hardened

7
testsuite/wanted-results/parallel-freebsd
View file

@ -63,10 +63,3 @@ par_shebang /usr/local/bin/perl -w ./shebang-wrap-opt with
par_shebang with
par_shebang /usr/local/bin/perl -w ./shebang-wrap-opt options
par_shebang options
par_shellshock_bug par_shellshock_bug 2>&1
par_shellshock_bug bug #43358: shellshock breaks exporting functions using --env name
par_shellshock_bug Non-shellshock-hardened to non-shellshock-hardened
par_shellshock_bug Function non-shellshock-hardened
par_shellshock_bug bug #43358: shellshock breaks exporting functions using --env name
par_shellshock_bug Non-shellshock-hardened to shellshock-hardened
par_shellshock_bug Function shellshock-hardened

35
testsuite/wanted-results/parallel-local-1s
View file

@ -714,17 +714,30 @@ par_test_XI_mI a7 b1 2 3 4 5 6 7
par_test_XI_mI a8 b1 2 3 4 5 6 7 8
par_test_XI_mI a9 b1 2 3 4 5 6 7 8 9
par_test_XI_mI a10 b1 2 3 4 5 6 7 8 9 10
par_test_cpu_detection 2-8-8-8 2 8 8 8
par_test_cpu_detection 1-4-8-4 1 4 8 4
par_test_cpu_detection 1-2-4-2 1 2 4 2
par_test_cpu_detection 1-2-2-2 1 2 2 2
par_test_cpu_detection 2-24-48-24 2 24 48 24
par_test_cpu_detection 1-2-2-2 1 2 2 2
par_test_cpu_detection 1-8-8-8 1 8 8 8
par_test_cpu_detection 1-4-4-4 1 4 4 4
par_test_cpu_detection 1-6-6-6 1 6 6 6
par_test_cpu_detection 4-48-48-48 4 24 48 24
par_test_cpu_detection 1-4-8-4 1 4 8 4
par_test_cpu_detection 2-8-8-8 Xeon 8 core server in Germany
par_test_cpu_detection 2 8 8 8
par_test_cpu_detection 1-4-8-4 Core i7-3632QM Acer laptop
par_test_cpu_detection 1 4 8 4
par_test_cpu_detection 1-2-4-2 Core i5-2410M laptop firewall
par_test_cpu_detection 1 2 4 2
par_test_cpu_detection 1-2-2-2 dual core laptop(?)
par_test_cpu_detection 1 2 2 2
par_test_cpu_detection 2-24-48-24 24-core (maxwell?)
par_test_cpu_detection 2 24 48 24
par_test_cpu_detection 1-2-2-2 HP Laptop Compaq 6530b
par_test_cpu_detection 1 2 2 2
par_test_cpu_detection 1-8-8-8 Huawei P Smart Octa-core (4x2.36 GHz Cortex-A53 & 4x1.7 GHz Cortex-A53)
par_test_cpu_detection 1 8 8 8
par_test_cpu_detection 1-4-4-4 x96 quad-core Android TV-box
par_test_cpu_detection 1 4 4 4
par_test_cpu_detection 1-6-6-6 Kramses 200 USD laptop 6-core
par_test_cpu_detection 1 6 6 6
par_test_cpu_detection 4-48-48-48 Dell R815 4 CPU 48-core
par_test_cpu_detection 4 24 48 24
par_test_cpu_detection 1-4-8-4 4-core/8 thread Lenovo T480
par_test_cpu_detection 1 4 8 4
par_test_cpu_detection 4-64-64-64 Dell R815 4 CPU 64-core
par_test_cpu_detection 4 32 64 32
par_test_gt_quoting ### Test of quoting of > bug
par_test_gt_quoting >/dev/null
par_test_gt_quoting ### Test of quoting of > bug if line continuation

5
testsuite/wanted-results/parallel-local19
View file

@ -108,11 +108,6 @@ a1.gifb1c1 a2.gifb2c2 a3.gifb3c3 a4.gifb4c4 a5.gifb5c5 a6.gifb6c6
echo '### Test -m with 10000 args'; seq 10000 | perl -pe 's/$/.gif/' | parallel -j1 -km echo a{}b{.}c{.} | tee >(wc; sleep 1) >(md5sum; sleep 1) >/dev/null; wait; sleep 1
### Test -m with 10000 args
c606aec1723ee5cc15f2a1b95d83d3cf -
2 29996 186684
echo '### Test -X with 10000 args'; seq 10000 | perl -pe 's/$/.gif/' | parallel -j1 -kX echo a{}b{.}c{.} | tee >(wc; sleep 1) >(md5sum; sleep 1) >/dev/null; wait; sleep 1
### Test -X with 10000 args
2830a5b41659f3c0bb34a755fe5f1518 -
2 10000 196682
echo '### Test -X with 10000 args and 5 expansions'
### Test -X with 10000 args and 5 expansions
seq 10000 | perl -pe 's/$/.gif/' | parallel -j1 -kX echo a{}b{.}c{.}{.}{} | wc -l

33
testsuite/wanted-results/parallel-macos
View file

@ -1,16 +1,17 @@
par_big_func 1 2124 50976
par_big_func 1 1212 29028
par_big_func_name 1 1324 31776
par_big_func_name 1 1176 28224
par_big_var_func_name 1 1324 31776
par_big_var_func_name 1 344 8228
par_many_args 1 16420 32840
par_many_args 1 3580 7160
par_many_func 1 1168 28032
par_many_func 1 500 11972
par_many_var 1 1368 32832
par_many_var 1 300 7172
par_many_var_big_func 1 1176 28224
par_many_var_big_func 1 492 11780
par_many_var_func 1 1768 42432
par_many_var_func 1 1568 37572
par_big_func 1 7968 191232
par_big_func 1 5368 128772
par_big_func_name 1 4964 119136
par_big_func_name 1 36 864
par_big_var_func_name 1 4960 119040
par_big_var_func_name 1 4960 119040
par_big_var_func_name 1 3416 81924
par_many_args 1 16408 32816
par_many_args 1 3592 7184
par_many_func 1 4376 105024
par_many_func 1 2292 54980
par_many_var 1 5128 123072
par_many_var 1 1540 36932
par_many_var_big_func 1 4408 105792
par_many_var_big_func 1 2260 54212
par_many_var_func 1 6628 159072
par_many_var_func 1 40 932

7
testsuite/wanted-results/parallel-virtualbox1
View file

@ -1,10 +1,3 @@
par_shellshock ### bug #43358: shellshock breaks exporting functions using --env
par_shellshock shellshock-hardened to shellshock-hardened
par_shellshock Function shellshock-hardened
par_shellshock 2bug #43358: shellshock breaks exporting functions using --env
par_shellshock shellshock-hardened to non-shellshock-hardened
par_shellshock parallel: Warning: Could not figure out number of cpus on centos3 (). Using 1.
par_shellshock Function non-shellshock-hardened
par_warning_on_centos3 ### bug #37589: Red Hat 9 (Shrike) perl v5.8.0 built for i386-linux-thread-multi error
par_warning_on_centos3 vagrant@centos3 /usr/local/bin/parallel-20120822 Bareword found where operator expected at /tmp/parallel-20120822 line 1249, near "$Global::original_stderr init_progress"
par_warning_on_centos3 vagrant@centos3 /usr/local/bin/parallel-20120822 (Missing operator before init_progress?)