parallel: Fixed bug #59010: Max command line length for MacOS

doc/haikus

@@ -1,9 +1,14 @@
 Quote of the month:
 
+  It's not a data migration party until GNU Parallel is involved...
+  involved
+  involved
+    -- rrees @rrees@twitter
+
   Great tool, gets jobs done fast.
   Great tool, gets jobs done fast.
   Great tool, gets jobs done fast.
-    -- Paul F. De La Cruz @pdelacruzcc
+    -- Paul F. De La Cruz @pdelacruzcc@twitter
 
   I get a weird sense of satisfaction every single time I see the
   lovely logo of #GNU Parallel (plus, what an underrated piece of

src/parallel

@@ -7407,16 +7407,19 @@ sub sct_gnu_linux($) {
 	my %phy_seen;
 	my $physicalid;
 	for(@cpuinfo) {
+	    # physical id : 0
 	    if(/^physical id.*[:](.*)/) {
 		$physicalid = $1;
 		if(not $phy_seen{$1}++) {
 		    $cpu->{'sockets'}++;
 		}
 	    }
+	    # core id : 3
 	    if(/^core id.*[:](.*)/ and not $seen{$physicalid,$1}++) {
 		$cpu->{'cores'}++;
 	    }
-	    /^processor.*[:]/i and $cpu->{'threads'}++;
+	    # processor : 2
+	    /^processor.*[:]\s*\d/i and $cpu->{'threads'}++;
 	}
 	$cpu->{'cores'} ||= $cpu->{'threads'} || $cpu->{'sockets'};
     }
@@ -10519,7 +10522,7 @@ sub slot($) {
 		my $envc = (keys %ENV);
 		my $envn = length join"",(keys %ENV);
 		my $envv = length join"",(values %ENV);
-		$darwin_max_len = 3+($max_len - $envn - $envv)/5 - $envc*2;
+		$darwin_max_len = -146+($max_len - $envn - $envv) - $envc*10;
 		::debug("init",
 			"length: $darwin_max_len ".
 			"3+($max_len - $envn - $envv)/5 - $envc*2");
@@ -10840,6 +10843,8 @@ sub len($) {
     # Length of command w/ all replacement args removed
     $len += $self->{'len'}{'noncontext'} + @{$self->{'command'}} -1;
     ::debug("length", "noncontext + command: $len\n");
+    # MacOS has an overhead of 8 bytes per argument
+    my $darwin = ($^O eq "darwin") ? 8 : 0;
     my $recargs = $self->number_of_recargs();
     if($self->{'context_replace'}) {
 	# Context is duplicated for each arg
@@ -10858,16 +10863,23 @@ sub len($) {
 		" Groups: ", $self->{'len'}{'noncontextgroups'}, "\n");
 	# Add space between context groups
 	$len += ($recargs-1) * ($self->{'len'}{'contextgroups'});
+	if($darwin) {
+	    $len += $recargs * $self->{'len'}{'contextgroups'} * $darwin;
+	}
     } else {
 	# Each replacement string may occur several times
 	# Add the length for each time
 	$len += 1*$self->{'len'}{'context'};
 	::debug("length", "context+noncontext + command: $len\n");
 	for my $replstring (keys %{$self->{'replacecount'}}) {
-	    # (space between regargs + length of replacement)
+	    # (space between recargs + length of replacement)
 	    # * number this replacement is used
 	    $len += ($recargs -1 + $self->{'len'}{$replstring}) *
 		$self->{'replacecount'}{$replstring};
+	    if($darwin) {
+		$len += ($recargs * $self->{'replacecount'}{$replstring}
+			 * $darwin);
+	    }
 	}
     }
     if(defined $Global::parallel_env) {

testsuite/REQUIREMENTS

@@ -18,7 +18,8 @@ install_packages() {
 
     # SHEBANG TOOLS
     shebang_pkgs="gnuplot octave ruby r-base-core"
-
+    # BUILD TOOLS
+    build_pkgs="bison"
     # SQL TOOLS
     sql_pkgs="libdbd-pg-perl libdbd-sqlite3-perl libdbd-csv-perl"
     sql_pkgs="$sql_pkgs libdbd-mysql-perl rlwrap"
@@ -315,6 +316,10 @@ bash_versions() {
     echo You may have to kill conftest
     (cd /tmp
      git clone https://git.savannah.gnu.org/git/bash.git
+     listtags() {
+	 (cd bash;
+	  git tag | grep -v -- '-.*-' | grep -v bash-4.1.11)
+     }
      make_one() {
 	 rsync -a --delete bash/ $1/
 	 cd $1
@@ -326,10 +331,10 @@ bash_versions() {
      }
      export -f make_one
      echo '# Building bash'
-     (cd bash; git tag | grep -v -- '-.*-') |
+     listtags |
 	 stdout parallel --lb --tag \
 		'/usr/local/bin/{} --version || make_one {}'
-     (cd bash; git tag | grep -v -- '-.*-') |
+     listtags |
 	 parallel -k -v --tag '/usr/local/bin/{} --version'
     )
 }
@@ -337,21 +342,28 @@ bash_versions() {
 rsync_versions() {
     (cd /tmp
      git clone https://github.com/WayneD/rsync
+     listtags() {
+	 (cd rsync;
+	  git tag | grep -v -- '-.*-' | grep -v pre)
+     }
      make_one() {
 	 rsync -a --delete rsync/ rsync-$1/
 	 cd rsync-$1
 	 git reset --hard
 	 git checkout $1
+         autoreconf --install -W gnu
+	 make proto
+	 # Make "lib/addrinfo.h" ?
 	 LDFLAGS=-static ./configure &&
 	     make -j2 &&
 	     sudo cp rsync /usr/local/bin/rsync-$1
      }
      export -f make_one
      echo '# Building rsync'
-     (cd rsync; git tag | grep -v -- '-.*-' | grep -v pre) |
+     listtags |
 	 stdout parallel --lb --tag \
 		'/usr/local/bin/rsync-{} --version || make_one {}'
-     (cd rsync; git tag | grep -v -- '-.*-' | grep -v pre) |
+     listtags |
 	 parallel -k -v --tag '/usr/local/bin/rsync-{} --version'
     )
 }
@@ -375,5 +387,6 @@ run() {
     add_freebsd &&
     tmux_versions &&
     bash_versions &&
+    rsync_versions &&
     misc
 }

testsuite/tests-to-run/parallel-freebsd.sh

@@ -65,20 +65,6 @@ par_shebang() {
     ./shebang-wrap-opt wrap works with options
 }
 
-par_shellshock_bug() {
-    bash -c 'echo bug \#43358: shellshock breaks exporting functions using --env name;
-      echo Non-shellshock-hardened to non-shellshock-hardened;
-      funky() { echo Function $1; };
-      export -f funky;
-      PARALLEL_SHELL=bash parallel --env funky -S localhost funky ::: non-shellshock-hardened'
-
-    bash -c 'echo bug \#43358: shellshock breaks exporting functions using --env name;
-      echo Non-shellshock-hardened to shellshock-hardened;
-      funky() { echo Function $1; };
-      export -f funky;
-      PARALLEL_SHELL=bash parallel --env funky -S parallel@192.168.1.72 funky ::: shellshock-hardened'
-}
-
 par_load() {
     echo '### Test --load (must give 1=true)'
     parallel -j0 -N0 --timeout 5 --nice 10 'bzip2 < /dev/zero >/dev/null' ::: 1 2 3 4 5 6 &

testsuite/tests-to-run/parallel-local-ssh8.sh

@@ -62,7 +62,7 @@ par_path_remote_csh() {
   echo Done
 _EOS
   stdout ssh nopathcsh@lo -T |
-      perl -ne '/logged in/..0 and print' |
+      perl -ne '/Users logged in/ and next; /logged in/..0 and print' |
       uniq
 }
 

testsuite/tests-to-run/parallel-local19.sh

@@ -58,13 +58,6 @@ echo '### Test -m with 10000 args';
   wait; 
   sleep 1
 
-echo '### Test -X with 10000 args'; 
-  seq 10000 | perl -pe 's/$/.gif/' | 
-  parallel -j1 -kX echo a{}b{.}c{.} | 
-  tee >(wc; sleep 1) >(md5sum; sleep 1) >/dev/null; 
-  wait; 
-  sleep 1
-
 echo '### Test -X with 10000 args and 5 expansions'
   seq 10000 | perl -pe 's/$/.gif/' | parallel -j1 -kX echo a{}b{.}c{.}{.}{} | wc -l
   seq 10000 | perl -pe 's/$/.gif/' | parallel -j1 -kX echo a{}b{.}c{.}{.} | wc -l

testsuite/tests-to-run/parallel-macos.sh

@@ -10,7 +10,7 @@ true <<'EOF'
 # macosx.p = 10.7.5
 # El capitan = 10.11.4
 
-. `which binsearch`
+. `which find-first-fail`
 doit() {
     nfunc=$1
     lfunc=$2
@@ -18,6 +18,7 @@ doit() {
     nvar=$4
     lvar=$5
     lvarname=$6
+    onechar=$7
     varval="$(perl -e 'print "x "x('$lvar'/2)')"
     varname=$(perl -e 'print "x"x'$lvarname)
     funcval="$(perl -e 'print "x "x('$lfunc'/2)')"
@@ -26,27 +27,32 @@ doit() {
     for a in `seq $nfunc`; do eval "f$funcname$a() { $funcval; }" ; done
     for a in `seq $nfunc`; do eval "export -f f$funcname$a" ; done
     myrun() {
-	/bin/echo $(perl -e 'print " x"x('$1'/2-5)')
+	/bin/echo $(perl -e 'print "a"x('$2')." x"x('$1'/2)')
     }
     export -f myrun
     binlen=dummy
-    binlen=$(binsearch -q myrun)
+    binlen=$(find-first-fail -q myrun $onechar)
     perl -e '
     $envc=(keys %ENV);
     $envn=length join"",(keys %ENV);
     $envv=length join"",(values %ENV);
-    $maxlen=3+(262144 - $envn - $envv) / 5 - $envc*2;
+    $onechar='$onechar';
+    $maxlen=5-39+262144 - $envn - $envv - $onechar*5 - $envc*10;
     print("Max len = $maxlen\n");
     $bin='$binlen';
-    print("$bin=",$bin-$maxlen," $envc $envn $envv\n");
+    print("$bin=",$bin-$maxlen," $onechar $envc $envn $envv\n");
        '
 }
 export -f doit
 
 val="$(seq 2 100 1000)"
 val="10 20 50 100 200 500 1000"
-val="11 23 57 101 207 503 1007"
+val="12 103 304 506 1005"
-parallel --shuf --tag -k doit ::: $val ::: $val ::: $val ::: $val ::: $val ::: $val
+parallel --timeout 20 --shuf --tag -k doit ::: $val ::: $val ::: $val ::: $val ::: $val ::: $val ::: $val
+
+# Test with random data
+(seq 10;seq 100;seq 100;seq 100;seq 100; seq 300 ;seq 1000) |
+    shuf | parallel -n 7 --timeout 20 --tag -k doit
 EOF
 
 # Each should generate at least 2 commands
@@ -68,7 +74,7 @@ par_many_var() {
     export -f pecho
     gen() { seq -f %f 1000000000000000 1000000000050000 | head -c $1; }
     for a in `seq 6000`; do eval "export a$a=1" ; done
-    gen 10000 | stdout parallel -Xkj1  'pecho {} {} {} {} | wc' |
+    gen 40000 | stdout parallel -Xkj1  'pecho {} {} {} {} | wc' |
 	perl -pe 's/\d{10,}.\d+ //g'
 }
 
@@ -81,7 +87,7 @@ par_many_var_func() {
     for a in `seq 2000`; do eval "export a$a=1" ; done
     for a in `seq 2000`; do eval "a$a() { 1; }" ; done
     for a in `seq 2000`; do eval export -f a$a ; done
-    gen 20000 | stdout parallel -Xkj1  'pecho {} {} {} {} | wc' |
+    gen 40000 | stdout parallel -Xkj1  'pecho {} {} {} {} | wc' |
 	perl -pe 's/\d{10,}.\d+ //g'
 }
 
@@ -93,7 +99,7 @@ par_many_func() {
     export -f pecho
     for a in `seq 5000`; do eval "a$a() { 1; }" ; done
     for a in `seq 5000`; do eval export -f a$a ; done
-    gen 10000 | stdout parallel -Xkj1  'pecho {} {} {} {} | wc' |
+    gen 40000 | stdout parallel -Xkj1  'pecho {} {} {} {} | wc' |
 	perl -pe 's/\d{10,}.\d+ //g'
 }
 
@@ -106,7 +112,7 @@ par_big_func() {
     big=`seq 1000`
     for a in `seq 1`; do eval "a$a() { '$big'; }" ; done
     for a in `seq 1`; do eval export -f a$a ; done
-    gen 20000 | stdout parallel --load 2 -Xkj1  'pecho {} {} {} {} | wc' |
+    gen 80000 | stdout parallel --load 2 -Xkj1  'pecho {} {} {} {} | wc' |
 	perl -pe 's/\d{10,}.\d+ //g'
 }
 
@@ -120,7 +126,7 @@ par_many_var_big_func() {
     for a in `seq 5000`; do eval "export a$a=1" ; done
     for a in `seq 10`; do eval "a$a() { '$big'; }" ; done
     for a in `seq 10`; do eval export -f a$a ; done
-    gen 10000 | stdout parallel -Xkj1  'pecho {} {} {} {} | wc' |
+    gen 40000 | stdout parallel -Xkj1  'pecho {} {} {} {} | wc' |
 	perl -pe 's/\d{10,}.\d+ //g'
 }
 
@@ -132,7 +138,7 @@ par_big_func_name() {
     export -f pecho
     big=`perl -e print\"x\"x10000`
     for a in `seq 10`; do eval "export a$big$a=1" ; done
-    gen 15000 | stdout parallel -Xkj1  'pecho {} {} {} {} | wc' |
+    gen 30000 | stdout parallel -Xkj1  'pecho {} {} {} {} | wc' |
 	perl -pe 's/\d{10,}.\d+ //g'
 }
 
@@ -146,7 +152,7 @@ par_big_var_func_name() {
     for a in `seq 10`; do eval "export a$big$a=1" ; done
     for a in `seq 10`; do eval "a$big$a() { 1; }" ; done
     for a in `seq 10`; do eval export -f a$big$a ; done
-    gen 10000 | stdout parallel --load 4 -Xkj1  'pecho {} {} {} {} | wc' |
+    gen 80000 | stdout parallel --load 4 -Xkj1  'pecho {} {} {} {} | wc' |
 	perl -pe 's/\d{10,}.\d+ //g'
 }
 

testsuite/tests-to-run/parallel-virtualbox1.sh

@@ -16,21 +16,6 @@ par_warning_on_centos3() {
 	     ::: /usr/local/bin/parallel-20120822 `which parallel`
 }
 
-par_shellshock() {
-    # Bash on centos3 is non-shellshock-hardened
-    echo '### bug #43358: shellshock breaks exporting functions using --env'
-    echo shellshock-hardened to shellshock-hardened
-    funky() { echo Function $1; }
-    export -f funky
-    parallel --env funky -S parallel@localhost funky ::: shellshock-hardened
-
-    echo '2bug #43358: shellshock breaks exporting functions using --env'
-    echo shellshock-hardened to non-shellshock-hardened
-    funky() { echo Function $1; }
-    export -f funky
-    parallel --env funky -S centos3 funky ::: non-shellshock-hardened
-}
-
 export -f $(compgen -A function | grep par_)
 compgen -A function | grep par_ | LC_ALL=C sort |
     parallel --timeout 1000% -j6 --tag -k --joblog /tmp/jl-`basename $0` '{} 2>&1' |

testsuite/wanted-results/parallel-centos3

@@ -0,0 +1,9 @@
+### These tests requires VirtualBox running with the following images
+vagrant@centos3
+par_shellshock_bug	par_shellshock_bug 2>&1
+par_shellshock_bug	bug #43358: shellshock breaks exporting functions using --env name
+par_shellshock_bug	Non-shellshock-hardened to non-shellshock-hardened
+par_shellshock_bug	Function non-shellshock-hardened
+par_shellshock_bug	bug #43358: shellshock breaks exporting functions using --env name
+par_shellshock_bug	Non-shellshock-hardened to shellshock-hardened
+par_shellshock_bug	Function shellshock-hardened

testsuite/wanted-results/parallel-freebsd

@@ -63,10 +63,3 @@ par_shebang	/usr/local/bin/perl -w ./shebang-wrap-opt with
 par_shebang	with
 par_shebang	/usr/local/bin/perl -w ./shebang-wrap-opt options
 par_shebang	options
-par_shellshock_bug	par_shellshock_bug 2>&1
-par_shellshock_bug	bug #43358: shellshock breaks exporting functions using --env name
-par_shellshock_bug	Non-shellshock-hardened to non-shellshock-hardened
-par_shellshock_bug	Function non-shellshock-hardened
-par_shellshock_bug	bug #43358: shellshock breaks exporting functions using --env name
-par_shellshock_bug	Non-shellshock-hardened to shellshock-hardened
-par_shellshock_bug	Function shellshock-hardened

testsuite/wanted-results/parallel-local-1s

@@ -714,17 +714,30 @@ par_test_XI_mI	a7 b1 2 3 4 5 6 7
 par_test_XI_mI	a8 b1 2 3 4 5 6 7 8
 par_test_XI_mI	a9 b1 2 3 4 5 6 7 8 9
 par_test_XI_mI	a10 b1 2 3 4 5 6 7 8 9 10
-par_test_cpu_detection	2-8-8-8	2 8 8 8
+par_test_cpu_detection	2-8-8-8 Xeon 8 core server in Germany
-par_test_cpu_detection	1-4-8-4	1 4 8 4
+par_test_cpu_detection	2 8 8 8
-par_test_cpu_detection	1-2-4-2	1 2 4 2
+par_test_cpu_detection	1-4-8-4 Core i7-3632QM Acer laptop
-par_test_cpu_detection	1-2-2-2	1 2 2 2
+par_test_cpu_detection	1 4 8 4
-par_test_cpu_detection	2-24-48-24	2 24 48 24
+par_test_cpu_detection	1-2-4-2 Core i5-2410M laptop firewall
-par_test_cpu_detection	1-2-2-2	1 2 2 2
+par_test_cpu_detection	1 2 4 2
-par_test_cpu_detection	1-8-8-8	1 8 8 8
+par_test_cpu_detection	1-2-2-2 dual core laptop(?)
-par_test_cpu_detection	1-4-4-4	1 4 4 4
+par_test_cpu_detection	1 2 2 2
-par_test_cpu_detection	1-6-6-6	1 6 6 6
+par_test_cpu_detection	2-24-48-24 24-core (maxwell?)
-par_test_cpu_detection	4-48-48-48	4 24 48 24
+par_test_cpu_detection	2 24 48 24
-par_test_cpu_detection	1-4-8-4	1 4 8 4
+par_test_cpu_detection	1-2-2-2 HP Laptop Compaq 6530b
+par_test_cpu_detection	1 2 2 2
+par_test_cpu_detection	1-8-8-8 Huawei P Smart Octa-core (4x2.36 GHz Cortex-A53 & 4x1.7 GHz Cortex-A53)
+par_test_cpu_detection	1 8 8 8
+par_test_cpu_detection	1-4-4-4 x96 quad-core Android TV-box
+par_test_cpu_detection	1 4 4 4
+par_test_cpu_detection	1-6-6-6 Kramses 200 USD laptop 6-core
+par_test_cpu_detection	1 6 6 6
+par_test_cpu_detection	4-48-48-48 Dell R815 4 CPU 48-core
+par_test_cpu_detection	4 24 48 24
+par_test_cpu_detection	1-4-8-4 4-core/8 thread Lenovo T480
+par_test_cpu_detection	1 4 8 4
+par_test_cpu_detection	4-64-64-64 Dell R815 4 CPU 64-core
+par_test_cpu_detection	4 32 64 32
 par_test_gt_quoting	### Test of quoting of > bug
 par_test_gt_quoting	>/dev/null
 par_test_gt_quoting	### Test of quoting of > bug if line continuation

testsuite/wanted-results/parallel-local19

@@ -108,11 +108,6 @@ a1.gifb1c1 a2.gifb2c2 a3.gifb3c3 a4.gifb4c4 a5.gifb5c5 a6.gifb6c6
 echo '### Test -m with 10000 args';   seq 10000 | perl -pe 's/$/.gif/' |   parallel -j1 -km echo a{}b{.}c{.} |   tee >(wc; sleep 1) >(md5sum; sleep 1) >/dev/null;   wait;   sleep 1
 ### Test -m with 10000 args
 c606aec1723ee5cc15f2a1b95d83d3cf  -
-      2   29996  186684
-echo '### Test -X with 10000 args';   seq 10000 | perl -pe 's/$/.gif/' |   parallel -j1 -kX echo a{}b{.}c{.} |   tee >(wc; sleep 1) >(md5sum; sleep 1) >/dev/null;   wait;   sleep 1
-### Test -X with 10000 args
-2830a5b41659f3c0bb34a755fe5f1518  -
-      2   10000  196682
 echo '### Test -X with 10000 args and 5 expansions'
 ### Test -X with 10000 args and 5 expansions
   seq 10000 | perl -pe 's/$/.gif/' | parallel -j1 -kX echo a{}b{.}c{.}{.}{} | wc -l

testsuite/wanted-results/parallel-macos

@@ -1,16 +1,17 @@
-par_big_func	       1    2124   50976
+par_big_func	       1    7968  191232
-par_big_func	       1    1212   29028
+par_big_func	       1    5368  128772
-par_big_func_name	       1    1324   31776
+par_big_func_name	       1    4964  119136
-par_big_func_name	       1    1176   28224
+par_big_func_name	       1      36     864
-par_big_var_func_name	       1    1324   31776
+par_big_var_func_name	       1    4960  119040
-par_big_var_func_name	       1     344    8228
+par_big_var_func_name	       1    4960  119040
-par_many_args	       1   16420   32840
+par_big_var_func_name	       1    3416   81924
-par_many_args	       1    3580    7160
+par_many_args	       1   16408   32816
-par_many_func	       1    1168   28032
+par_many_args	       1    3592    7184
-par_many_func	       1     500   11972
+par_many_func	       1    4376  105024
-par_many_var	       1    1368   32832
+par_many_func	       1    2292   54980
-par_many_var	       1     300    7172
+par_many_var	       1    5128  123072
-par_many_var_big_func	       1    1176   28224
+par_many_var	       1    1540   36932
-par_many_var_big_func	       1     492   11780
+par_many_var_big_func	       1    4408  105792
-par_many_var_func	       1    1768   42432
+par_many_var_big_func	       1    2260   54212
-par_many_var_func	       1    1568   37572
+par_many_var_func	       1    6628  159072
+par_many_var_func	       1      40     932

testsuite/wanted-results/parallel-virtualbox1

@@ -1,10 +1,3 @@
-par_shellshock	### bug #43358: shellshock breaks exporting functions using --env
-par_shellshock	shellshock-hardened to shellshock-hardened
-par_shellshock	Function shellshock-hardened
-par_shellshock	2bug #43358: shellshock breaks exporting functions using --env
-par_shellshock	shellshock-hardened to non-shellshock-hardened
-par_shellshock	parallel: Warning: Could not figure out number of cpus on centos3 (). Using 1.
-par_shellshock	Function non-shellshock-hardened
 par_warning_on_centos3	### bug #37589: Red Hat 9 (Shrike) perl v5.8.0 built for i386-linux-thread-multi error
 par_warning_on_centos3	vagrant@centos3 /usr/local/bin/parallel-20120822	Bareword found where operator expected at /tmp/parallel-20120822 line 1249, near "$Global::original_stderr init_progress"
 par_warning_on_centos3	vagrant@centos3 /usr/local/bin/parallel-20120822		(Missing operator before init_progress?)