parallel: Fix https://lists.gnu.org/archive/html/parallel/2020-08/msg00003.html

src/parallel

@@ -6249,36 +6249,48 @@ sub memfree_recompute() {
 		# Buffers:         19876 kB
 		# Cached:         431192 kB
 		# SwapCached:          0 kB
-		"linux" =>
+		"linux" => (
-		q[ print 1024 * qx{ ].
+		    q{
-		q[   awk '/^((Swap)?Cached|MemFree|Buffers):/ ].
+			print 1024 * qx{
-		q[     { sum += \$2} END { print sum }' ].
+			awk '/^((Swap)?Cached|MemFree|Buffers):/
-		q[   /proc/meminfo } ],
+			{ sum += \$2} END { print sum }'
+			    /proc/meminfo }
+		    }),
 		# Android uses same code as GNU/Linux
-		"android" =>
+		"android" => (
-		q[ print 1024 * qx{ ].
+		    q{
-		q[   awk '/^((Swap)?Cached|MemFree|Buffers):/ ].
+			print 1024 * qx{
-		q[     { sum += \$2} END { print sum }' ].
+			awk '/^((Swap)?Cached|MemFree|Buffers):/
-		q[   /proc/meminfo } ],
+			{ sum += \$2} END { print sum }'
-
+			    /proc/meminfo }
+		    }),
 		# $ vmstat 1 1
-		#     procs           memory                   page                              faults       cpu
+		#   procs         memory         page                   faults       cpu
-		# r     b     w      avm    free   re   at    pi   po    fr   de    sr     in     sy    cs  us sy id
+		# r  b  w    avm    free re at  pi po  fr de  sr   in   sy  cs  us sy id
-		# 1     0     0   242793  389737    5    1     0    0     0    0     0    107    978    60   1  1 99
+		# 1  0  0 242793  389737  5  1   0  0   0  0   0  107  978  60   1  1 99
-		"hpux" =>
+		"hpux" => (
-		q[ print (((reverse `vmstat 1 1`)[0] ].
+		    q{
-		q[   =~ /(?:\d+\D+){4}(\d+)/)[0]*1024) ],
+			print (((reverse `vmstat 1 1`)[0]
+			    =~ /(?:\d+\D+){4}(\d+)/)[0]*1024)
+		    }),
 		# $ vmstat 1 2
 		# kthr      memory            page            disk          faults      cpu
 		# r b w   swap  free  re  mf pi po fr de sr s3 s4 -- --   in   sy   cs us sy id
 		# 0 0 0 6496720 5170320 68 260 8 2  1  0  0 -0  3  0  0  309 1371  255  1  2 97
-		# 0 0 0 6434088 5072656 7 15  8  0  0  0  0  0 261 0  0 1889 1899 3222  0  8 92
+		# 0 0 0 6434088 5072656  7  15 8 0  0  0  0  0 261 0  0 1889 1899 3222  0  8 92
 		#
 		# The second free value is correct
-		"solaris" =>
+		"solaris" => (
-		q[ print (((reverse `vmstat 1 2`)[0] ].
+		    q{
-		q[   =~ /(?:\d+\D+){4}(\d+)/)[0]*1024) ],
+			print (((reverse `vmstat 1 2`)[0]
-		"freebsd" => q{
+				=~ /(?:\d+\D+){4}(\d+)/)[0]*1024)
+		    }),
+		# hw.pagesize: 4096
+		# vm.stats.vm.v_cache_count: 0
+		# vm.stats.vm.v_inactive_count: 79574
+		# vm.stats.vm.v_free_count: 4507
+		"freebsd" => (
+		    q{
 			for(qx{/sbin/sysctl -a}) {
 			    if (/^([^:]+):\s+(.+)\s*$/s) {
 				$sysctl->{$1} = $2;
@@ -6288,7 +6300,7 @@ sub memfree_recompute() {
 			    ($sysctl->{"vm.stats.vm.v_cache_count"}
 			     + $sysctl->{"vm.stats.vm.v_inactive_count"}
 			     + $sysctl->{"vm.stats.vm.v_free_count"});
-		    },
+		    }),
 		# Mach Virtual Memory Statistics: (page size of 4096 bytes)
 		# Pages free:                         198061.
 		# Pages active:                       159701.
@@ -6302,12 +6314,13 @@ sub memfree_recompute() {
 	        # Pageins:                           1798068.
 	        # Pageouts:                              257.
 		# Object cache: 6603 hits of 1713223 lookups (0% hit rate)
-		'darwin' =>
+		'darwin' => (
-		q[ $vm = `vm_stat`;
+		    q{
-		  print (($vm =~ /page size of (\d+)/)[0] *
+			$vm = `vm_stat`;
-		  (($vm =~ /Pages free:\s+(\d+)/)[0] +
+			print (($vm =~ /page size of (\d+)/)[0] *
-		   ($vm =~ /Pages inactive:\s+(\d+)/)[0]));
+			       (($vm =~ /Pages free:\s+(\d+)/)[0] +
-		],
+				($vm =~ /Pages inactive:\s+(\d+)/)[0]));
+		    }),
 		);
 	    my $perlscript = "";
 	    # Make a perl script that detects the OS ($^O) and runs
@@ -6315,8 +6328,7 @@ sub memfree_recompute() {
 	    for my $os (keys %script_of) {
 		$perlscript .= 'if($^O eq "'.$os.'") { '.$script_of{$os}.'}';
 	    }
-	    $perlscript =~ s/[\t\n ]+/ /g;
+	    $script = "perl -e " . ::Q(::spacefree(1,$perlscript));
-	    $script = "perl -e " . ::Q($perlscript);
 	}
 	return $script;
     }
@@ -10469,6 +10481,7 @@ sub slot($) {
 
 {
     my $already_spread;
+    my $env_size;
 
     sub populate($) {
 	# Add arguments from arg_queue until the number of arguments or
@@ -10486,7 +10499,25 @@ sub slot($) {
 	my $next_arg;
 	my $max_len = $Global::minimal_command_line_length
 	    || Limits::Command::max_length();
-
+	if($^O eq "darwin") {
+	    # On Mac bash 3.2.48(1)-release the length also depends on
+	    # number of functions, variables and the size of those.
+	    # 21 is found by trial and error and seems to be a safe value.
+	    # test_with() {
+	    # 	gen500k() {
+	    # 	    seq -f %f 1000000000000000 1000000000050000 |
+	    # 		head -c 131000;
+	    # 	}
+	    # 	for a in `seq 5100`; do eval "export a$a=1" ; done;
+	    # 	for a in `seq 5100`; do eval "a$a() { 1; }" ; done;
+	    # 	for a in `seq 5100`; do eval export -f a$a ; done;
+	    # 	gen500k | stdout parallel -Xj1  'echo {} {} {} {} | wc' |
+	    # 	    perl -pe 's/\d{10,} //g'
+	    # }
+            $env_size ||= (keys %ENV)*21
+                + length(join'',keys %ENV) + length(join'',%ENV);
+            $max_len -= $env_size;
+        }
 	if($opt::cat or $opt::fifo) {
 	    # Get the empty arg added by --pipepart (if any)
 	    $Global::JobQueue->{'commandlinequeue'}->{'arg_queue'}->get();

src/parallel_alternatives.pod

@@ -2616,6 +2616,85 @@ https://pypi.org/project/papply/ (Last checked: 2020-04)
 
 =head2 Todo
 
+test_many_var() {
+gen500k() { seq -f %f 1000000000000000 1000000000050000 | head -c 131000; }
+for a in `seq 11000`; do eval "export a$a=1" ; done
+gen500k | stdout parallel --timeout 5 -Xj1  'echo {} {} {} {} | wc' | perl -pe 's/\d{3,5} //g'
+}
+
+test_many_var_func() {
+gen500k() { seq -f %f 1000000000000000 1000000000050000 | head -c 131000; }
+for a in `seq 5100`; do eval "export a$a=1" ; done
+for a in `seq 5100`; do eval "a$a() { 1; }" ; done
+for a in `seq 5100`; do eval export -f a$a ; done
+gen500k | stdout parallel --timeout 21 -Xj1  'echo {} {} {} {} | wc' | perl -pe 's/\d{3,5} //g'
+}
+
+test_many_var_func() {
+gen500k() { seq -f %f 1000000000000000 1000000000050000 | head -c 131000; }
+for a in `seq 8000`; do eval "a$a() { 1; }" ; done
+for a in `seq 8000`; do eval export -f a$a ; done
+gen500k | stdout parallel --timeout 6 -Xj1  'echo {} {} {} {} | wc' | perl -pe 's/\d{3,5} //g'
+}
+
+test_big_func() {
+gen500k() { seq -f %f 1000000000000000 1000000000050000 | head -c 131000; }
+big=`seq 1000`
+for a in `seq 50`; do eval "a$a() { '$big'; }" ; done
+for a in `seq 50`; do eval export -f a$a ; done
+gen500k | stdout parallel --timeout 4 -Xj1  'echo {} {} {} {} | wc' | perl -pe 's/\d{3,5} //g'
+}
+
+test_many_var_big_func() {
+gen500k() { seq -f %f 1000000000000000 1000000000050000 | head -c 131000; }
+big=`seq 1000`
+for a in `seq 5100`; do eval "export a$a=1" ; done
+for a in `seq 20`; do eval "a$a() { '$big'; }" ; done
+for a in `seq 20`; do eval export -f a$a ; done
+gen500k | stdout parallel --timeout 6 -Xj1  'echo {} {} {} {} | wc' | perl -pe 's/\d{3,5} //g'
+}
+
+test_big_func_name() {
+gen500k() { seq -f %f 1000000000000000 1000000000050000 | head -c 131000; }
+big=`perl -e print\"x\"x10000`
+for a in `seq 20`; do eval "export a$big$a=1" ; done
+gen500k | stdout parallel --timeout 8 -Xj1  'echo {} {} {} {} | wc' | perl -pe 's/\d{3,5} //g'
+}
+
+test_big_var_func_name() {
+gen500k() { seq -f %f 1000000000000000 1000000000050000 | head -c 131000; }
+big=`perl -e print\"x\"x10000`
+for a in `seq 2`; do eval "export a$big$a=1" ; done
+for a in `seq 2`; do eval "a$big$a() { '$big'; }" ; done
+for a in `seq 2`; do eval export -f a$big$a ; done
+gen500k | stdout parallel --timeout 1000 -Xj1  'echo {} {} {} {} | wc' | perl -pe 's/\d{3,5} //g'
+}
+
+
+
+tange@macosx:~$ for a in `seq 100`; do eval export a$a=fffffffffffffffffffffffff ; donetange@macosx:~$ seq 50000 | stdout parallel -Xj1  'echo {} {} | wc' | perl -pe 's/\d{3,5} //g'
+tange@macosx:~$ for a in `seq 100`; do eval export a$a=fffffffffffffffffffffffff ; donetange@macosx:~$ seq 50000 | stdout parallel -Xj1  'echo {} {} | wc' | perl -pe 's/\d{3,5} //g'
+tange@macosx:~$ for a in `seq 100`; do eval export -f a$a ; done
+
+
+seq 100000 | stdout parallel -Xj1  'echo {} {} | wc'
+export a=`seq 10000`
+seq 100000 | stdout parallel -Xj1  'echo {} {} | wc'
+
+
+
+    my $already_spread;
+    my $env_size;
+
+        if($^O eq "darwin") {
+            $env_size ||= 500+length(join'',%ENV);
+            $max_len -= $env_size;
+        }
+
+
+PASH: Light-touch Data-Parallel Shell Processing
+https://arxiv.org/pdf/2007.09436.pdf
+
 https://gitlab.com/netikras/bthread
 
 https://github.com/JeiKeiLim/simple_distribute_job

testsuite/REQUIREMENTS

@@ -6,26 +6,33 @@ install_packages() {
     INSTALL=${INSTALL:-"sudo aptitude -y install"}
 
     # The testsuite depends on this:
-    test_pkgs="imagemagick expect autossh sshpass jq libpod-simple-perl pod2pdf gawk"
+    test_pkgs="imagemagick expect autossh sshpass jq libpod-simple-perl"
-    test_pkgs="$test_pkgs lua5.3 clisp php-cli nodejs mono-csharp-shell libevent-dev"
+    test_pkgs="$test_pkgs pod2pdf gawk lua5.3 clisp php-cli nodejs"
-    test_pkgs="$test_pkgs tcl libtext-csv-perl xterm libc6-i386 libcrypt1:i386"
+    test_pkgs="$test_pkgs mono-csharp-shell libevent-dev tcl libtext-csv-perl"
+    test_pkgs="$test_pkgs xterm libc6-i386 libcrypt1:i386"
+    test_pkgs="$test_pkgs libtest-nowarnings-perl"
 
     # DEBIAN package
-    packaging_pkgs="dpkg-dev build-essential debhelper osc cvs automake-1.15 python3-m2crypto"
+    packaging_pkgs="dpkg-dev build-essential debhelper osc cvs automake-1.15"
+    packaging_pkgs="$packaging_pkgs python3-m2crypto alien"
     # SHEBANG TOOLS
     shebang_pkgs="gnuplot octave ruby r-base-core"
     # SQL TOOLS
-    sql_pkgs="libdbd-pg-perl libdbd-sqlite3-perl libdbd-csv-perl libdbd-mysql-perl rlwrap"
+    sql_pkgs="libdbd-pg-perl libdbd-sqlite3-perl libdbd-csv-perl"
+    sql_pkgs="$sql_pkgs libdbd-mysql-perl rlwrap"
     # Compression
-    compression_pkgs="zstd clzip liblz4-tool lzop pigz pixz gzip plzip pbzip2 lzma xz-utils lzip bzip2 lbzip2 lrzip"
+    compression_pkgs="zstd clzip liblz4-tool lzop pigz pixz gzip plzip pbzip2"
+    compression_pkgs="$compression_pkgs lzma xz-utils lzip bzip2 lbzip2 lrzip"
     compression_pkgs_missing="pxz"
     # Shells
     # (csh = bsd-csh that is broken)
-    shell_pkgs="ash dash fdclone fish fizsh ksh ksh93 mksh posh rc rush sash tcsh yash zsh"
+    shell_pkgs="ash dash fdclone fish fizsh ksh ksh93 mksh posh rc rush sash"
+    shell_pkgs="$shell_pkgs tcsh yash zsh"
     # Databases
     database_pkgs="postgresql mysql-server sqlite"
-    sudo dpkg --add-architecture i386
+    sudo dpkg --add-architecture i386; sudo apt update
-    if $INSTALL $test_pkgs $packaging_pkgs $shebang_pkgs $sql_pkgs $compression_pkgs $shell_pkgs $database_pkgs; then
+    if $INSTALL $test_pkgs $packaging_pkgs $shebang_pkgs $sql_pkgs \
+		$compression_pkgs $shell_pkgs $database_pkgs; then
 	# OK
 	true
     else
@@ -35,6 +42,53 @@ install_packages() {
     sudo apt update &
 }
 
+install_oracle_client() {
+    # https://salsa.debian.org/perl-team/modules/packages/libdbd-oracle-perl/blob/master/debian/README.Debian
+    # sql oracle:// && return 0
+    (cd /tmp
+
+     get_rpm_install_deb() {
+	 glob="$1"
+	 url="$oracleurl/$2"
+	 testglob="$3"
+	 if [ ! -e `echo $glob*rpm` ] ; then
+	     wget $url
+	 fi
+	 if [ ! -e `echo $glob*deb` ] ; then
+	     echo Convert `echo $glob*rpm` to deb
+	     fakeroot alien `echo $glob*rpm`
+	 fi
+	 if [ ! -e `echo $testglob` ] ; then
+	     echo Install `echo $glob*deb`
+	     sudo dpkg -i `echo $glob*deb`
+	 fi
+     }
+
+     oracleurl=https://download.oracle.com/otn_software/linux/instantclient/19600
+     client=oracle-instantclient19.6
+     ver=19.6.0.0.0-1.x86_64.rpm
+     get_rpm_install_deb 'oracle-instantclient*devel' $client-devel-$ver /usr/share/doc/oracle-instantclient*-devel/copyright
+     get_rpm_install_deb 'oracle-instantclient*basic'   $client-basic-$ver /usr/share/oracle/*/client*/doc/BASIC_README
+     get_rpm_install_deb 'oracle-instantclient*sqlplus' $client-sqlplus-$ver /usr/lib/oracle/*/client*/bin/sqlplus
+    )     
+
+    echo Add this to .bashrc:
+    ORACLE_HOME=`echo /usr/lib/oracle/*/client*`
+    echo ORACLE_HOME=$ORACLE_HOME
+    if grep -q ORACLE_HOME=/ ~/.bashrc; then
+	perl -i.old -pe "s:ORACLE_HOME=/.*:ORACLE_HOME=$ORACLE_HOME:" ~/.bashrc
+    else
+	echo ORACLE_HOME=$ORACLE_HOME >> ~/.bashrc
+	echo 'PATH=$PATH:$ORACLE_HOME/bin' >> ~/.bashrc
+	echo 'export ORACLE_HOME' >> ~/.bashrc
+	echo 'export ORACLE_SID=XE' >> ~/.bashrc
+    fi
+    perl -MCPAN -e 'install DBD::Oracle'
+    # TODO set up vagrant oracle server
+    # TODO set up default passwd
+    # test it works: sql oracle://
+}
+
 setup_databases() {
     # DATABASES
     echo '# Create PostgreSQL'
@@ -46,15 +100,15 @@ setup_databases() {
     sudo su - postgres -c "sql pg:/// \"ALTER USER \\\"`whoami`\\\" WITH PASSWORD '`whoami`';\""
 
     mysqlrootpass=${mysqlrootpass:-b+Ydjq4ejT4E}
-
+    dburl=mysql://root:"$mysqlrootpass"@/mysql
     echo '# Create MySQL'
     sudo su mysql mysqladmin create `whoami`
     # Default Debian way of getting "root" access
     sudo mysql --defaults-file=/etc/mysql/debian.cnf mysql <<< "ALTER USER 'root'@'localhost' IDENTIFIED BY '$mysqlrootpass';"
 
     # Drop database and user if needed
-    sudo sql mysql://root:"$mysqlrootpass"@/mysql "DROP DATABASE `whoami`;DROP USER '`whoami`'@'localhost';"
+    sudo sql "$dburl" "DROP DATABASE `whoami`;DROP USER '`whoami`'@'localhost';"
-    sudo sql mysql://root:"$mysqlrootpass"@/mysql "CREATE DATABASE `whoami`;CREATE USER '`whoami`'@'localhost' IDENTIFIED BY '`whoami`'; GRANT ALL ON `whoami`.* TO '`whoami`'@'localhost';"
+    sudo sql "$dburl" "CREATE DATABASE `whoami`;CREATE USER '`whoami`'@'localhost' IDENTIFIED BY '`whoami`'; GRANT ALL ON `whoami`.* TO '`whoami`'@'localhost';"
 }
 
 add_server_to_hosts() {
@@ -69,7 +123,8 @@ add_server_to_hosts() {
 }
 
 shellsplus() {
-    shells="bash sh csh ash dash tcsh zsh ksh ksh2020 ksh93 fish fizsh mksh posh rc sash yash nopathbash nopathcsh"
+    shells="bash sh csh ash dash tcsh zsh ksh ksh2020 ksh93 fish fizsh mksh"
+    shells="$shells posh rc sash yash nopathbash nopathcsh"
     shellsplus="parallel $shells"
     parallel -k echo ::: $shellsplus
 }
@@ -93,7 +148,9 @@ create_shell_logins() {
 	}
 	SSHPASS=`goodpasswd`
 	export SSHPASS
-	append-if-not-exists /etc/shells $(which $shell || which ${shell#"nopath"})
+	append-if-not-exists /etc/shells $(which $shell ||
+					       which ${shell#"nopath"})
+	sudo killall -u $shell
 	sudo deluser $shell && sudo mv /home/$shell /tmp/$shell.$RANDOM
 	sudo groupdel $shell
 	if echo $shell | grep -q parallel; then
@@ -112,7 +169,8 @@ create_shell_logins() {
 
     echo '# (Re-)create user'
     # Racecondition: if multiple adds a group it will the same group ID
-    shellsplus | parallel --timeout 15 --retries 5 --tag -j1 del_add_user ||
+    shellsplus |
+	parallel --lb --halt soon,fail=1 --timeout 1000% --retries 5 --tag -j1 del_add_user ||
 	(echo Creation failed: $?; false)
 }
 
@@ -120,15 +178,19 @@ copy_ssh_keys() {
     make_sshkey() {
 	shell="$1"
 	echo Add server keys for lo and server &&
-	    ssh $shell@lo 'rm -f .ssh/id_rsa; ssh-keyscan lo >>.ssh/known_hosts; ssh-keyscan server >> .ssh/known_hosts' &&
+	    ssh $shell@lo 'rm -f .ssh/id_rsa; 
+	                   ssh-keyscan lo >>.ssh/known_hosts;
+                           ssh-keyscan server >> .ssh/known_hosts' &&
 	    echo Do ssh-keygen &&
 	    echo | ssh -t $shell@lo ssh-keygen -b 1024 -f .ssh/id_rsa &&
 	    echo Do ssh $shell@lo 'cat .ssh/id_rsa.pub >> .ssh/authorized_keys' &&
 	    ssh $shell@lo 'cat .ssh/id_rsa.pub | tee -a .ssh/authorized_keys' |
-	      ssh parallel@lo 'cat >> .ssh/authorized_keys' &&
+		ssh parallel@lo 'cat >> .ssh/authorized_keys' &&
 	    ssh $shell@lo "echo ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAxg+dh+BL1myqo6A+pHcQCKVV5v7cewdkN8xGtMDqm7xGgh+e5A44W7qKViIx641d6yoWb012XzDz2MKetG4Kosoma8Z/qkS27p6901RmI3ap2JFffzBESlpQtW1FyxQPlMyBfrd4ONy8xW6R/gEkjC3yOcXCQO2494/c46ouXs5gtE84Cusr3RsopR4bI7oloH1GQZ4vsHSFSakq8IwcujaSh1mmribMONLb2TjDpmE7tAY+yfOVWHPQ2J+EU1//Go60EZfSRKOu81oxW7SJ2uGgWfWcs2T1lRtT8Yh+TwVTz8UFV75kVtpZ10x5baN+ncsSpaBK+8sFLfoMvA9yQQ'==' tange@freebsd71.tange.dk >> .ssh/authorized_keys" &&
 	    echo Do env_parallel --install &&
-	    ssh $shell@lo 'mkdir -p .parallel; touch .parallel/will-cite; env_parallel --install' &&
+	    ssh $shell@lo 'mkdir -p .parallel;
+                           touch .parallel/will-cite;
+                           env_parallel --install' &&
 	    echo OK make_sshkey $shell &&
 	    echo >&2 &&
 	    echo OK make_sshkey $shell >&2 &&
@@ -155,9 +217,11 @@ copy_ssh_keys() {
     export -f ssh_copy_id
 
     echo '# copy id from any X to any Y'
-    parallel -u --bar -j3 --timeout 3 --retries 10 --tag ssh_copy_id {1}@lo {2}@lo ::: $(shellsplus) ::: $(shellsplus)
+    parallel -u --bar -j3 --timeout 3 --retries 10 --tag \
+	     ssh_copy_id {1}@lo {2}@lo ::: $(shellsplus) ::: $(shellsplus)
     echo '# Test the copying went well'
-    parallel    --bar -j2 --timeout 9 --retries 10 --tag ssh_a_to_b ::: $(shellsplus) ::: $(shellsplus)
+    parallel    --bar -j2 --timeout 9 --retries 10 --tag \
+		ssh_a_to_b ::: $(shellsplus) ::: $(shellsplus)
 
     echo '# change paths to no path'
     (
@@ -183,16 +247,21 @@ lsh_setup() {
     cd
     mkdir -p .lsh
     lsh-make-seed -o ".lsh/yarrow-seed-file"
-    lsh -c aes256-ctr --sloppy-host-authentication --capture-to ~/.lsh/host-acls lo echo Added host-auth
+    lsh -c aes256-ctr --sloppy-host-authentication \
-    lsh -c aes256-ctr --sloppy-host-authentication --capture-to ~/.lsh/host-acls localhost echo Added host-auth
+	--capture-to ~/.lsh/host-acls lo echo Added host-auth
+    lsh -c aes256-ctr --sloppy-host-authentication \
+	--capture-to ~/.lsh/host-acls localhost echo Added host-auth
     lsh-keygen | lsh-writekey -c none
-    lsh-export-key --openssh < ~/.lsh/identity.pub | lsh -c aes256-ctr lo 'cat >>.ssh/authorized_keys'
+    lsh-export-key --openssh < ~/.lsh/identity.pub |
-    lsh-export-key --openssh < ~/.lsh/identity.pub | ssh csh@lo 'cat >>.ssh/authorized_keys'
+	lsh -c aes256-ctr lo 'cat >>.ssh/authorized_keys'
+    lsh-export-key --openssh < ~/.lsh/identity.pub |
+	ssh csh@lo 'cat >>.ssh/authorized_keys'
 }
 
 add_freebsd() {
     echo "# Add public key to freebsd7.t"
-    ssh freebsd7.t cat .ssh/id_rsa.pub | ssh parallel@localhost 'cat >>.ssh/authorized_keys'
+    ssh freebsd7.t cat .ssh/id_rsa.pub |
+	ssh parallel@localhost 'cat >>.ssh/authorized_keys'
 
     echo Add:
     echo HostkeyAlgorithms +ssh-dss
@@ -204,7 +273,8 @@ add_freebsd() {
     echo KexAlgorithms diffie-hellman-group-exchange-sha256,ecdh-sha2-nistp256,ecdh-sha2-nistp384,ecdh-sha2-nistp521,curve25519-sha256@libssh.org,diffie-hellman-group1-sha1,diffie-hellman-group14-sha1,diffie-hellman-group-exchange-sha1
     echo
     echo If you get:
-    echo Unable to negotiate with server port 22: no matching cipher found. Their offer: aes128-cbc,3des-cbc,blowfish-cbc,cast128-cbc,arcfour,aes192-cbc,aes256-cbc,rijndael-cbc@lysator.liu.se
+    echo Unable to negotiate with server port 22: no matching cipher found.
+    echo Their offer: aes128-cbc,3des-cbc,blowfish-cbc,cast128-cbc,arcfour,aes192-cbc,aes256-cbc,rijndael-cbc@lysator.liu.se
     echo add this to .ssh/config
     echo   Ciphers +aes256-cbc
 }
@@ -247,7 +317,8 @@ bash_versions() {
      export -f make_one
      echo '# Building bash'
      (cd bash; git tag | grep -v -- '-.*-') |
-	 stdout parallel --lb --tag '/usr/local/bin/{} --version || make_one {}'
+	 stdout parallel --lb --tag \
+		'/usr/local/bin/{} --version || make_one {}'
      (cd bash; git tag | grep -v -- '-.*-') |
 	 parallel -k -v --tag '/usr/local/bin/{} --version'
     )
@@ -263,6 +334,7 @@ misc() {
 
 run() {
     install_packages &&
+    install_oracle_client &&
     setup_databases &&
     add_server_to_hosts &&
     create_shell_logins &&

testsuite/tests-to-run/parallel-local-1s.sh

@@ -214,7 +214,8 @@ par_compress_prg_fails() {
 	echo $?) | tail -n1
     }
     export -f doit
-    parallel --tag -k doit ::: '' --line-buffer ::: '' --tag ::: '' --files
+    stdout parallel --tag -k doit ::: '' --line-buffer ::: '' --tag ::: '' --files |
+	grep -v -- -dc
 }
 
 par_pxz_complains() {

testsuite/tests-to-run/parallel-macos.sh

@@ -0,0 +1,74 @@
+#!/bin/bash
+
+. `which env_parallel.bash`
+env_parallel --session
+
+par_many_var() {
+    gen500k() { seq -f %f 1000000000000000 1000000000050000 | head -c 131000; }
+    for a in `seq 6000`; do eval "export a$a=1" ; done
+    gen500k | stdout parallel --load 5 -Xkj1  'echo {} {} {} {} | wc' |
+	perl -pe 's/\d{10,}.\d+ //g'
+}
+
+par_many_var_func() {
+    gen500k() { seq -f %f 1000000000000000 1000000000050000 | head -c 131000; }
+    for a in `seq 5000`; do eval "export a$a=1" ; done
+    for a in `seq 5000`; do eval "a$a() { 1; }" ; done
+    for a in `seq 5000`; do eval export -f a$a ; done
+    gen500k | stdout parallel --load 21 -Xkj1  'echo {} {} {} {} | wc' |
+	perl -pe 's/\d{10,}.\d+ //g'
+}
+
+par_many_func() {
+    gen500k() { seq -f %f 1000000000000000 1000000000050000 | head -c 131000; }
+    for a in `seq 5000`; do eval "a$a() { 1; }" ; done
+    for a in `seq 5000`; do eval export -f a$a ; done
+    gen500k | stdout parallel --load 6 -Xkj1  'echo {} {} {} {} | wc' |
+	perl -pe 's/\d{10,}.\d+ //g'
+}
+
+par_big_func() {
+    gen500k() { seq -f %f 1000000000000000 1000000000050000 | head -c 131000; }
+    big=`seq 1000`
+    for a in `seq 50`; do eval "a$a() { '$big'; }" ; done
+    for a in `seq 50`; do eval export -f a$a ; done
+    gen500k | stdout parallel --load 3 -Xkj1  'echo {} {} {} {} | wc' |
+	perl -pe 's/\d{10,}.\d+ //g'
+}
+
+par_many_var_big_func() {
+    gen500k() { seq -f %f 1000000000000000 1000000000050000 | head -c 131000; }
+    big=`seq 1000`
+    for a in `seq 5000`; do eval "export a$a=1" ; done
+    for a in `seq 10`; do eval "a$a() { '$big'; }" ; done
+    for a in `seq 10`; do eval export -f a$a ; done
+    gen500k | stdout parallel --load 6 -Xkj1  'echo {} {} {} {} | wc' |
+	perl -pe 's/\d{10,}.\d+ //g'
+}
+
+par_big_func_name() {
+    gen500k() { seq -f %f 1000000000000000 1000000000050000 | head -c 131000; }
+    big=`perl -e print\"x\"x10000`
+    for a in `seq 10`; do eval "export a$big$a=1" ; done
+    gen500k | stdout parallel --load 7 -Xkj1  'echo {} {} {} {} | wc' |
+	perl -pe 's/\d{10,}.\d+ //g'
+}
+
+par_big_var_func_name() {
+    gen500k() { seq -f %f 1000000000000000 1000000000050000 | head -c 131000; }
+    big=`perl -e print\"x\"x10000`
+    for a in `seq 10`; do eval "export a$big$a=1" ; done
+    for a in `seq 10`; do eval "a$big$a() { 1; }" ; done
+    for a in `seq 10`; do eval export -f a$big$a ; done
+    gen500k | stdout parallel --load 5 -Xkj1  'echo {} {} {} {} | wc' |
+	perl -pe 's/\d{10,}.\d+ //g'
+}
+
+scp /usr/local/bin/parallel macosx.p:bin/
+
+export -f $(compgen -A function | grep par_)
+#compgen -A function |
+compgen -A function |
+    grep par_ |
+    LC_ALL=C sort |
+    env_parallel --timeout 3000% --tag -k -S macosx.p

testsuite/vagrant/setup.sh

@@ -31,26 +31,31 @@ export -f destroy_one
 
 
 server_list() {
-    cat <<SSHOK
+    grep -v '#' <<SSHOK
-generic/arch.98
+#generic/arch.98
-generic/centos6.6
+hfm4/centos4.4
-generic/centos7.7
+hfm4/centos5.5
-generic/centos8.8
+#generic/centos6.6
-generic/debian8.18
+#generic/centos7.7
-generic/debian9.19
+#generic/centos8.8
-generic/debian10.30
+MarcinOrlowski/debian4-i386.14
-generic/devuan3.43
+twolfman/debian6-lamp-drush.16
-generic/freebsd11.71
+puphpet/debian75-x64.17
-generic/freebsd12.72
+#generic/debian8.18
-generic/gentoo.99
+#generic/debian9.19
-generic/netbsd9.89
+#generic/debian10.30
-generic/oracle7.127
+#generic/devuan3.43
-generic/rhel6.106
+#generic/freebsd11.71
-generic/rhel7.107
+#generic/freebsd12.72
-generic/rhel8.108
+#generic/gentoo.99
-generic/ubuntu1604.216
+#generic/netbsd9.89
-generic/ubuntu1804.218
+#generic/oracle7.127
-generic/ubuntu2004.220
+#generic/rhel6.106
+#generic/rhel7.107
+#generic/rhel8.108
+#generic/ubuntu1604.216
+#generic/ubuntu1804.218
+#generic/ubuntu2004.220
 SSHOK
 
     # Ignore for now

testsuite/wanted-results/parallel-macos

@@ -0,0 +1,62 @@
+par_big_func	       1    2592   62208
+par_big_func	       1    2592   62208
+par_big_func	       1    2592   62208
+par_big_func	       1    2592   62208
+par_big_func	       1    2592   62208
+par_big_func	       1    2592   62208
+par_big_func	       1    2592   62208
+par_big_func	       1    2592   62208
+par_big_func	       1    1100   26340
+par_big_func_name	       1    2432   58368
+par_big_func_name	       1    2432   58368
+par_big_func_name	       1    2432   58368
+par_big_func_name	       1    2432   58368
+par_big_func_name	       1    2432   58368
+par_big_func_name	       1    2432   58368
+par_big_func_name	       1    2432   58368
+par_big_func_name	       1    2432   58368
+par_big_func_name	       1    2380   57060
+par_big_var_func_name	       1    2428   58272
+par_big_var_func_name	       1    2428   58272
+par_big_var_func_name	       1    2428   58272
+par_big_var_func_name	       1    2428   58272
+par_big_var_func_name	       1    2428   58272
+par_big_var_func_name	       1    2428   58272
+par_big_var_func_name	       1    2428   58272
+par_big_var_func_name	       1    2428   58272
+par_big_var_func_name	       1    2412   57828
+par_many_func	       1    2532   60768
+par_many_func	       1    2532   60768
+par_many_func	       1    2532   60768
+par_many_func	       1    2532   60768
+par_many_func	       1    2532   60768
+par_many_func	       1    2532   60768
+par_many_func	       1    2532   60768
+par_many_func	       1    2532   60768
+par_many_func	       1    1580   37860
+par_many_var	       1    2868   68832
+par_many_var	       1    2868   68832
+par_many_var	       1    2868   68832
+par_many_var	       1    2868   68832
+par_many_var	       1    2868   68832
+par_many_var	       1    2868   68832
+par_many_var	       1    2868   68832
+par_many_var	       1    1760   42180
+par_many_var_big_func	       1    2576   61824
+par_many_var_big_func	       1    2576   61824
+par_many_var_big_func	       1    2576   61824
+par_many_var_big_func	       1    2576   61824
+par_many_var_big_func	       1    2576   61824
+par_many_var_big_func	       1    2576   61824
+par_many_var_big_func	       1    2576   61824
+par_many_var_big_func	       1    2576   61824
+par_many_var_big_func	       1    1228   29412
+par_many_var_func	       1    2532   60768
+par_many_var_func	       1    2532   60768
+par_many_var_func	       1    2532   60768
+par_many_var_func	       1    2532   60768
+par_many_var_func	       1    2532   60768
+par_many_var_func	       1    2532   60768
+par_many_var_func	       1    2532   60768
+par_many_var_func	       1    2532   60768
+par_many_var_func	       1    1580   37860