From 72dd439f2046747869b40f0493bf0a3a4704bedc Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Tue, 11 Aug 2020 22:55:31 +0200 Subject: [PATCH] parallel: Fix https://lists.gnu.org/archive/html/parallel/2020-08/msg00003.html --- src/parallel | 95 ++++++++++----- src/parallel_alternatives.pod | 79 ++++++++++++ testsuite/REQUIREMENTS | 126 +++++++++++++++----- testsuite/tests-to-run/parallel-local-1s.sh | 3 +- testsuite/tests-to-run/parallel-macos.sh | 74 ++++++++++++ testsuite/vagrant/setup.sh | 45 +++---- testsuite/wanted-results/parallel-macos | 62 ++++++++++ 7 files changed, 404 insertions(+), 80 deletions(-) create mode 100644 testsuite/tests-to-run/parallel-macos.sh create mode 100644 testsuite/wanted-results/parallel-macos diff --git a/src/parallel b/src/parallel index e0a5dc8e..bcce58a3 100755 --- a/src/parallel +++ b/src/parallel @@ -6249,36 +6249,48 @@ sub memfree_recompute() { # Buffers: 19876 kB # Cached: 431192 kB # SwapCached: 0 kB - "linux" => - q[ print 1024 * qx{ ]. - q[ awk '/^((Swap)?Cached|MemFree|Buffers):/ ]. - q[ { sum += \$2} END { print sum }' ]. - q[ /proc/meminfo } ], + "linux" => ( + q{ + print 1024 * qx{ + awk '/^((Swap)?Cached|MemFree|Buffers):/ + { sum += \$2} END { print sum }' + /proc/meminfo } + }), # Android uses same code as GNU/Linux - "android" => - q[ print 1024 * qx{ ]. - q[ awk '/^((Swap)?Cached|MemFree|Buffers):/ ]. - q[ { sum += \$2} END { print sum }' ]. - q[ /proc/meminfo } ], - + "android" => ( + q{ + print 1024 * qx{ + awk '/^((Swap)?Cached|MemFree|Buffers):/ + { sum += \$2} END { print sum }' + /proc/meminfo } + }), # $ vmstat 1 1 - # procs memory page faults cpu - # r b w avm free re at pi po fr de sr in sy cs us sy id - # 1 0 0 242793 389737 5 1 0 0 0 0 0 107 978 60 1 1 99 - "hpux" => - q[ print (((reverse `vmstat 1 1`)[0] ]. - q[ =~ /(?:\d+\D+){4}(\d+)/)[0]*1024) ], + # procs memory page faults cpu + # r b w avm free re at pi po fr de sr in sy cs us sy id + # 1 0 0 242793 389737 5 1 0 0 0 0 0 107 978 60 1 1 99 + "hpux" => ( + q{ + print (((reverse `vmstat 1 1`)[0] + =~ /(?:\d+\D+){4}(\d+)/)[0]*1024) + }), # $ vmstat 1 2 # kthr memory page disk faults cpu # r b w swap free re mf pi po fr de sr s3 s4 -- -- in sy cs us sy id # 0 0 0 6496720 5170320 68 260 8 2 1 0 0 -0 3 0 0 309 1371 255 1 2 97 - # 0 0 0 6434088 5072656 7 15 8 0 0 0 0 0 261 0 0 1889 1899 3222 0 8 92 + # 0 0 0 6434088 5072656 7 15 8 0 0 0 0 0 261 0 0 1889 1899 3222 0 8 92 # # The second free value is correct - "solaris" => - q[ print (((reverse `vmstat 1 2`)[0] ]. - q[ =~ /(?:\d+\D+){4}(\d+)/)[0]*1024) ], - "freebsd" => q{ + "solaris" => ( + q{ + print (((reverse `vmstat 1 2`)[0] + =~ /(?:\d+\D+){4}(\d+)/)[0]*1024) + }), + # hw.pagesize: 4096 + # vm.stats.vm.v_cache_count: 0 + # vm.stats.vm.v_inactive_count: 79574 + # vm.stats.vm.v_free_count: 4507 + "freebsd" => ( + q{ for(qx{/sbin/sysctl -a}) { if (/^([^:]+):\s+(.+)\s*$/s) { $sysctl->{$1} = $2; @@ -6288,7 +6300,7 @@ sub memfree_recompute() { ($sysctl->{"vm.stats.vm.v_cache_count"} + $sysctl->{"vm.stats.vm.v_inactive_count"} + $sysctl->{"vm.stats.vm.v_free_count"}); - }, + }), # Mach Virtual Memory Statistics: (page size of 4096 bytes) # Pages free: 198061. # Pages active: 159701. @@ -6302,12 +6314,13 @@ sub memfree_recompute() { # Pageins: 1798068. # Pageouts: 257. # Object cache: 6603 hits of 1713223 lookups (0% hit rate) - 'darwin' => - q[ $vm = `vm_stat`; - print (($vm =~ /page size of (\d+)/)[0] * - (($vm =~ /Pages free:\s+(\d+)/)[0] + - ($vm =~ /Pages inactive:\s+(\d+)/)[0])); - ], + 'darwin' => ( + q{ + $vm = `vm_stat`; + print (($vm =~ /page size of (\d+)/)[0] * + (($vm =~ /Pages free:\s+(\d+)/)[0] + + ($vm =~ /Pages inactive:\s+(\d+)/)[0])); + }), ); my $perlscript = ""; # Make a perl script that detects the OS ($^O) and runs @@ -6315,8 +6328,7 @@ sub memfree_recompute() { for my $os (keys %script_of) { $perlscript .= 'if($^O eq "'.$os.'") { '.$script_of{$os}.'}'; } - $perlscript =~ s/[\t\n ]+/ /g; - $script = "perl -e " . ::Q($perlscript); + $script = "perl -e " . ::Q(::spacefree(1,$perlscript)); } return $script; } @@ -10469,6 +10481,7 @@ sub slot($) { { my $already_spread; + my $env_size; sub populate($) { # Add arguments from arg_queue until the number of arguments or @@ -10486,7 +10499,25 @@ sub slot($) { my $next_arg; my $max_len = $Global::minimal_command_line_length || Limits::Command::max_length(); - + if($^O eq "darwin") { + # On Mac bash 3.2.48(1)-release the length also depends on + # number of functions, variables and the size of those. + # 21 is found by trial and error and seems to be a safe value. + # test_with() { + # gen500k() { + # seq -f %f 1000000000000000 1000000000050000 | + # head -c 131000; + # } + # for a in `seq 5100`; do eval "export a$a=1" ; done; + # for a in `seq 5100`; do eval "a$a() { 1; }" ; done; + # for a in `seq 5100`; do eval export -f a$a ; done; + # gen500k | stdout parallel -Xj1 'echo {} {} {} {} | wc' | + # perl -pe 's/\d{10,} //g' + # } + $env_size ||= (keys %ENV)*21 + + length(join'',keys %ENV) + length(join'',%ENV); + $max_len -= $env_size; + } if($opt::cat or $opt::fifo) { # Get the empty arg added by --pipepart (if any) $Global::JobQueue->{'commandlinequeue'}->{'arg_queue'}->get(); diff --git a/src/parallel_alternatives.pod b/src/parallel_alternatives.pod index 605498e7..75e5277f 100644 --- a/src/parallel_alternatives.pod +++ b/src/parallel_alternatives.pod @@ -2616,6 +2616,85 @@ https://pypi.org/project/papply/ (Last checked: 2020-04) =head2 Todo +test_many_var() { +gen500k() { seq -f %f 1000000000000000 1000000000050000 | head -c 131000; } +for a in `seq 11000`; do eval "export a$a=1" ; done +gen500k | stdout parallel --timeout 5 -Xj1 'echo {} {} {} {} | wc' | perl -pe 's/\d{3,5} //g' +} + +test_many_var_func() { +gen500k() { seq -f %f 1000000000000000 1000000000050000 | head -c 131000; } +for a in `seq 5100`; do eval "export a$a=1" ; done +for a in `seq 5100`; do eval "a$a() { 1; }" ; done +for a in `seq 5100`; do eval export -f a$a ; done +gen500k | stdout parallel --timeout 21 -Xj1 'echo {} {} {} {} | wc' | perl -pe 's/\d{3,5} //g' +} + +test_many_var_func() { +gen500k() { seq -f %f 1000000000000000 1000000000050000 | head -c 131000; } +for a in `seq 8000`; do eval "a$a() { 1; }" ; done +for a in `seq 8000`; do eval export -f a$a ; done +gen500k | stdout parallel --timeout 6 -Xj1 'echo {} {} {} {} | wc' | perl -pe 's/\d{3,5} //g' +} + +test_big_func() { +gen500k() { seq -f %f 1000000000000000 1000000000050000 | head -c 131000; } +big=`seq 1000` +for a in `seq 50`; do eval "a$a() { '$big'; }" ; done +for a in `seq 50`; do eval export -f a$a ; done +gen500k | stdout parallel --timeout 4 -Xj1 'echo {} {} {} {} | wc' | perl -pe 's/\d{3,5} //g' +} + +test_many_var_big_func() { +gen500k() { seq -f %f 1000000000000000 1000000000050000 | head -c 131000; } +big=`seq 1000` +for a in `seq 5100`; do eval "export a$a=1" ; done +for a in `seq 20`; do eval "a$a() { '$big'; }" ; done +for a in `seq 20`; do eval export -f a$a ; done +gen500k | stdout parallel --timeout 6 -Xj1 'echo {} {} {} {} | wc' | perl -pe 's/\d{3,5} //g' +} + +test_big_func_name() { +gen500k() { seq -f %f 1000000000000000 1000000000050000 | head -c 131000; } +big=`perl -e print\"x\"x10000` +for a in `seq 20`; do eval "export a$big$a=1" ; done +gen500k | stdout parallel --timeout 8 -Xj1 'echo {} {} {} {} | wc' | perl -pe 's/\d{3,5} //g' +} + +test_big_var_func_name() { +gen500k() { seq -f %f 1000000000000000 1000000000050000 | head -c 131000; } +big=`perl -e print\"x\"x10000` +for a in `seq 2`; do eval "export a$big$a=1" ; done +for a in `seq 2`; do eval "a$big$a() { '$big'; }" ; done +for a in `seq 2`; do eval export -f a$big$a ; done +gen500k | stdout parallel --timeout 1000 -Xj1 'echo {} {} {} {} | wc' | perl -pe 's/\d{3,5} //g' +} + + + +tange@macosx:~$ for a in `seq 100`; do eval export a$a=fffffffffffffffffffffffff ; donetange@macosx:~$ seq 50000 | stdout parallel -Xj1 'echo {} {} | wc' | perl -pe 's/\d{3,5} //g' +tange@macosx:~$ for a in `seq 100`; do eval export a$a=fffffffffffffffffffffffff ; donetange@macosx:~$ seq 50000 | stdout parallel -Xj1 'echo {} {} | wc' | perl -pe 's/\d{3,5} //g' +tange@macosx:~$ for a in `seq 100`; do eval export -f a$a ; done + + +seq 100000 | stdout parallel -Xj1 'echo {} {} | wc' +export a=`seq 10000` +seq 100000 | stdout parallel -Xj1 'echo {} {} | wc' + + + + my $already_spread; + my $env_size; + + if($^O eq "darwin") { + $env_size ||= 500+length(join'',%ENV); + $max_len -= $env_size; + } + + +PASH: Light-touch Data-Parallel Shell Processing +https://arxiv.org/pdf/2007.09436.pdf + https://gitlab.com/netikras/bthread https://github.com/JeiKeiLim/simple_distribute_job diff --git a/testsuite/REQUIREMENTS b/testsuite/REQUIREMENTS index e0c4e16d..b72c611a 100644 --- a/testsuite/REQUIREMENTS +++ b/testsuite/REQUIREMENTS @@ -6,26 +6,33 @@ install_packages() { INSTALL=${INSTALL:-"sudo aptitude -y install"} # The testsuite depends on this: - test_pkgs="imagemagick expect autossh sshpass jq libpod-simple-perl pod2pdf gawk" - test_pkgs="$test_pkgs lua5.3 clisp php-cli nodejs mono-csharp-shell libevent-dev" - test_pkgs="$test_pkgs tcl libtext-csv-perl xterm libc6-i386 libcrypt1:i386" + test_pkgs="imagemagick expect autossh sshpass jq libpod-simple-perl" + test_pkgs="$test_pkgs pod2pdf gawk lua5.3 clisp php-cli nodejs" + test_pkgs="$test_pkgs mono-csharp-shell libevent-dev tcl libtext-csv-perl" + test_pkgs="$test_pkgs xterm libc6-i386 libcrypt1:i386" + test_pkgs="$test_pkgs libtest-nowarnings-perl" # DEBIAN package - packaging_pkgs="dpkg-dev build-essential debhelper osc cvs automake-1.15 python3-m2crypto" + packaging_pkgs="dpkg-dev build-essential debhelper osc cvs automake-1.15" + packaging_pkgs="$packaging_pkgs python3-m2crypto alien" # SHEBANG TOOLS shebang_pkgs="gnuplot octave ruby r-base-core" # SQL TOOLS - sql_pkgs="libdbd-pg-perl libdbd-sqlite3-perl libdbd-csv-perl libdbd-mysql-perl rlwrap" + sql_pkgs="libdbd-pg-perl libdbd-sqlite3-perl libdbd-csv-perl" + sql_pkgs="$sql_pkgs libdbd-mysql-perl rlwrap" # Compression - compression_pkgs="zstd clzip liblz4-tool lzop pigz pixz gzip plzip pbzip2 lzma xz-utils lzip bzip2 lbzip2 lrzip" + compression_pkgs="zstd clzip liblz4-tool lzop pigz pixz gzip plzip pbzip2" + compression_pkgs="$compression_pkgs lzma xz-utils lzip bzip2 lbzip2 lrzip" compression_pkgs_missing="pxz" # Shells # (csh = bsd-csh that is broken) - shell_pkgs="ash dash fdclone fish fizsh ksh ksh93 mksh posh rc rush sash tcsh yash zsh" + shell_pkgs="ash dash fdclone fish fizsh ksh ksh93 mksh posh rc rush sash" + shell_pkgs="$shell_pkgs tcsh yash zsh" # Databases database_pkgs="postgresql mysql-server sqlite" - sudo dpkg --add-architecture i386 - if $INSTALL $test_pkgs $packaging_pkgs $shebang_pkgs $sql_pkgs $compression_pkgs $shell_pkgs $database_pkgs; then + sudo dpkg --add-architecture i386; sudo apt update + if $INSTALL $test_pkgs $packaging_pkgs $shebang_pkgs $sql_pkgs \ + $compression_pkgs $shell_pkgs $database_pkgs; then # OK true else @@ -35,6 +42,53 @@ install_packages() { sudo apt update & } +install_oracle_client() { + # https://salsa.debian.org/perl-team/modules/packages/libdbd-oracle-perl/blob/master/debian/README.Debian + # sql oracle:// && return 0 + (cd /tmp + + get_rpm_install_deb() { + glob="$1" + url="$oracleurl/$2" + testglob="$3" + if [ ! -e `echo $glob*rpm` ] ; then + wget $url + fi + if [ ! -e `echo $glob*deb` ] ; then + echo Convert `echo $glob*rpm` to deb + fakeroot alien `echo $glob*rpm` + fi + if [ ! -e `echo $testglob` ] ; then + echo Install `echo $glob*deb` + sudo dpkg -i `echo $glob*deb` + fi + } + + oracleurl=https://download.oracle.com/otn_software/linux/instantclient/19600 + client=oracle-instantclient19.6 + ver=19.6.0.0.0-1.x86_64.rpm + get_rpm_install_deb 'oracle-instantclient*devel' $client-devel-$ver /usr/share/doc/oracle-instantclient*-devel/copyright + get_rpm_install_deb 'oracle-instantclient*basic' $client-basic-$ver /usr/share/oracle/*/client*/doc/BASIC_README + get_rpm_install_deb 'oracle-instantclient*sqlplus' $client-sqlplus-$ver /usr/lib/oracle/*/client*/bin/sqlplus + ) + + echo Add this to .bashrc: + ORACLE_HOME=`echo /usr/lib/oracle/*/client*` + echo ORACLE_HOME=$ORACLE_HOME + if grep -q ORACLE_HOME=/ ~/.bashrc; then + perl -i.old -pe "s:ORACLE_HOME=/.*:ORACLE_HOME=$ORACLE_HOME:" ~/.bashrc + else + echo ORACLE_HOME=$ORACLE_HOME >> ~/.bashrc + echo 'PATH=$PATH:$ORACLE_HOME/bin' >> ~/.bashrc + echo 'export ORACLE_HOME' >> ~/.bashrc + echo 'export ORACLE_SID=XE' >> ~/.bashrc + fi + perl -MCPAN -e 'install DBD::Oracle' + # TODO set up vagrant oracle server + # TODO set up default passwd + # test it works: sql oracle:// +} + setup_databases() { # DATABASES echo '# Create PostgreSQL' @@ -46,15 +100,15 @@ setup_databases() { sudo su - postgres -c "sql pg:/// \"ALTER USER \\\"`whoami`\\\" WITH PASSWORD '`whoami`';\"" mysqlrootpass=${mysqlrootpass:-b+Ydjq4ejT4E} - + dburl=mysql://root:"$mysqlrootpass"@/mysql echo '# Create MySQL' sudo su mysql mysqladmin create `whoami` # Default Debian way of getting "root" access sudo mysql --defaults-file=/etc/mysql/debian.cnf mysql <<< "ALTER USER 'root'@'localhost' IDENTIFIED BY '$mysqlrootpass';" # Drop database and user if needed - sudo sql mysql://root:"$mysqlrootpass"@/mysql "DROP DATABASE `whoami`;DROP USER '`whoami`'@'localhost';" - sudo sql mysql://root:"$mysqlrootpass"@/mysql "CREATE DATABASE `whoami`;CREATE USER '`whoami`'@'localhost' IDENTIFIED BY '`whoami`'; GRANT ALL ON `whoami`.* TO '`whoami`'@'localhost';" + sudo sql "$dburl" "DROP DATABASE `whoami`;DROP USER '`whoami`'@'localhost';" + sudo sql "$dburl" "CREATE DATABASE `whoami`;CREATE USER '`whoami`'@'localhost' IDENTIFIED BY '`whoami`'; GRANT ALL ON `whoami`.* TO '`whoami`'@'localhost';" } add_server_to_hosts() { @@ -69,7 +123,8 @@ add_server_to_hosts() { } shellsplus() { - shells="bash sh csh ash dash tcsh zsh ksh ksh2020 ksh93 fish fizsh mksh posh rc sash yash nopathbash nopathcsh" + shells="bash sh csh ash dash tcsh zsh ksh ksh2020 ksh93 fish fizsh mksh" + shells="$shells posh rc sash yash nopathbash nopathcsh" shellsplus="parallel $shells" parallel -k echo ::: $shellsplus } @@ -93,7 +148,9 @@ create_shell_logins() { } SSHPASS=`goodpasswd` export SSHPASS - append-if-not-exists /etc/shells $(which $shell || which ${shell#"nopath"}) + append-if-not-exists /etc/shells $(which $shell || + which ${shell#"nopath"}) + sudo killall -u $shell sudo deluser $shell && sudo mv /home/$shell /tmp/$shell.$RANDOM sudo groupdel $shell if echo $shell | grep -q parallel; then @@ -112,7 +169,8 @@ create_shell_logins() { echo '# (Re-)create user' # Racecondition: if multiple adds a group it will the same group ID - shellsplus | parallel --timeout 15 --retries 5 --tag -j1 del_add_user || + shellsplus | + parallel --lb --halt soon,fail=1 --timeout 1000% --retries 5 --tag -j1 del_add_user || (echo Creation failed: $?; false) } @@ -120,15 +178,19 @@ copy_ssh_keys() { make_sshkey() { shell="$1" echo Add server keys for lo and server && - ssh $shell@lo 'rm -f .ssh/id_rsa; ssh-keyscan lo >>.ssh/known_hosts; ssh-keyscan server >> .ssh/known_hosts' && + ssh $shell@lo 'rm -f .ssh/id_rsa; + ssh-keyscan lo >>.ssh/known_hosts; + ssh-keyscan server >> .ssh/known_hosts' && echo Do ssh-keygen && echo | ssh -t $shell@lo ssh-keygen -b 1024 -f .ssh/id_rsa && echo Do ssh $shell@lo 'cat .ssh/id_rsa.pub >> .ssh/authorized_keys' && ssh $shell@lo 'cat .ssh/id_rsa.pub | tee -a .ssh/authorized_keys' | - ssh parallel@lo 'cat >> .ssh/authorized_keys' && + ssh parallel@lo 'cat >> .ssh/authorized_keys' && ssh $shell@lo "echo ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAxg+dh+BL1myqo6A+pHcQCKVV5v7cewdkN8xGtMDqm7xGgh+e5A44W7qKViIx641d6yoWb012XzDz2MKetG4Kosoma8Z/qkS27p6901RmI3ap2JFffzBESlpQtW1FyxQPlMyBfrd4ONy8xW6R/gEkjC3yOcXCQO2494/c46ouXs5gtE84Cusr3RsopR4bI7oloH1GQZ4vsHSFSakq8IwcujaSh1mmribMONLb2TjDpmE7tAY+yfOVWHPQ2J+EU1//Go60EZfSRKOu81oxW7SJ2uGgWfWcs2T1lRtT8Yh+TwVTz8UFV75kVtpZ10x5baN+ncsSpaBK+8sFLfoMvA9yQQ'==' tange@freebsd71.tange.dk >> .ssh/authorized_keys" && echo Do env_parallel --install && - ssh $shell@lo 'mkdir -p .parallel; touch .parallel/will-cite; env_parallel --install' && + ssh $shell@lo 'mkdir -p .parallel; + touch .parallel/will-cite; + env_parallel --install' && echo OK make_sshkey $shell && echo >&2 && echo OK make_sshkey $shell >&2 && @@ -155,9 +217,11 @@ copy_ssh_keys() { export -f ssh_copy_id echo '# copy id from any X to any Y' - parallel -u --bar -j3 --timeout 3 --retries 10 --tag ssh_copy_id {1}@lo {2}@lo ::: $(shellsplus) ::: $(shellsplus) + parallel -u --bar -j3 --timeout 3 --retries 10 --tag \ + ssh_copy_id {1}@lo {2}@lo ::: $(shellsplus) ::: $(shellsplus) echo '# Test the copying went well' - parallel --bar -j2 --timeout 9 --retries 10 --tag ssh_a_to_b ::: $(shellsplus) ::: $(shellsplus) + parallel --bar -j2 --timeout 9 --retries 10 --tag \ + ssh_a_to_b ::: $(shellsplus) ::: $(shellsplus) echo '# change paths to no path' ( @@ -183,16 +247,21 @@ lsh_setup() { cd mkdir -p .lsh lsh-make-seed -o ".lsh/yarrow-seed-file" - lsh -c aes256-ctr --sloppy-host-authentication --capture-to ~/.lsh/host-acls lo echo Added host-auth - lsh -c aes256-ctr --sloppy-host-authentication --capture-to ~/.lsh/host-acls localhost echo Added host-auth + lsh -c aes256-ctr --sloppy-host-authentication \ + --capture-to ~/.lsh/host-acls lo echo Added host-auth + lsh -c aes256-ctr --sloppy-host-authentication \ + --capture-to ~/.lsh/host-acls localhost echo Added host-auth lsh-keygen | lsh-writekey -c none - lsh-export-key --openssh < ~/.lsh/identity.pub | lsh -c aes256-ctr lo 'cat >>.ssh/authorized_keys' - lsh-export-key --openssh < ~/.lsh/identity.pub | ssh csh@lo 'cat >>.ssh/authorized_keys' + lsh-export-key --openssh < ~/.lsh/identity.pub | + lsh -c aes256-ctr lo 'cat >>.ssh/authorized_keys' + lsh-export-key --openssh < ~/.lsh/identity.pub | + ssh csh@lo 'cat >>.ssh/authorized_keys' } add_freebsd() { echo "# Add public key to freebsd7.t" - ssh freebsd7.t cat .ssh/id_rsa.pub | ssh parallel@localhost 'cat >>.ssh/authorized_keys' + ssh freebsd7.t cat .ssh/id_rsa.pub | + ssh parallel@localhost 'cat >>.ssh/authorized_keys' echo Add: echo HostkeyAlgorithms +ssh-dss @@ -204,7 +273,8 @@ add_freebsd() { echo KexAlgorithms diffie-hellman-group-exchange-sha256,ecdh-sha2-nistp256,ecdh-sha2-nistp384,ecdh-sha2-nistp521,curve25519-sha256@libssh.org,diffie-hellman-group1-sha1,diffie-hellman-group14-sha1,diffie-hellman-group-exchange-sha1 echo echo If you get: - echo Unable to negotiate with server port 22: no matching cipher found. Their offer: aes128-cbc,3des-cbc,blowfish-cbc,cast128-cbc,arcfour,aes192-cbc,aes256-cbc,rijndael-cbc@lysator.liu.se + echo Unable to negotiate with server port 22: no matching cipher found. + echo Their offer: aes128-cbc,3des-cbc,blowfish-cbc,cast128-cbc,arcfour,aes192-cbc,aes256-cbc,rijndael-cbc@lysator.liu.se echo add this to .ssh/config echo Ciphers +aes256-cbc } @@ -247,7 +317,8 @@ bash_versions() { export -f make_one echo '# Building bash' (cd bash; git tag | grep -v -- '-.*-') | - stdout parallel --lb --tag '/usr/local/bin/{} --version || make_one {}' + stdout parallel --lb --tag \ + '/usr/local/bin/{} --version || make_one {}' (cd bash; git tag | grep -v -- '-.*-') | parallel -k -v --tag '/usr/local/bin/{} --version' ) @@ -263,6 +334,7 @@ misc() { run() { install_packages && + install_oracle_client && setup_databases && add_server_to_hosts && create_shell_logins && diff --git a/testsuite/tests-to-run/parallel-local-1s.sh b/testsuite/tests-to-run/parallel-local-1s.sh index fcb8b857..3959360e 100644 --- a/testsuite/tests-to-run/parallel-local-1s.sh +++ b/testsuite/tests-to-run/parallel-local-1s.sh @@ -214,7 +214,8 @@ par_compress_prg_fails() { echo $?) | tail -n1 } export -f doit - parallel --tag -k doit ::: '' --line-buffer ::: '' --tag ::: '' --files + stdout parallel --tag -k doit ::: '' --line-buffer ::: '' --tag ::: '' --files | + grep -v -- -dc } par_pxz_complains() { diff --git a/testsuite/tests-to-run/parallel-macos.sh b/testsuite/tests-to-run/parallel-macos.sh new file mode 100644 index 00000000..4c40295d --- /dev/null +++ b/testsuite/tests-to-run/parallel-macos.sh @@ -0,0 +1,74 @@ +#!/bin/bash + +. `which env_parallel.bash` +env_parallel --session + +par_many_var() { + gen500k() { seq -f %f 1000000000000000 1000000000050000 | head -c 131000; } + for a in `seq 6000`; do eval "export a$a=1" ; done + gen500k | stdout parallel --load 5 -Xkj1 'echo {} {} {} {} | wc' | + perl -pe 's/\d{10,}.\d+ //g' +} + +par_many_var_func() { + gen500k() { seq -f %f 1000000000000000 1000000000050000 | head -c 131000; } + for a in `seq 5000`; do eval "export a$a=1" ; done + for a in `seq 5000`; do eval "a$a() { 1; }" ; done + for a in `seq 5000`; do eval export -f a$a ; done + gen500k | stdout parallel --load 21 -Xkj1 'echo {} {} {} {} | wc' | + perl -pe 's/\d{10,}.\d+ //g' +} + +par_many_func() { + gen500k() { seq -f %f 1000000000000000 1000000000050000 | head -c 131000; } + for a in `seq 5000`; do eval "a$a() { 1; }" ; done + for a in `seq 5000`; do eval export -f a$a ; done + gen500k | stdout parallel --load 6 -Xkj1 'echo {} {} {} {} | wc' | + perl -pe 's/\d{10,}.\d+ //g' +} + +par_big_func() { + gen500k() { seq -f %f 1000000000000000 1000000000050000 | head -c 131000; } + big=`seq 1000` + for a in `seq 50`; do eval "a$a() { '$big'; }" ; done + for a in `seq 50`; do eval export -f a$a ; done + gen500k | stdout parallel --load 3 -Xkj1 'echo {} {} {} {} | wc' | + perl -pe 's/\d{10,}.\d+ //g' +} + +par_many_var_big_func() { + gen500k() { seq -f %f 1000000000000000 1000000000050000 | head -c 131000; } + big=`seq 1000` + for a in `seq 5000`; do eval "export a$a=1" ; done + for a in `seq 10`; do eval "a$a() { '$big'; }" ; done + for a in `seq 10`; do eval export -f a$a ; done + gen500k | stdout parallel --load 6 -Xkj1 'echo {} {} {} {} | wc' | + perl -pe 's/\d{10,}.\d+ //g' +} + +par_big_func_name() { + gen500k() { seq -f %f 1000000000000000 1000000000050000 | head -c 131000; } + big=`perl -e print\"x\"x10000` + for a in `seq 10`; do eval "export a$big$a=1" ; done + gen500k | stdout parallel --load 7 -Xkj1 'echo {} {} {} {} | wc' | + perl -pe 's/\d{10,}.\d+ //g' +} + +par_big_var_func_name() { + gen500k() { seq -f %f 1000000000000000 1000000000050000 | head -c 131000; } + big=`perl -e print\"x\"x10000` + for a in `seq 10`; do eval "export a$big$a=1" ; done + for a in `seq 10`; do eval "a$big$a() { 1; }" ; done + for a in `seq 10`; do eval export -f a$big$a ; done + gen500k | stdout parallel --load 5 -Xkj1 'echo {} {} {} {} | wc' | + perl -pe 's/\d{10,}.\d+ //g' +} + +scp /usr/local/bin/parallel macosx.p:bin/ + +export -f $(compgen -A function | grep par_) +#compgen -A function | +compgen -A function | + grep par_ | + LC_ALL=C sort | + env_parallel --timeout 3000% --tag -k -S macosx.p diff --git a/testsuite/vagrant/setup.sh b/testsuite/vagrant/setup.sh index f098139e..95a888b9 100644 --- a/testsuite/vagrant/setup.sh +++ b/testsuite/vagrant/setup.sh @@ -31,26 +31,31 @@ export -f destroy_one server_list() { - cat <