From 36a8238a20164114b95e69ddcd87a7dedf12688d Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Fri, 21 Sep 2018 00:15:14 +0200 Subject: [PATCH] parallel: bug #54647: parset ignores empty lines. parallel: minix is working again. parallel: bug #54128: command too long when exporting big env testsuite: polarhome parallelized. --- doc/release_new_version | 7 +- src/env_parallel.bash | 2 +- src/env_parallel.csh | 2 +- src/env_parallel.sh | 8 +- src/env_parallel.tcsh | 2 +- src/parallel | 82 ++- src/parallel_alternatives.pod | 115 ++-- testsuite/Makefile | 2 +- testsuite/tests-to-run/niceload01.sh | 4 +- testsuite/tests-to-run/niceload03.sh | 2 +- testsuite/tests-to-run/parallel-freebsd.sh | 2 +- testsuite/tests-to-run/parallel-local-0.3s.sh | 13 +- testsuite/tests-to-run/parallel-local-ssh1.sh | 13 +- testsuite/tests-to-run/parallel-local-ssh3.sh | 2 +- testsuite/tests-to-run/parallel-local-ssh4.sh | 2 +- testsuite/tests-to-run/parallel-local-ssh7.sh | 8 +- testsuite/tests-to-run/parallel-local-ssh9.sh | 22 + testsuite/tests-to-run/parallel-local1.sh | 2 +- testsuite/tests-to-run/parallel-local114.sh | 2 +- testsuite/tests-to-run/parallel-local13.sh | 2 +- testsuite/tests-to-run/parallel-local150.sh | 2 +- testsuite/tests-to-run/parallel-local164.sh | 2 +- testsuite/tests-to-run/parallel-local19.sh | 2 +- testsuite/tests-to-run/parallel-local2.sh | 2 +- testsuite/tests-to-run/parallel-local22.sh | 2 +- testsuite/tests-to-run/parallel-local23.sh | 2 +- testsuite/tests-to-run/parallel-local4.sh | 2 +- testsuite/tests-to-run/parallel-local5.sh | 2 +- testsuite/tests-to-run/parallel-local7.sh | 2 +- testsuite/tests-to-run/parallel-local9.sh | 2 +- testsuite/tests-to-run/parallel-polarhome.sh | 121 ++-- testsuite/tests-to-run/parallel-remote1.sh | 2 +- testsuite/tests-to-run/sem01.sh | 4 +- testsuite/tests-to-run/test30.sh | 2 +- testsuite/tests-to-run/test45.sh | 2 +- testsuite/tests-to-run/test60.sh | 2 +- testsuite/tests-to-run/test61.sh | 2 +- testsuite/wanted-results/parallel-local-0.3s | 86 +-- testsuite/wanted-results/parallel-local-10s | 2 +- testsuite/wanted-results/parallel-local-ssh1 | 24 - testsuite/wanted-results/parallel-local-ssh9 | 17 + testsuite/wanted-results/parallel-local19 | 4 +- testsuite/wanted-results/parallel-local23 | 4 +- testsuite/wanted-results/parallel-polarhome | 645 +++++++++++++----- 44 files changed, 792 insertions(+), 439 deletions(-) diff --git a/doc/release_new_version b/doc/release_new_version index 9c6747c0..2853e4b6 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -206,7 +206,7 @@ to:parallel@gnu.org, bug-parallel@gnu.org stable-bcc: Jesse Alama -Subject: GNU Parallel 20180922 ('Myanmar/Jacksonville/Chemnitz') released <<[stable]>> +Subject: GNU Parallel 20180922 ('Danske/Korea/India/Myanmar/Jacksonville/Chemnitz') released <<[stable]>> GNU Parallel 20180922 ('') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/ @@ -219,7 +219,10 @@ Quote of the month: New in this release: -* +* Updated graph showing GNU Parallel's overhead per job for each version. https://www.gnu.org/software/parallel/process-time-j2-1700MHz-3000-1000.pdf + +http://carc.unm.edu/education--training/workshops-and-seminars.html + <> diff --git a/src/env_parallel.bash b/src/env_parallel.bash index 45291ba9..b3ce0810 100755 --- a/src/env_parallel.bash +++ b/src/env_parallel.bash @@ -7,7 +7,7 @@ # after which 'env_parallel' works # # -# Copyright (C) 2016,2017,2018 +# Copyright (C) 2016-2018 # Ole Tange and Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or modify diff --git a/src/env_parallel.csh b/src/env_parallel.csh index e6171244..5ea95d10 100755 --- a/src/env_parallel.csh +++ b/src/env_parallel.csh @@ -7,7 +7,7 @@ # after which 'env_parallel' works # # -# Copyright (C) 2016,2017,2018 +# Copyright (C) 2016-2018 # Ole Tange and Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or modify diff --git a/src/env_parallel.sh b/src/env_parallel.sh index 7e95e9e7..ce7d0211 100755 --- a/src/env_parallel.sh +++ b/src/env_parallel.sh @@ -7,7 +7,7 @@ # after which 'env_parallel' works # # -# Copyright (C) 2016,2017,2018 +# Copyright (C) 2016-2018 # Ole Tange and Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or modify @@ -181,12 +181,12 @@ env_parallel() { # type returns: # ll is an alias for ls -l (in ash) # bash is a tracked alias for /bin/bash - # true is a shell builtin + # true is a shell builtin (in bash) # myfunc is a function (in bash) # myfunc is a shell function (in zsh) - # which is /usr/bin/which + # which is /usr/bin/which (in sh, bash) # which is hashed (/usr/bin/which) - # aliased to `alias | /usr/bin/which --tty-only --read-alias --show-dot --show-tilde' + # gi is aliased to `grep -i' (in bash) # Return 0 if found, 1 otherwise LANG=C type "$@" | perl -pe '$exit += (s/ is an alias for .*// || diff --git a/src/env_parallel.tcsh b/src/env_parallel.tcsh index 7b4672ee..9ea73521 100755 --- a/src/env_parallel.tcsh +++ b/src/env_parallel.tcsh @@ -7,7 +7,7 @@ # after which 'env_parallel' works # # -# Copyright (C) 2016,2017,2018 +# Copyright (C) 2016-2018 # Ole Tange and Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or modify diff --git a/src/parallel b/src/parallel index b0f56414..a764dbfe 100755 --- a/src/parallel +++ b/src/parallel @@ -1,7 +1,6 @@ #!/usr/bin/env perl -# Copyright (C) 2007,2008,2009,2010,2011,2012,2013,2014,2015,2016, -# 2017,2018 Ole Tange and Free Software Foundation, Inc. +# Copyright (C) 2007-2018 Ole Tange and Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -1555,7 +1554,7 @@ sub check_invalid_option_combinations { sub init_globals { # Defaults: - $Global::version = 20180824; + $Global::version = 20180916; $Global::progname = 'parallel'; $Global::infinity = 2**31; $Global::debug = 0; @@ -3257,19 +3256,8 @@ sub get_job_with_sshlogin { return undef; } } - - my $clean_command = $job->replaced(); - if($clean_command =~ /^\s*$/) { - # Do not run empty lines - if(not $Global::JobQueue->empty()) { - return get_job_with_sshlogin($sshlogin); - } else { - return undef; - } - } $job->set_sshlogin($sshlogin); - if($opt::retries and $clean_command and - $job->failed_here()) { + if($opt::retries and $job->failed_here()) { # This command with these args failed for this sshlogin my ($no_of_failed_sshlogins,$min_failures) = $job->min_failed(); # Only look at the Global::host that have > 0 jobslots @@ -4366,17 +4354,17 @@ sub die_bug { sub version { # Returns: N/A - print join("\n", - "GNU $Global::progname $Global::version", - "Copyright (C) 2007,2008,2009,2010,2011,2012,2013,2014,2015,2016,2017,2018", - "Ole Tange and Free Software Foundation, Inc.", - "License GPLv3+: GNU GPL version 3 or later ", - "This is free software: you are free to change and redistribute it.", - "GNU $Global::progname comes with no warranty.", - "", - "Web site: http://www.gnu.org/software/${Global::progname}\n", - "When using programs that use GNU Parallel to process data for publication", - "please cite as described in 'parallel --citation'.\n", + print join + ("\n", + "GNU $Global::progname $Global::version", + "Copyright (C) 2007-2018 Ole Tange and Free Software Foundation, Inc.", + "License GPLv3+: GNU GPL version 3 or later ", + "This is free software: you are free to change and redistribute it.", + "GNU $Global::progname comes with no warranty.", + "", + "Web site: http://www.gnu.org/software/${Global::progname}\n", + "When using programs that use GNU Parallel to process data for publication", + "please cite as described in 'parallel --citation'.\n", ); } @@ -4489,8 +4477,8 @@ sub embed { } print "#!$Global::shell -# Copyright (C) 2007,2008,2009,2010,2011,2012,2013,2014,2015,2016, -# 2017,2018 $user, Ole Tange and Free Software Foundation, Inc. +# Copyright (C) 2007-2018 $user, Ole Tange and Free Software +# Foundation, Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -5022,9 +5010,10 @@ sub which { # Filter for SysV-style `ps` my $sysv = q( ps -ef | perl -ane '1..1 and /^(.*)CO?MM?A?N?D/ and $s=length $1;). q(s/^.{$s}//; print "@F[1,2] $_"' ); - # Crazy msys: ' is not accepted on the cmd line, but " are treated as ' - my $msys = q( ps -ef | perl -ane "1..1 and /^(.*)CO?MM?A?N?D/ and $s=length $1;). - q(s/^.{$s}//; print qq{@F[1,2] $_}" ); + # Minix uses cols 2,3 and can have newlines in the command + # so lines not having numbers in cols 2,3 must be ignored + my $minix = q( ps -ef | perl -ane '1..1 and /^(.*)CO?MM?A?N?D/ and $s=length $1;). + q(s/^.{$s}// and $F[2]>0 and $F[3]>0 and print "@F[2,3] $_"' ); # BSD-style `ps` my $bsd = q(ps -o pid,ppid,command -ax); %pid_parentpid_cmd = @@ -5040,7 +5029,8 @@ sub which { 'hpux' => $sysv, 'linux' => $sysv, 'mirbsd' => $bsd, - 'msys' => $msys, + 'minix' => $minix, + 'msys' => $sysv, 'MSWin32' => $sysv, 'netbsd' => $bsd, 'nto' => $sysv, @@ -5639,7 +5629,8 @@ sub swap_activity { # Should we update the swap_activity file? my $update_swap_activity_file = 0; if(-r $self->{'swap_activity_file'}) { - open(my $swap_fh, "<", $self->{'swap_activity_file'}) || ::die_bug("swap_activity_file-r"); + open(my $swap_fh, "<", $self->{'swap_activity_file'}) || + ::die_bug("swap_activity_file-r"); my $swap_out = <$swap_fh>; close $swap_fh; if($swap_out =~ /^(\d+)$/) { @@ -6170,6 +6161,8 @@ sub compute_number_of_processes { return 0; } else { $job = $Global::JobQueue->get(); + # Replacement must happen here due to seq() + $job and $job->replaced(); push(@jobs, $job); return 1; } @@ -9954,6 +9947,10 @@ sub len { $self->{'replacecount'}{$replstring}; } } + if(defined $Global::parallel_env) { + # If we are using --env, add the prefix for that, too. + $len += length $Global::parallel_env; + } if($Global::quoting) { # Pessimistic length if -q is set # Worse than worst case: every char needs to be quoted with \ @@ -9964,8 +9961,12 @@ sub len { # Worse than worst case: every char needs to be quoted with \ twice $len *= 4; } - # If we are using --env, add the prefix for that, too. - $len += 0; + if(@opt::sshlogin) { + # Pessimistic length if remote + # Worst case is BASE64 encoding 3 bytes -> 4 bytes + $len = int($len*4/3); + } + return $len; } @@ -10513,14 +10514,9 @@ sub get { "(e.g. 'cat')."); ::wait_and_exit(255); } - } else { - if($cmd_line->number_of_args() == 0) { - # We did not get more args - maybe at EOF string? - return undef; - } elsif($cmd_line->replaced() eq "") { - # Empty command - get the next instead - return $self->get(); - } + } elsif($cmd_line->number_of_args() == 0) { + # We did not get more args - maybe at EOF string? + return undef; } $self->set_seq($self->seq()+1); return $cmd_line; diff --git a/src/parallel_alternatives.pod b/src/parallel_alternatives.pod index 086827c1..39125ec7 100644 --- a/src/parallel_alternatives.pod +++ b/src/parallel_alternatives.pod @@ -624,17 +624,20 @@ special characters: echo "The Cure" > My\ brother\'s\ 12\"\ records ls | \ - map 'echo -n `gzip < "%" | wc -c`; echo -n '*100/'; wc -c < "%"' | bc + map 'echo -n `gzip < "%" | wc -c`; echo -n '*100/'; wc -c < "%"' | + bc It works with GNU B: ls | \ - parallel 'echo -n `gzip < {} | wc -c`; echo -n '*100/'; wc -c < {}' | bc + parallel \ + 'echo -n `gzip < {} | wc -c`; echo -n '*100/'; wc -c < {}' | bc And you can even get the file name prepended: ls | \ - parallel --tag '(echo -n `gzip < {} | wc -c`'*100/'; wc -c < {}) | bc' + parallel --tag \ + '(echo -n `gzip < {} | wc -c`'*100/'; wc -c < {}) | bc' B has no support for grouping. So this gives the wrong results without any warnings: @@ -655,23 +658,22 @@ completely for certain jobs (and may even be considered less readable): parallel echo -n {} ::: 1 2 3 - map -p 4 'echo -n % 2>&1 | sed -e "s/^/$$:/"' 1 2 3 | sort | cut -f2- -d: + map -p 4 'echo -n % 2>&1 | sed -e "s/^/$$:/"' 1 2 3 | \ + sort | cut -f2- -d: Bs replacement strings (% %D %B %E) can be simulated in GNU B by putting this in B<~/.parallel/config>: --rpl '%' - --rpl '%D $_=::shell_quote(::dirname($_));' + --rpl '%D $_=Q(::dirname($_));' --rpl '%B s:.*/::;s:\.[^/.]+$::;' --rpl '%E s:.*\.::' -B cannot handle bundled options: B - B does not have an argument separator on the command line, but uses the first argument as command. This makes quoting harder which again may affect readability. Compare: - map -p 2 perl\\\ -ne\\\ \\\'/^\\\\S+\\\\s+\\\\S+\\\$/\\\ and\\\ print\\\ \\\$ARGV,\\\"\\\\n\\\"\\\' * + map -p 2 'perl -ne '"'"'/^\S+\s+\S+$/ and print $ARGV,"\n"'"'" * parallel -q perl -ne '/^\S+\s+\S+$/ and print $ARGV,"\n"' ::: * @@ -680,16 +682,11 @@ context replace: parallel --xargs echo 'BEGIN{'{}'}END' ::: 1 2 3 -B does not set exit value according to whether one of the jobs -failed: - - parallel false ::: 1 || echo Job failed - - map false 1 || echo Never run + map "echo 'BEGIN{'%'}END'" 1 2 3 B requires Perl v5.10.0 making it harder to use on old systems. -B has no way of using % in the command (GNU Parallel has -I to +B has no way of using % in the command (GNU B has -I to specify another replacement string than B<{}>). By design B is option incompatible with B, it does not @@ -710,18 +707,21 @@ B only works on files and the only way to specify files is using a quoted glob string (such as \*.jpg). It is not possible to list the files manually. -As replacement strings it uses FULLPATH DIRNAME BASENAME EXT RELDIR RELPATH +As replacement strings it uses FULLPATH DIRNAME BASENAME EXT RELDIR +RELPATH -These can be simulated using GNU B by putting this in B<~/.parallel/config>: +These can be simulated using GNU B by putting this in +B<~/.parallel/config>: - --rpl 'FULLPATH $_=::shell_quote($_);chomp($_=qx{readlink -f $_});' - --rpl 'DIRNAME $_=::shell_quote(::dirname($_));chomp($_=qx{readlink -f $_});' + --rpl 'FULLPATH $_=Q($_);chomp($_=qx{readlink -f $_});' + --rpl 'DIRNAME $_=Q(::dirname($_));chomp($_=qx{readlink -f $_});' --rpl 'BASENAME s:.*/::;s:\.[^/.]+$::;' --rpl 'EXT s:.*\.::' - --rpl 'RELDIR $_=::shell_quote($_);chomp(($_,$c)=qx{readlink -f $_;pwd});s:\Q$c/\E::;$_=::dirname($_);' - --rpl 'RELPATH $_=::shell_quote($_);chomp(($_,$c)=qx{readlink -f $_;pwd});s:\Q$c/\E::;' + --rpl 'RELDIR $_=Q($_);chomp(($_,$c)=qx{readlink -f $_;pwd});s:\Q$c/\E::;$_=::dirname($_);' + --rpl 'RELPATH $_=Q($_);chomp(($_,$c)=qx{readlink -f $_;pwd});s:\Q$c/\E::;' -B deals badly with filenames containing " and newline, and it fails for output larger than 200k: +B deals badly with filenames containing " and newline, and it +fails for output larger than 200k: ladon '*' -- seq 36000 | wc @@ -755,17 +755,23 @@ B can run multiple jobs in parallel. Just like B output from B jobs running in parallel mix together by default. B can buffer into files (placed in -/run/shm), but these are not cleaned up - not even if B dies -unexpectedly. If the total output is big (in the order of RAM+swap) -it can cause the system to run out of memory. +/run/shm), but these are not cleaned up if B dies +unexpectedly (e.g. by Ctrl-C). If the total output is big (in the +order of RAM+swap) it can cause the system to slow to a crawl and +eventually run out of memory. B gives no error if the command is unknown, and like B -redirection requires wrapping with B. +redirection and composed commands require wrapping with B. + +Input lines can at most be 4096 bytes. You can at most have 16 {}'s in +the command template. More than that either crashes the program or +simple does not execute the command. + +B has no equivalent for B<--pipe>, or B<--sshlogin>. B makes it possible to set resource limits on the running jobs. This can be emulated by GNU B using B's B: - jobflow -limits=mem=100M,cpu=3,fsize=20M,nofiles=300 myjob parallel 'ulimit -v 102400 -t 3 -f 204800 -n 300 myjob' @@ -832,7 +838,7 @@ setting SSHPASS and by using B<--ssh "sshpass ssh">. To make the emulation easier, make a simple alias: - alias par_emul="parallel -j0 --ssh 'sshpass ssh' --nonall --tag --linebuffer" + alias par_emul="parallel -j0 --ssh 'sshpass ssh' --nonall --tag --lb" If you want to supply a password run: @@ -850,8 +856,10 @@ If the above is set up you can then do: orgalorg -o frontend1 -o frontend2 -p -C top -bid 1 par_emul -S frontend1 -S frontend2 top -bid 1 - orgalorg -o frontend1 -o frontend2 -p -er /tmp -n 'md5sum /tmp/bigfile' -S bigfile - par_emul -S frontend1 -S frontend2 --basefile bigfile --workdir /tmp md5sum /tmp/bigfile + orgalorg -o frontend1 -o frontend2 -p -er /tmp -n \ + 'md5sum /tmp/bigfile' -S bigfile + par_emul -S frontend1 -S frontend2 --basefile bigfile --workdir /tmp \ + md5sum /tmp/bigfile B has a progress indicator for the transferring of a file. GNU B does not. @@ -865,8 +873,6 @@ Rust parallel focuses on speed. It is almost as fast as B. It implements a few features from GNU B, but lacks many functions. All these fail: - # Show what would be executed - parallel --dry-run echo ::: a # Read arguments from file parallel -a file echo # Changing the delimiter @@ -874,8 +880,6 @@ functions. All these fail: These do something different from GNU B - # Read more arguments at a time -n - parallel -n 2 echo ::: 1 a 2 b # -q to protect quoted $ and space parallel -q perl -e '$a=shift; print "$a"x10000000' ::: a b c # Generation of combination of inputs @@ -885,7 +889,7 @@ These do something different from GNU B # --pipe seq 100000 | parallel --pipe wc # linked arguments - parallel echo ::: S M L :::+ small medium large ::: R G B :::+ red green blue + parallel echo ::: S M L :::+ sml med lrg ::: R G B :::+ red grn blu # Run different shell dialects zsh -c 'parallel echo \={} ::: zsh && true' csh -c 'parallel echo \$\{\} ::: shell && true' @@ -903,6 +907,14 @@ will fail. A malicious user can setup the right permissions and symlink the output file to one of the user's files and next time the user uses Rust parallel it will overwrite this file. + attacker$ mkdir /tmp/parallel + attacker$ chmod a+rwX /tmp/parallel + # Symlink to the file the attacker wants to zero out + attacker$ ln -s ~victim/.important-file /tmp/parallel/stderr_1 + victim$ seq 1000 | parallel echo + # This file is now overwritten with stderr from 'echo' + victim$ cat ~victim/.important-file + If /tmp/parallel runs full during the run, Rust parallel does not report this, but finishes with success - thereby risking data loss. @@ -1286,7 +1298,8 @@ E.g. With GNU B this can be emulated by: - echo foosuffix | parallel --plus 'p={%suffix}; echo ${p}_new_suffix' + echo foosuffix | + parallel --plus 'p={%suffix}; echo ${p}_new_suffix' Opposite B GNU B works fine if the input contains double space, ' and ": @@ -1501,9 +1514,9 @@ https://github.com/soveran/map =head2 Todo -Url for map, spread +Url for spread -machma. Requires Go >= 1.7. +https://github.com/fd0/machma Requires Go >= 1.7. https://github.com/k-bx/par requires Haskell to work. This limits the number of platforms this can work on. @@ -1518,6 +1531,8 @@ https://github.com/Julian/Verge https://github.com/amattn/paral +https://github.com/mmstick/concurr + pyargs @@ -1559,7 +1574,8 @@ This test stresses whether output mixes. chmod +x mycommand # Run 30 jobs in parallel - seq 30 | $paralleltool ./mycommand > >(tr -s abcdef) 2> >(tr -s abcdef >&2) + seq 30 | + $paralleltool ./mycommand > >(tr -s abcdef) 2> >(tr -s abcdef >&2) # 'a c e' and 'b d f' should always stay together # and there should only be a single line per job @@ -1673,6 +1689,18 @@ Some tools can only insert the argument once. echo bar | $paralleltool echo {} foo {} +=head2 INPUTSIZE: Length of input should not be limited + +Some tools limit the length of the input lines artificially with no good +reason. GNU B does not: + + perl -e 'print "foo."."x"x100_000_000' | parallel echo {.} + +GNU B limits the command to run to 128 KB due to execve(1): + + perl -e 'print "x"x131_000' | parallel echo {} | wc + + =head2 NUMWORDS: Speed depends on number of words Some tools become very slow if output lines have many words. @@ -1705,10 +1733,10 @@ If you pay 10000 EUR you should feel free to use GNU Parallel without citing. Copyright (C) 2007-10-18 Ole Tange, http://ole.tange.dk -Copyright (C) 2008,2009,2010 Ole Tange, http://ole.tange.dk +Copyright (C) 2008-2010 Ole Tange, http://ole.tange.dk -Copyright (C) 2010,2011,2012,2013,2014,2015,2016,2017,2018 Ole Tange, -http://ole.tange.dk and Free Software Foundation, Inc. +Copyright (C) 2010-2018 Ole Tange, http://ole.tange.dk and Free +Software Foundation, Inc. Parts of the manual concerning B compatibility is inspired by the manual of B from GNU findutils 4.4.2. @@ -1716,8 +1744,7 @@ the manual of B from GNU findutils 4.4.2. =head1 LICENSE -Copyright (C) 2007,2008,2009,2010,2011,2012,2013,2014,2015,2016,2017,2018 -Free Software Foundation, Inc. +Copyright (C) 2007-2018 Free Software Foundation, Inc. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by diff --git a/testsuite/Makefile b/testsuite/Makefile index e38f9c5a..0cf7bdcb 100644 --- a/testsuite/Makefile +++ b/testsuite/Makefile @@ -7,7 +7,7 @@ testsuite: 3 make stopvm 1: ../src/parallel tests-to-run/* wanted-results/* prereqlocal startdb prereqremote - TRIES=1 time bash Start.sh '' 'mem|polarhome|tutorial' || true + TRIES=1 time bash Start.sh '' '100s|mem|polarhome|tutorial' || true touch ~/.parallel/will-cite make stopvm diff --git a/testsuite/tests-to-run/niceload01.sh b/testsuite/tests-to-run/niceload01.sh index 3f5af59a..ffc00e8d 100755 --- a/testsuite/tests-to-run/niceload01.sh +++ b/testsuite/tests-to-run/niceload01.sh @@ -2,7 +2,7 @@ echo '### Test niceload -q' niceload -q perl -e '$a = "works";$b="This $a\n"; print($b);' -echo +echo freepl >/dev/null freepl >/dev/null & @@ -21,7 +21,7 @@ freepl >/dev/null & # niceload -q -l 5 perl -e '$a=join"",<>; while(1){push @a,$a}' & -cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj0 -k --joblog /tmp/jl-`basename $0` -L1 +cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj0 -k --joblog /tmp/jl-`basename $0` -L1 -r echo '### --rm and --runmem' niceload -H --rm 1g free -g | perl -ane '/Mem:/ and print $F[5],"\n"' | grep '[1-9]' >/dev/null && echo OK--rm niceload -H --runmem 1g free -g | perl -ane '/Mem:/ and print $F[5],"\n"' | grep '[1-9]' >/dev/null && echo OK--runmem diff --git a/testsuite/tests-to-run/niceload03.sh b/testsuite/tests-to-run/niceload03.sh index f406d26e..76eec46e 100755 --- a/testsuite/tests-to-run/niceload03.sh +++ b/testsuite/tests-to-run/niceload03.sh @@ -10,7 +10,7 @@ int() { } export -f int -cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj0 -k --joblog /tmp/jl-`basename $0` -L1 +cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj0 -k --joblog /tmp/jl-`basename $0` -L1 -r # The seq 30000000 should take > 1 cpu sec to run. echo '### --soft -f and test if child is actually suspended and thus takes longer' niceload --soft -f 0.5 'seq 30000000 | nice wc;echo This should finish last' & diff --git a/testsuite/tests-to-run/parallel-freebsd.sh b/testsuite/tests-to-run/parallel-freebsd.sh index ea1d3619..5c2ecbd0 100644 --- a/testsuite/tests-to-run/parallel-freebsd.sh +++ b/testsuite/tests-to-run/parallel-freebsd.sh @@ -79,7 +79,7 @@ par_load() { echo '### Test --load (must give 1=true)' parallel -j0 -N0 --timeout 5 --nice 10 'bzip2 < /dev/zero >/dev/null' ::: 1 2 3 4 5 6 & parallel --argsep ,, --joblog - -N0 parallel --load 100% echo ::: 1 ,, 1 | - parallel --colsep '\t' --header : echo '{=4 $_=$_>5=}' + parallel -k --colsep '\t' --header : echo '{=4 $_=$_>5=}' } par_env_parallel() { diff --git a/testsuite/tests-to-run/parallel-local-0.3s.sh b/testsuite/tests-to-run/parallel-local-0.3s.sh index 8e267256..21249f8d 100644 --- a/testsuite/tests-to-run/parallel-local-0.3s.sh +++ b/testsuite/tests-to-run/parallel-local-0.3s.sh @@ -27,7 +27,7 @@ export -f stdsort # Test amount of parallelization # parallel --shuf --jl /tmp/myjl -j1 'export JOBS={1};'bash tests-to-run/parallel-local-0.3s.sh ::: {1..16} ::: {1..5} -cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj13 -k --joblog /tmp/jl-`basename $0` -L1 +cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj13 -k --joblog /tmp/jl-`basename $0` -L1 -r echo '### Test bug #45619: "--halt" erroneous error exit code (should give 0)'; seq 10 | parallel --halt now,fail=1 true; echo $? @@ -837,6 +837,17 @@ par_perlexpr_with_newline() { rm *"Dad's \"famous\" 1' pizza" } +par_empty_command() { + echo 'bug #54647: parset ignores empty lines' + # really due to this. Should give an empty line due to -v: + parallel -v :::: <(echo) + . `which env_parallel.bash` + parset a,b,c :::: <(echo echo A; echo; echo echo C) + echo Empty: $b + parset a,b,c :::: <(echo echo A; echo echo B; echo echo C) + echo B: $b +} + export -f $(compgen -A function | grep par_) compgen -A function | grep par_ | sort | parallel -j6 --tag -k --joblog +/tmp/jl-`basename $0` '{} 2>&1' diff --git a/testsuite/tests-to-run/parallel-local-ssh1.sh b/testsuite/tests-to-run/parallel-local-ssh1.sh index 30b21c5b..9027fd05 100644 --- a/testsuite/tests-to-run/parallel-local-ssh1.sh +++ b/testsuite/tests-to-run/parallel-local-ssh1.sh @@ -6,7 +6,7 @@ mkdir tmp cd tmp unset run_test -cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | stdout parallel -vj300% -k --joblog /tmp/jl-`basename $0` -L1 +cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | stdout parallel -vj300% -k --joblog /tmp/jl-`basename $0` -L1 -r echo '### Test --load remote' ssh parallel@lo 'seq 10 | parallel --nice 19 --timeout 15 -j0 -qN0 perl -e while\(1\)\{\ \}' & sleep 1; @@ -20,17 +20,6 @@ echo '### Stop if all hosts are filtered and there are no hosts left to run on' echo '### Can csh propagate a variable containing \n'; export A=$(seq 3); parallel -S csh@lo --env A bash -c \''echo "$A"'\' ::: dummy -echo '### bug #41805: Idea: propagate --env for parallel --number-of-cores' - echo '** test_zsh' - FOO=test_zsh parallel --env FOO,HOME -S zsh@lo -N0 env ::: "" |sort|egrep 'FOO|^HOME' - echo '** test_zsh_filter' - FOO=test_zsh_filter parallel --filter-hosts --env FOO,HOME -S zsh@lo -N0 env ::: "" |sort|egrep 'FOO|^HOME' - echo '** test_csh' - FOO=test_csh parallel --env FOO,HOME -S csh@lo -N0 env ::: "" |sort|egrep 'FOO|^HOME' - echo '** test_csh_filter' - FOO=test_csh_filter parallel --filter-hosts --env FOO,HOME -S csh@lo -N0 env ::: "" |sort|egrep 'FOO|^HOME' - echo '** bug #41805 done' - echo '### Test bug #34241: --pipe should not spawn unneeded processes' seq 5 | ssh csh@lo parallel -k --block 5 --pipe -j10 cat\\\;echo Block_end diff --git a/testsuite/tests-to-run/parallel-local-ssh3.sh b/testsuite/tests-to-run/parallel-local-ssh3.sh index d3c0cfd1..205d987c 100644 --- a/testsuite/tests-to-run/parallel-local-ssh3.sh +++ b/testsuite/tests-to-run/parallel-local-ssh3.sh @@ -2,7 +2,7 @@ # SSH only allowed to localhost/lo -cat <<'EOF' | parallel -vj100% --retries 3 -k --joblog /tmp/jl-`basename $0` -L1 +cat <<'EOF' | parallel -vj100% --retries 3 -k --joblog /tmp/jl-`basename $0` -L1 -r echo '### trailing space in sshlogin' echo 'sshlogin trailing space' | parallel --sshlogin "ssh -l parallel localhost " echo diff --git a/testsuite/tests-to-run/parallel-local-ssh4.sh b/testsuite/tests-to-run/parallel-local-ssh4.sh index 910eef7c..7fce198b 100644 --- a/testsuite/tests-to-run/parallel-local-ssh4.sh +++ b/testsuite/tests-to-run/parallel-local-ssh4.sh @@ -5,7 +5,7 @@ unset run_once # SSH only allowed to localhost/lo # --retries if ssh dies -cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | parallel -vj4 --retries 2 -k --joblog /tmp/jl-`basename $0` -L1 +cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | parallel -vj4 --retries 2 -k --joblog /tmp/jl-`basename $0` -L1 -r echo '### --env _' fUbAr="OK FUBAR" parallel -S parallel@lo --env _ echo '$fUbAr $DEBEMAIL' ::: test fUbAr="OK FUBAR" parallel -S csh@lo --env _ echo '$fUbAr $DEBEMAIL' ::: test diff --git a/testsuite/tests-to-run/parallel-local-ssh7.sh b/testsuite/tests-to-run/parallel-local-ssh7.sh index 3364a112..692af765 100755 --- a/testsuite/tests-to-run/parallel-local-ssh7.sh +++ b/testsuite/tests-to-run/parallel-local-ssh7.sh @@ -1833,20 +1833,20 @@ par_mksh_environment_too_big() { echo 'bug #50815: env_parallel should warn if the environment is too big' . `which env_parallel.mksh`; - bigvar="$(perl -e 'print "x"x119000')" + bigvar="$(perl -e 'print "x"x110000')" env_parallel echo ::: OK_bigvar env_parallel -S lo echo ::: OK_bigvar_remote - bigvar="$(perl -e 'print "\""x119000')" + bigvar="$(perl -e 'print "\""x110000')" env_parallel echo ::: OK_bigvar_quote env_parallel -S lo echo ::: OK_bigvar_quote_remote bigvar=u - eval 'bigfunc() { a="'"$(perl -e 'print "x"x119000')"'"; };' + eval 'bigfunc() { a="'"$(perl -e 'print "x"x110000')"'"; };' env_parallel echo ::: OK_bigfunc env_parallel -S lo echo ::: OK_bigfunc_remote - eval 'bigfunc() { a="'"$(perl -e 'print "\""x119000')"'"; };' + eval 'bigfunc() { a="'"$(perl -e 'print "\""x110000')"'"; };' env_parallel echo ::: OK_bigfunc_quote env_parallel -S lo echo ::: OK_bigfunc_quote_remote bigfunc() { true; } diff --git a/testsuite/tests-to-run/parallel-local-ssh9.sh b/testsuite/tests-to-run/parallel-local-ssh9.sh index 5f6ac109..cd684aaf 100644 --- a/testsuite/tests-to-run/parallel-local-ssh9.sh +++ b/testsuite/tests-to-run/parallel-local-ssh9.sh @@ -131,6 +131,28 @@ _EOF ssh zsh@lo "$myscript" } +par_propagate_env() { + echo '### bug #41805: Idea: propagate --env for parallel --number-of-cores' + echo '** test_zsh' + FOO=test_zsh parallel --env FOO,HOME -S zsh@lo -N0 env ::: "" |sort|egrep 'FOO|^HOME' + echo '** test_zsh_filter' + FOO=test_zsh_filter parallel --filter-hosts --env FOO,HOME -S zsh@lo -N0 env ::: "" |sort|egrep 'FOO|^HOME' + echo '** test_csh' + FOO=test_csh parallel --env FOO,HOME -S csh@lo -N0 env ::: "" |sort|egrep 'FOO|^HOME' + echo '** test_csh_filter' + FOO=test_csh_filter parallel --filter-hosts --env FOO,HOME -S csh@lo -N0 env ::: "" |sort|egrep 'FOO|^HOME' + echo '** bug #41805 done' +} + +par_env_parallel_big_env() { + echo '### bug #54128: command too long when exporting big env' + . `which env_parallel.bash` + a=`rand | head -c 75000` + env_parallel -Slo echo should not ::: fail 2>&1 + a=`rand | head -c 76000` + env_parallel -Slo echo should not ::: fail 2>/dev/null || echo OK +} + export -f $(compgen -A function | grep par_) #compgen -A function | grep par_ | sort | parallel --delay $D -j$P --tag -k '{} 2>&1' #compgen -A function | grep par_ | sort | diff --git a/testsuite/tests-to-run/parallel-local1.sh b/testsuite/tests-to-run/parallel-local1.sh index 0787f1fb..295bbd50 100644 --- a/testsuite/tests-to-run/parallel-local1.sh +++ b/testsuite/tests-to-run/parallel-local1.sh @@ -1,6 +1,6 @@ #!/bin/bash -cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj0 -k --joblog /tmp/jl-`basename $0` -L1 +cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj0 -k --joblog /tmp/jl-`basename $0` -L1 -r echo "bug #43654: --bar with command not using {} - only last output line " COLUMNS=80 stdout parallel --bar true {.} ::: 1 | perl -pe 's/.*\r/\r/' diff --git a/testsuite/tests-to-run/parallel-local114.sh b/testsuite/tests-to-run/parallel-local114.sh index b1d610d6..7e948afe 100755 --- a/testsuite/tests-to-run/parallel-local114.sh +++ b/testsuite/tests-to-run/parallel-local114.sh @@ -1,6 +1,6 @@ #!/bin/bash -cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj0 -k --joblog /tmp/jl-`basename $0` -L1 +cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj0 -k --joblog /tmp/jl-`basename $0` -L1 -r echo "### bug #41482: --pipe --compress blocks at different -j/seq combinations" seq 1 | parallel -k -j2 --compress -N1 -L1 --pipe cat; echo echo 1-4 + 1-4 diff --git a/testsuite/tests-to-run/parallel-local13.sh b/testsuite/tests-to-run/parallel-local13.sh index c71d11f8..0fc41788 100644 --- a/testsuite/tests-to-run/parallel-local13.sh +++ b/testsuite/tests-to-run/parallel-local13.sh @@ -13,7 +13,7 @@ stdsort() { } export -f stdsort -cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj100 -k --joblog /tmp/jl-`basename $0` -L1 +cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj100 -k --joblog /tmp/jl-`basename $0` -L1 -r echo '### -0 -n3 echo < files0.xi' stdout xargs -0 -n3 echo < files0.xi stdout parallel -k -0 -n3 echo < files0.xi diff --git a/testsuite/tests-to-run/parallel-local150.sh b/testsuite/tests-to-run/parallel-local150.sh index c0bcadda..1ae904f5 100644 --- a/testsuite/tests-to-run/parallel-local150.sh +++ b/testsuite/tests-to-run/parallel-local150.sh @@ -7,7 +7,7 @@ median() { perl -e '@a=sort {$a<=>$b} <>;print $a[$#a/2]';} export -f median # -L1 will join lines ending in ' ' -cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | parallel -vj0 -k --joblog /tmp/jl-`basename $0` -L1 +cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | parallel -vj0 -k --joblog /tmp/jl-`basename $0` -L1 -r echo '### bug #41565: Print happens in blocks - not after each job complete' echo 'The timing here is important: a full second between each' perl -e 'for(1..30){print("$_\n");`sleep 1`}' | parallel -j3 'echo {#}' | timestamp -dd | perl -pe '$_=int($_+0.3)."\n"' | median diff --git a/testsuite/tests-to-run/parallel-local164.sh b/testsuite/tests-to-run/parallel-local164.sh index e3fe386c..ed4f3565 100644 --- a/testsuite/tests-to-run/parallel-local164.sh +++ b/testsuite/tests-to-run/parallel-local164.sh @@ -1,7 +1,7 @@ #!/bin/bash # -L1 will join lines ending in ' ' -cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj0 -k --joblog /tmp/jl-`basename $0` -L1 +cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj0 -k --joblog /tmp/jl-`basename $0` -L1 -r echo '### bug #38354: -J profile_name should read from `pwd`/profile_name before ~/.parallel/profile_name' echo "echo echo from ./local_test_profile" > local_test_profile; parallel --profile local_test_profile echo ::: 1; diff --git a/testsuite/tests-to-run/parallel-local19.sh b/testsuite/tests-to-run/parallel-local19.sh index 832b9c2c..4da53dde 100755 --- a/testsuite/tests-to-run/parallel-local19.sh +++ b/testsuite/tests-to-run/parallel-local19.sh @@ -34,7 +34,7 @@ ls | parallel -kv rm -- {.}/abc-{.}-{} 2>&1 #test05.sh:find . -type d -print0 | perl -0 -pe 's:^./::' | parallel -0 -v rmdir -- {} 2>&1 \ # -L1 will join lines ending in ' ' -cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | parallel -vj0 -k -L1 --joblog /tmp/jl-`basename $0` +cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | parallel -vj0 -k -L1 --joblog /tmp/jl-`basename $0` -r echo '### Test compress' seq 5 | parallel -j2 --tag --compress 'seq {} | pv -q -L 10' diff --git a/testsuite/tests-to-run/parallel-local2.sh b/testsuite/tests-to-run/parallel-local2.sh index 81ba04fa..22e5956c 100644 --- a/testsuite/tests-to-run/parallel-local2.sh +++ b/testsuite/tests-to-run/parallel-local2.sh @@ -20,7 +20,7 @@ forceload () { # Force load avg > number of cpu cores forceload $(parallel --number-of-cores) -cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj0 -k --joblog /tmp/jl-`basename $0` -L1 +cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj0 -k --joblog /tmp/jl-`basename $0` -L1 -r echo "bug #38441: CPU usage goes to 100% if load is higher than --load at first job" /usr/bin/time -f %e parallel --load 100% true ::: a 2>&1 | perl -ne '$_ > 1 and print "More than 1 secs wall clock: OK\n"' diff --git a/testsuite/tests-to-run/parallel-local22.sh b/testsuite/tests-to-run/parallel-local22.sh index 3ad8f13b..a5c4a68b 100755 --- a/testsuite/tests-to-run/parallel-local22.sh +++ b/testsuite/tests-to-run/parallel-local22.sh @@ -1,6 +1,6 @@ #!/bin/bash -cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj0 -k --joblog /tmp/jl-`basename $0` -L1 +cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj0 -k --joblog /tmp/jl-`basename $0` -L1 -r echo '### Test of xargs -m command lines > 130k'; seq 1 60000 | parallel -m -j1 echo a{}b{}c | tee >(wc >/tmp/awc$$) >(sort | md5sum) >/tmp/a$$; wait; diff --git a/testsuite/tests-to-run/parallel-local23.sh b/testsuite/tests-to-run/parallel-local23.sh index 0e793e49..b51de27b 100755 --- a/testsuite/tests-to-run/parallel-local23.sh +++ b/testsuite/tests-to-run/parallel-local23.sh @@ -6,7 +6,7 @@ cp -a input-files/testdir2 tmp NICEPAR="nice nice parallel" export NICEPAR -cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj0 -k --joblog /tmp/jl-`basename $0` -L1 +cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj0 -k --joblog /tmp/jl-`basename $0` -L1 -r echo '### bug #42329: --line-buffer gives wrong output'; $NICEPAR --line-buffer --tag seq ::: 10000000 | wc -c; $NICEPAR --line-buffer seq ::: 10000000 | wc -c diff --git a/testsuite/tests-to-run/parallel-local4.sh b/testsuite/tests-to-run/parallel-local4.sh index 997af4cb..da931c74 100644 --- a/testsuite/tests-to-run/parallel-local4.sh +++ b/testsuite/tests-to-run/parallel-local4.sh @@ -1,6 +1,6 @@ #!/bin/bash -cat <<'EOF' | sed -e 's/;$/; /;' | stdout parallel -vj0 -k --joblog /tmp/jl-`basename $0` -L1 +cat <<'EOF' | sed -e 's/;$/; /;' | stdout parallel -vj0 -k --joblog /tmp/jl-`basename $0` -L1 -r echo '### -L -n with pipe' seq 14 | parallel --pipe -k -L 3 -n 2 'cat;echo 6 Ln line record' diff --git a/testsuite/tests-to-run/parallel-local5.sh b/testsuite/tests-to-run/parallel-local5.sh index 2d4be46c..04da4ecd 100644 --- a/testsuite/tests-to-run/parallel-local5.sh +++ b/testsuite/tests-to-run/parallel-local5.sh @@ -5,7 +5,7 @@ echo '### Test --pipe' seq 1 1000000 >/tmp/parallel-seq shuf --random-source=/tmp/parallel-seq /tmp/parallel-seq >/tmp/blocktest -cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | parallel -vj2 -k --joblog /tmp/jl-`basename $0` -L1 +cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | parallel -vj2 -k --joblog /tmp/jl-`basename $0` -L1 -r echo '### Test 200M records with too small block'; ( echo start; diff --git a/testsuite/tests-to-run/parallel-local7.sh b/testsuite/tests-to-run/parallel-local7.sh index 8c720933..0b1fd480 100755 --- a/testsuite/tests-to-run/parallel-local7.sh +++ b/testsuite/tests-to-run/parallel-local7.sh @@ -24,7 +24,7 @@ export -f par_tmux # echo '### bug #48841: --tmux(pane) --fg should start tmux in foreground' # stdout /usr/bin/time -f %e script -q -f -c /tmp/parallel-local7-script /dev/null | perl -ne '$_ >= 26 and $_ <= 45 and print "OK\n"' -cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj8 --delay 1 --timeout 100 --retries 1 -k --joblog /tmp/jl-`basename $0` -L1 +cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj8 --delay 1 --timeout 100 --retries 1 -k --joblog /tmp/jl-`basename $0` -L1 -r echo '### tmux-1.9' seq 0000 10 1000 | PARALLEL_TMUX=tmux-1.9 par_tmux diff --git a/testsuite/tests-to-run/parallel-local9.sh b/testsuite/tests-to-run/parallel-local9.sh index 59e3aeb6..bafb6149 100644 --- a/testsuite/tests-to-run/parallel-local9.sh +++ b/testsuite/tests-to-run/parallel-local9.sh @@ -7,7 +7,7 @@ export XAP NICEPAR="nice nice parallel" export NICEPAR -cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | stdout parallel -vj4 -k --joblog /tmp/jl-`basename $0` -L1 +cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | stdout parallel -vj4 -k --joblog /tmp/jl-`basename $0` -L1 -r echo 'bug #41412: --timeout + --delay causes deadlock'; seq 10 | parallel -j10 --timeout 1 --delay 0.3 echo; parallel -j3 --timeout 1 --delay 2 echo ::: 1 2 3; diff --git a/testsuite/tests-to-run/parallel-polarhome.sh b/testsuite/tests-to-run/parallel-polarhome.sh index 37faca39..96dde3de 100644 --- a/testsuite/tests-to-run/parallel-polarhome.sh +++ b/testsuite/tests-to-run/parallel-polarhome.sh @@ -8,14 +8,17 @@ env_parallel --session P_ALL="openstep qnx pidora alpha tru64 hpux-ia64 syllable raspbian solaris openindiana aix hpux debian-ppc suse solaris-x86 mandriva ubuntu scosysv unixware centos miros macosx redhat netbsd openbsd freebsd debian dragonfly vax ultrix minix irix hurd beaglebone cubieboard2" P="$P_ALL" -# 2018-04-22 MAXTIME=20 -MAXTIME=25 -RETRIES=4 -MAXPROC=150 +# tru64 takes 22s to run 4 parallels +MAXTIME=50 +RETRIES=2 +MAXPROC=${maxproc:-11} +MAXINNERPROC=${maxinnerproc:-3} -# select a running master (debian or ubuntu) -MASTER=$(parallel -j0 --halt now,success=1 ssh {} echo {} \ - ::: {suse,ubuntu,debian}.polarhome.com) +export PARALLEL_SSH="ssh -oLogLevel=quiet" + +# select a running master (suse, ubuntu, or debian) +MASTER=$(parallel -j0 --delay 0.1 --halt now,success=1 $PARALLEL_SSH {} echo {} \ + ::: {ubuntu,suse,debian}.polarhome.com) parallel -j0 --delay 0.1 --retries $RETRIES \ rsync -a /usr/local/bin/{parallel,env_parallel,env_parallel.*[^~],parcat} \ @@ -29,26 +32,37 @@ doit() { export RETRIES export MAXPROC export RET_TIME_K="-k --retries $RETRIES --timeout $MAXTIME" - - echo MAXTIME=$MAXTIME RETRIES=$RETRIES MAXPROC=$MAXPROC + + echo MAXTIME=$MAXTIME RETRIES=$RETRIES MAXPROC=$MAXPROC MAXINNERPROC=$MAXINNERPROC echo '### Filter out working servers' - POLAR="` bin/parallel -j0 $RET_TIME_K $PARALLEL_SSH {} echo {} ::: $P`" + # syllable often gives false positive + parallel --timeout $MAXTIME -j10 ssh syllable true ::: {1..10} 2>/dev/null >/dev/null & + POLAR_ALL="`bin/parallel -j0 -k --timeout 10 echo {} ::: $P`" + POLAR="`bin/parallel -j0 -k --timeout 10 $PARALLEL_SSH {} echo {} ::: $P`" + diff <(echo "$POLAR_ALL") <(echo "$POLAR") S_POLAR=`bin/parallel -j0 $RET_TIME_K echo -S 1/{} ::: $POLAR` - + #" -S '1/sshminix minix'" + + sshwithpass() { + # Minix requires sshpass. The other servers will use ssh-keys + sshpass -f ~/.ssh/minix.password ssh -oLogLevel=quiet "$@" + } + export -f sshwithpass + copy() { # scp, but atomic (avoid half files if disconnected) host=$1 src="$2" dst="$3" cat "$src" | - stdout ssh -oLogLevel=quiet $host "mkdir -p bin;cat > bin/'$dst'.tmp && chmod 755 bin/'$dst'.tmp && mv bin/'$dst'.tmp bin/'$dst'" + sshwithpass $host "mkdir -p bin;cat > bin/'$dst'.tmp && chmod 755 bin/'$dst'.tmp && mv bin/'$dst'.tmp bin/'$dst'" 2>&1 } export -f copy - + par_nonall() { parallel -j$MAXPROC $RET_TIME_K --delay 0.1 --tag \ - --nonall $S_POLAR --argsep ,:- \ + --nonall $S_POLAR -S "1/sshminix minix" --argsep ,:- \ 'source setupenv >&/dev/null || . `pwd`/setupenv;' "$@" } export -f par_nonall @@ -56,53 +70,56 @@ doit() { echo '### Copy commands to servers' env_parallel -vj$MAXPROC $RET_TIME_K --delay 0.03 --tag copy {2} {1} {1/} \ ::: bin/{parallel,env_parallel,env_parallel.*[^~],parcat,stdout} \ - ::: $POLAR + ::: $POLAR minix echo Done copying + env_parallel -d '\n\n' -vkj$MAXINNERPROC --delay 2 <<'EOF' + echo echo '### Works on ...' echo par_nonall parallel echo Works on {} ::: '`hostname`' 2>&1 - # Test empty command - test_empty_cmd() { - echo '### Test if empty command in process list causes problems' - perl -e '$0=" ";sleep 10' & - parallel echo ::: OK_with_empty_cmd - } - export -f test_empty_cmd - PARALLEL='--env test_empty_cmd' par_nonall test_empty_cmd 2>&1 + echo + echo '### --number-of-cores/--number-of-cpus should work with no error' + echo + par_nonall 'parallel --number-of-sockets; parallel --number-of-cores' 2>&1 + par_nonall 'parallel --number-of-threads; parallel --number-of-cpus' 2>&1 echo echo '### Fails if tmpdir is R/O' echo - par_nonall "stdout parallel --tmpdir / echo ::: test read-only tmp |" \ - "perl -pe '\$exit += s:/[a-z0-9_]+.arg:/XXXXXXXX.arg:gi; \$exit += s/[0-9][0-9][0-9][0-9]/0000/gi; END { exit not \$exit }' &&" \ - "echo OK readonly tmp" 2>&1 + par_nonall "stdout parallel --tmpdir / echo ::: test read-only tmp | + perl -pe '\$exit += s:/[a-z0-9_]+.arg:/XXXXXXXX.arg:gi; \$exit += s/[0-9][0-9][0-9][0-9]/0000/gi; END { exit not \$exit }' && + echo OK readonly tmp" 2>&1 echo - echo '### --number-of-cores/--number-of-cpus should work with no error' - echo - par_nonall parallel --number-of-sockets 2>&1 - par_nonall parallel --number-of-cores 2>&1 - par_nonall parallel --number-of-threads 2>&1 - par_nonall parallel --number-of-cpus 2>&1 - - echo - echo '### Does exporting a bash function kill parallel' + echo '### Does exporting a bash function make parallel fail?' + echo 'If login shell is not bash compatible it fails' echo # http://zmwangx.github.io/blog/2015-11-25-bash-function-exporting-fiasco.html - par_nonall 'func() { cat <(echo bash only A); };export -f func; bin/parallel func ::: 1' 2>&1 + par_nonall 'echo test funcA + funcA() { + cat <(echo bash only A) + } + export -f funcA; + bin/parallel funcA ::: 1' 2>&1 echo - echo '### Does PARALLEL_SHELL help exporting a bash function not kill parallel' + echo '### Does PARALLEL_SHELL help exporting a bash function' + echo 'If login shell is not bash compatible it should work' echo - ( - mkdir -p tmp/bin; - cp /bin/bash tmp/bin - cd tmp - PARALLEL_SHELL=bin/bash par_nonall 'func() { cat <(echo bash only B); };export -f func; bin/parallel func ::: 1' - ) + mkdir -p tmp/bin + cp /bin/bash tmp/bin + cd tmp + export PARALLEL_SHELL=bin/bash + par_nonall 'echo test funcB + funcB() { + cat <(echo bash only B) + } + export -f funcB + export PARALLEL_SHELL=bin/bash + bin/parallel funcB ::: 1' 2>&1 echo echo '### env_parallel echo :::: <(echo OK)' @@ -110,10 +127,22 @@ doit() { echo par_nonall 'bin/env_parallel --install && echo install-OK' 2>&1 par_nonall 'env_parallel echo env_parallel ::: run-OK' 2>&1 - par_nonall 'env_parallel echo reading from process substitution :::: <(echo OK)' | - # csh on NetBSD does not support process substitution + # csh on NetBSD does not support process substitution + par_nonall 'env_parallel echo reading from process substitution :::: <(echo OK)' 2>&1 | grep -v ': /tmp/.*: No such file or directory' + # Test empty command name in process list + test_empty_cmd() { + echo '### Test if empty command name in process list causes problems' + perl -e '$0=" ";sleep 1000' & + pid=$! + parallel echo ::: OK_with_empty_cmd + kill $pid + } + export -f test_empty_cmd + export PARALLEL_SHELL=bin/bash + PARALLEL='--env test_empty_cmd' par_nonall test_empty_cmd 2>&1 + echo echo '### parset arr seq ::: 2 3 4' echo '(bash ksh mksh zsh only)' @@ -129,6 +158,8 @@ doit() { par_nonall 'parset var1,var2,var3 seq ::: 2 3 4; echo $var1,$var2,$var3' 2>&1 echo '### env_parset var1,var2,var3 seq ::: 2 3 4' par_nonall 'start=2; env_parset var1,var2,var3 seq \$start ::: 2 3 4; echo $var1,$var2,$var3' 2>&1 +EOF + } env_parallel -u -S$MASTER doit ::: 1 diff --git a/testsuite/tests-to-run/parallel-remote1.sh b/testsuite/tests-to-run/parallel-remote1.sh index 8663f320..20fb43b4 100644 --- a/testsuite/tests-to-run/parallel-remote1.sh +++ b/testsuite/tests-to-run/parallel-remote1.sh @@ -13,7 +13,7 @@ echo 'ssh "$@"; echo "$@" >>/tmp/myssh2-run' >/tmp/myssh2 chmod 755 /tmp/myssh1 /tmp/myssh2 seq 1 100 | parallel --sshdelay 0.03 --retries 10 --sshlogin "/tmp/myssh1 $SSHLOGIN1,/tmp/myssh2 $SSHLOGIN2" -k echo -cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/\;s/\$SSHLOGIN1/$SSHLOGIN1/\;s/\$SSHLOGIN2/$SSHLOGIN2/\;s/\$SSHLOGIN3/$SSHLOGIN3/ | parallel -vj3 -k -L1 +cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/\;s/\$SSHLOGIN1/$SSHLOGIN1/\;s/\$SSHLOGIN2/$SSHLOGIN2/\;s/\$SSHLOGIN3/$SSHLOGIN3/ | parallel -vj3 -k -L1 -r echo '### test --timeout --retries' parallel -j0 --timeout 5 --retries 3 -k ssh {} echo {} ::: 192.168.1.197 8.8.8.8 $SSHLOGIN1 $SSHLOGIN2 $SSHLOGIN3 diff --git a/testsuite/tests-to-run/sem01.sh b/testsuite/tests-to-run/sem01.sh index 3dc91041..a9ef466a 100755 --- a/testsuite/tests-to-run/sem01.sh +++ b/testsuite/tests-to-run/sem01.sh @@ -1,7 +1,7 @@ #!/bin/bash # -L1 will join lines ending in ' ' -cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj0 -k --joblog /tmp/jl-`basename $0` -L1 +cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj0 -k --joblog /tmp/jl-`basename $0` -L1 -r echo '### Test mutex. This should not mix output'; parallel --semaphore --id mutex -u seq 1 10 '|' pv -qL 20; parallel --semaphore --id mutex -u seq 11 20 '|' pv -qL 100; @@ -55,4 +55,4 @@ echo '### Test --st +1/-1' stdout sem --id st --wait -EOF \ No newline at end of file +EOF diff --git a/testsuite/tests-to-run/test30.sh b/testsuite/tests-to-run/test30.sh index e6520d18..679226cc 100644 --- a/testsuite/tests-to-run/test30.sh +++ b/testsuite/tests-to-run/test30.sh @@ -1,6 +1,6 @@ #!/bin/bash -cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | parallel -vj10 -k --joblog /tmp/jl-`basename $0` -L1 +cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | parallel -vj10 -k --joblog /tmp/jl-`basename $0` -L1 -r echo '### Test of --eta' seq 1 10 | stdout parallel --eta "sleep 1; echo {}" | wc -l diff --git a/testsuite/tests-to-run/test45.sh b/testsuite/tests-to-run/test45.sh index 3434101c..a8ca1e86 100644 --- a/testsuite/tests-to-run/test45.sh +++ b/testsuite/tests-to-run/test45.sh @@ -4,7 +4,7 @@ SERVER1=parallel-server3 SERVER2=parallel-server1 # -L1 will join lines ending in ' ' -cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | parallel -vj10 -k --joblog /tmp/jl-`basename $0` -L1 +cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | parallel -vj10 -k --joblog /tmp/jl-`basename $0` -L1 -r echo '### bug #32191: Deep recursion on subroutine main::get_job_with_sshlogin' seq 1 150 | stdout nice parallel -j9 --retries 2 -S localhost,: "/bin/non-existant 2>/dev/null" diff --git a/testsuite/tests-to-run/test60.sh b/testsuite/tests-to-run/test60.sh index 534f6101..dc25b90b 100644 --- a/testsuite/tests-to-run/test60.sh +++ b/testsuite/tests-to-run/test60.sh @@ -6,7 +6,7 @@ SSHLOGIN1=parallel@$SERVER1 SSHLOGIN2=parallel@$SERVER2 # -L1 will join lines ending in ' ' -cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/\;s/\$SSHLOGIN1/$SSHLOGIN1/\;s/\$SSHLOGIN2/$SSHLOGIN2/ | parallel -vj10 -k --joblog /tmp/jl-`basename $0` -L1 +cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/\;s/\$SSHLOGIN1/$SSHLOGIN1/\;s/\$SSHLOGIN2/$SSHLOGIN2/ | parallel -vj10 -k --joblog /tmp/jl-`basename $0` -L1 -r echo '### Test --onall'; parallel --onall -S $SSHLOGIN1,$SSHLOGIN2 '(echo {1} {2}) | awk \{print\ \$2}' ::: a b c ::: 1 2 diff --git a/testsuite/tests-to-run/test61.sh b/testsuite/tests-to-run/test61.sh index 2530e359..f9d19315 100644 --- a/testsuite/tests-to-run/test61.sh +++ b/testsuite/tests-to-run/test61.sh @@ -3,7 +3,7 @@ SERVER1=parallel-server1 SERVER2=parallel-server2 -cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | parallel -vj10 -k --joblog /tmp/jl-`basename $0` -L1 | perl -pe 's/(PARALLEL_PID....)\d+/$1XXXXX/g' +cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | parallel -vj10 -k --joblog /tmp/jl-`basename $0` -L1 -r | perl -pe 's/(PARALLEL_PID....)\d+/$1XXXXX/g' echo '### Test --return of weirdly named file' stdout parallel --return {} -vv -S parallel\@$SERVER1 echo '>'{} ::: 'aa<${#}" b' | perl -pe 's/\S*parallel-server\S*/one-server/;s:[a-z+=/\\0-9]{500,}:base64:i;'; rm 'aa<${#}" b' diff --git a/testsuite/wanted-results/parallel-local-0.3s b/testsuite/wanted-results/parallel-local-0.3s index 99eb67e5..b74ab173 100644 --- a/testsuite/wanted-results/parallel-local-0.3s +++ b/testsuite/wanted-results/parallel-local-0.3s @@ -389,7 +389,7 @@ d (echo a_b' ';echo c;echo d) | xargs -L1 echo a_b c d - echo '### xargs -L2 echo' +echo '### xargs -L2 echo' ### xargs -L2 echo (echo a_b' ';echo c;echo d;echo e) | parallel -km -L2 echo a_b c d @@ -400,7 +400,7 @@ e (echo a_b' ';echo c;echo d;echo e) | xargs -L2 echo a_b c d e - echo '### xargs -l echo' +echo '### xargs -l echo' ### xargs -l echo (echo a_b' ';echo c;echo d;echo e) | parallel -l -km echo # This behaves wrong a_b c @@ -414,7 +414,7 @@ e a_b c d e - echo '### xargs -l2 echo' +echo '### xargs -l2 echo' ### xargs -l2 echo (echo a_b' ';echo c;echo d;echo e) | parallel -km -l2 echo a_b c d @@ -425,7 +425,7 @@ e (echo a_b' ';echo c;echo d;echo e) | xargs -l2 echo a_b c d e - echo '### xargs -l1 echo' +echo '### xargs -l1 echo' ### xargs -l1 echo (echo a_b' ';echo c;echo d;echo e) | parallel -km -l1 echo a_b c @@ -439,7 +439,7 @@ e a_b c d e - echo '### xargs --max-lines=2 echo' +echo '### xargs --max-lines=2 echo' ### xargs --max-lines=2 echo (echo a_b' ';echo c;echo d;echo e) | parallel -km --max-lines 2 echo a_b c d @@ -450,7 +450,7 @@ e (echo a_b' ';echo c;echo d;echo e) | xargs --max-lines=2 echo a_b c d e - echo '### xargs --max-lines echo' +echo '### xargs --max-lines echo' ### xargs --max-lines echo (echo a_b' ';echo c;echo d;echo e) | parallel --max-lines -km echo # This behaves wrong a_b c @@ -464,7 +464,7 @@ e a_b c d e - echo '### test too long args' +echo '### test too long args' ### test too long args perl -e 'print "z"x1000000' | parallel echo 2>&1 parallel: Error: Command line too long (1000005 >= 131049) at input 0: zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz... @@ -491,7 +491,7 @@ xargs: argument line too long 7 8 9 10 parallel: Error: Command line too long (1000007 >= 10) at input 0: zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz... - echo '### Test -x' +echo '### Test -x' ### Test -x (seq 1 10; echo 12345; seq 12 15) | stdsort parallel -j1 -km -s 10 -x echo 1 2 @@ -546,7 +546,7 @@ xargs: argument line too long 5 6 7 8 9 10 - echo '### Test -a and --arg-file: Read input from file instead of stdin' +echo '### Test -a and --arg-file: Read input from file instead of stdin' ### Test -a and --arg-file: Read input from file instead of stdin seq 1 10 >/tmp/parallel_$$-1; parallel -k -a /tmp/parallel_$$-1 echo; rm /tmp/parallel_$$-1 1 @@ -570,7 +570,7 @@ xargs: argument line too long 8 9 10 - echo '### Test -i and --replace: Replace with argument' +echo '### Test -i and --replace: Replace with argument' ### Test -i and --replace: Replace with argument (echo a; echo END; echo b) | parallel -k -i -eEND echo repl{}ce replace @@ -590,13 +590,13 @@ replace replace (echo a; echo END; echo b) | parallel -k -I^ -eEND echo repl^ce replace - echo '### Test -E: Artificial end-of-file' +echo '### Test -E: Artificial end-of-file' ### Test -E: Artificial end-of-file (echo include this; echo END; echo not this) | parallel -k -E END echo include this (echo include this; echo END; echo not this) | parallel -k -EEND echo include this - echo '### Test -e and --eof: Artificial end-of-file' +echo '### Test -e and --eof: Artificial end-of-file' ### Test -e and --eof: Artificial end-of-file (echo include this; echo END; echo not this) | parallel -k -e END echo include this @@ -606,7 +606,7 @@ include this include this (echo include this; echo END; echo not this) | parallel -k --eof END echo include this - echo '### Test -n and --max-args: Max number of args per line (only with -X and -m)' +echo '### Test -n and --max-args: Max number of args per line (only with -X and -m)' ### Test -n and --max-args: Max number of args per line (only with -X and -m) (echo line 1;echo line 2;echo line 3) | parallel -k -n1 -m echo line 1 @@ -650,7 +650,7 @@ line 3 (echo line 1;echo line 1;echo line 2) | parallel -k --max-args 2 echo line 1 line 1 line 2 - echo '### Test --max-procs and -P: Number of processes' +echo '### Test --max-procs and -P: Number of processes' ### Test --max-procs and -P: Number of processes seq 1 10 | parallel -k --max-procs +0 echo max proc max proc 1 @@ -674,7 +674,7 @@ max proc 10 200% proc 8 200% proc 9 200% proc 10 - echo '### Test --delimiter and -d: Delimiter instead of newline' +echo '### Test --delimiter and -d: Delimiter instead of newline' ### Test --delimiter and -d: Delimiter instead of newline echo '# Yes there is supposed to be an extra newline for -d N' # Yes there is supposed to be an extra newline for -d N @@ -696,7 +696,7 @@ line 3 delimiter TAB line 1 line 2 line 3 - echo '### Test --max-chars and -s: Max number of chars in a line' +echo '### Test --max-chars and -s: Max number of chars in a line' ### Test --max-chars and -s: Max number of chars in a line (echo line 1;echo line 1;echo line 2) | parallel -k --max-chars 25 -X echo line 1 line 1 @@ -704,11 +704,11 @@ line 2 (echo line 1;echo line 1;echo line 2) | parallel -k -s 25 -X echo line 1 line 1 line 2 - echo '### Test --no-run-if-empty and -r: This should give no output' +echo '### Test --no-run-if-empty and -r: This should give no output' ### Test --no-run-if-empty and -r: This should give no output echo " " | parallel -r echo echo " " | parallel --no-run-if-empty echo - echo '### Test --help and -h: Help output (just check we get the same amount of lines)' +echo '### Test --help and -h: Help output (just check we get the same amount of lines)' ### Test --help and -h: Help output (just check we get the same amount of lines) echo Output from -h and --help Output from -h and --help @@ -716,11 +716,11 @@ Output from -h and --help 37 parallel --help | wc -l 37 - echo '### Test --version: Version output (just check we get the same amount of lines)' +echo '### Test --version: Version output (just check we get the same amount of lines)' ### Test --version: Version output (just check we get the same amount of lines) parallel --version | wc -l -11 - echo '### Test --verbose and -t' +10 +echo '### Test --verbose and -t' ### Test --verbose and -t (echo b; echo c; echo f) | parallel -k -t echo {}ar 2>&1 >/dev/null echo bar @@ -730,7 +730,7 @@ echo far echo bar echo car echo far - echo '### Test --show-limits' +echo '### Test --show-limits' ### Test --show-limits (echo b; echo c; echo f) | parallel -k --show-limits echo {}ar Maximal size of command: 131049 @@ -750,11 +750,11 @@ Execution of will continue now, and it will try to read its input and run commands; if this is not what you wanted to happen, please press CTRL-D or CTRL-C bar car far - echo '### Test empty line as input' +echo '### Test empty line as input' ### Test empty line as input echo | parallel echo empty input line empty input line - echo '### Tests if (cat | sh) works' +echo '### Tests if (cat | sh) works' ### Tests if (cat | sh) works perl -e 'for(1..25) {print "echo a $_; echo b $_\n"}' | parallel 2>&1 | sort a 1 @@ -807,7 +807,7 @@ b 6 b 7 b 8 b 9 - echo '### Test if xargs-mode works' +echo '### Test if xargs-mode works' ### Test if xargs-mode works perl -e 'for(1..25) {print "a $_\nb $_\n"}' | parallel echo 2>&1 | sort a 1 @@ -860,12 +860,12 @@ b 6 b 7 b 8 b 9 - echo '### Test -q' +echo '### Test -q' ### Test -q parallel -kq perl -e '$ARGV[0]=~/^\S+\s+\S+$/ and print $ARGV[0],"\n"' ::: "a b" c "d e f" g "h i" a b h i - echo '### Test -q {#}' +echo '### Test -q {#}' ### Test -q {#} parallel -kq echo {#} ::: a b 1 @@ -876,7 +876,7 @@ h i parallel -kq echo {\\#} ::: a b {\#} a {\#} b - echo '### Test long commands do not take up all memory' +echo '### Test long commands do not take up all memory' ### Test long commands do not take up all memory seq 1 100 | parallel -j0 -qv perl -e '$r=rand(shift);for($f=0;$f<$r;$f++){$a="a"x100};print shift,"\n"' 10000 2>/dev/null | sort 1 @@ -1079,7 +1079,7 @@ perl -e '$r=rand(shift);for($f=0;$f<$r;$f++){$a="a"x100};print shift,"\n"' 10000 perl -e '$r=rand(shift);for($f=0;$f<$r;$f++){$a="a"x100};print shift,"\n"' 10000 97 perl -e '$r=rand(shift);for($f=0;$f<$r;$f++){$a="a"x100};print shift,"\n"' 10000 98 perl -e '$r=rand(shift);for($f=0;$f<$r;$f++){$a="a"x100};print shift,"\n"' 10000 99 - echo '### Test 0-arguments' +echo '### Test 0-arguments' ### Test 0-arguments seq 1 2 | parallel -k -n0 echo n0 n0 @@ -1090,34 +1090,34 @@ L0 seq 1 2 | parallel -k -N0 echo N0 N0 N0 - echo '### Because of --tollef -l, then -l0 == -l1, sorry' +echo '### Because of --tollef -l, then -l0 == -l1, sorry' ### Because of --tollef -l, then -l0 == -l1, sorry seq 1 2 | parallel -k -l0 echo l0 l0 1 l0 2 - echo '### Test replace {}' +echo '### Test replace {}' ### Test replace {} seq 1 2 | parallel -k -N0 echo replace {} curlies replace curlies replace curlies - echo '### Test arguments on commandline' +echo '### Test arguments on commandline' ### Test arguments on commandline parallel -k -N0 echo args on cmdline ::: 1 2 args on cmdline args on cmdline - echo '### Test --nice locally' +echo '### Test --nice locally' ### Test --nice locally parallel --nice 1 -vv 'PAR=a bash -c "echo \$PAR {}"' ::: b PAR=a bash -c "echo \$PAR b" a b - echo '### Test distribute arguments at EOF to 2 jobslots' +echo '### Test distribute arguments at EOF to 2 jobslots' ### Test distribute arguments at EOF to 2 jobslots seq 1 92 | parallel -j2 -kX -s 100 echo 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 - echo '### Test distribute arguments at EOF to 5 jobslots' +echo '### Test distribute arguments at EOF to 5 jobslots' ### Test distribute arguments at EOF to 5 jobslots seq 1 92 | parallel -j5 -kX -s 100 echo 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 @@ -1127,7 +1127,7 @@ a b 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 - echo '### Test distribute arguments at EOF to infinity jobslots' +echo '### Test distribute arguments at EOF to infinity jobslots' ### Test distribute arguments at EOF to infinity jobslots seq 1 92 | parallel -j0 -kX -s 100 echo 2>/dev/null 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 @@ -1159,26 +1159,26 @@ a b 90 91 92 - echo '### Test -N is not broken by distribution - single line' +echo '### Test -N is not broken by distribution - single line' ### Test -N is not broken by distribution - single line seq 9 | parallel -N 10 echo 1 2 3 4 5 6 7 8 9 - echo '### Test -N is not broken by distribution - two lines' +echo '### Test -N is not broken by distribution - two lines' ### Test -N is not broken by distribution - two lines seq 19 | parallel -k -N 10 echo 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 - echo '### Test -N context replace' +echo '### Test -N context replace' ### Test -N context replace seq 19 | parallel -k -N 10 echo a{}b a1b a2b a3b a4b a5b a6b a7b a8b a9b a10b a11b a12b a13b a14b a15b a16b a17b a18b a19b - echo '### Test -L context replace' +echo '### Test -L context replace' ### Test -L context replace seq 19 | parallel -k -L 10 echo a{}b a1b a2b a3b a4b a5b a6b a7b a8b a9b a10b a11b a12b a13b a14b a15b a16b a17b a18b a19b - echo '**' +echo '**' ** echo '### Test {} multiple times in different commands' ### Test {} multiple times in different commands @@ -1313,6 +1313,10 @@ par_dryrun_append_joblog echo 4 par_dryrun_append_joblog 4 par_empty bug #: par_empty true +par_empty_command bug #54647: parset ignores empty lines +par_empty_command +par_empty_command Empty: +par_empty_command B: B par_empty_line ### Test bug: empty line for | sh with -k par_empty_line a par_empty_line b diff --git a/testsuite/wanted-results/parallel-local-10s b/testsuite/wanted-results/parallel-local-10s index 5d53f448..64481b79 100644 --- a/testsuite/wanted-results/parallel-local-10s +++ b/testsuite/wanted-results/parallel-local-10s @@ -258,7 +258,7 @@ par_linebuffer_tag_slow_output b b par_linebuffer_tag_slow_output b par_long_line_remote ### Deal with long command lines on remote servers par_long_line_remote 2 6 30006 -par_long_line_remote 2 50 250050 +par_long_line_remote 3 50 250050 par_maxlinelen_X_I ### Test max line length -X -I par_maxlinelen_X_I 3cfc69ee81b0fe7fdbe8eb059ad2da61 - par_maxlinelen_X_I Chars per line (817788/7): 116826 diff --git a/testsuite/wanted-results/parallel-local-ssh1 b/testsuite/wanted-results/parallel-local-ssh1 index a12928e0..2de05f3c 100644 --- a/testsuite/wanted-results/parallel-local-ssh1 +++ b/testsuite/wanted-results/parallel-local-ssh1 @@ -34,30 +34,6 @@ echo '### Can csh propagate a variable containing \n'; export A=$(seq 3); para 1 2 3 -echo '### bug #41805: Idea: propagate --env for parallel --number-of-cores' -### bug #41805: Idea: propagate --env for parallel --number-of-cores - echo '** test_zsh' -** test_zsh - FOO=test_zsh parallel --env FOO,HOME -S zsh@lo -N0 env ::: "" |sort|egrep 'FOO|^HOME' -FOO=test_zsh -HOME=~ - echo '** test_zsh_filter' -** test_zsh_filter - FOO=test_zsh_filter parallel --filter-hosts --env FOO,HOME -S zsh@lo -N0 env ::: "" |sort|egrep 'FOO|^HOME' -FOO=test_zsh_filter -HOME=~ - echo '** test_csh' -** test_csh - FOO=test_csh parallel --env FOO,HOME -S csh@lo -N0 env ::: "" |sort|egrep 'FOO|^HOME' -FOO=test_csh -HOME=~ - echo '** test_csh_filter' -** test_csh_filter - FOO=test_csh_filter parallel --filter-hosts --env FOO,HOME -S csh@lo -N0 env ::: "" |sort|egrep 'FOO|^HOME' -FOO=test_csh_filter -HOME=~ - echo '** bug #41805 done' -** bug #41805 done echo '### Test bug #34241: --pipe should not spawn unneeded processes' ### Test bug #34241: --pipe should not spawn unneeded processes seq 5 | ssh csh@lo parallel -k --block 5 --pipe -j10 cat\\\;echo Block_end diff --git a/testsuite/wanted-results/parallel-local-ssh9 b/testsuite/wanted-results/parallel-local-ssh9 index 6d763281..ccb5f63f 100644 --- a/testsuite/wanted-results/parallel-local-ssh9 +++ b/testsuite/wanted-results/parallel-local-ssh9 @@ -40,6 +40,20 @@ par_sh_embed your par_sh_embed code par_sh_embed here par_sh_embed Put your code here +par_propagate_env ### bug #41805: Idea: propagate --env for parallel --number-of-cores +par_propagate_env ** test_zsh +par_propagate_env FOO=test_zsh +par_propagate_env HOME=~ +par_propagate_env ** test_zsh_filter +par_propagate_env FOO=test_zsh_filter +par_propagate_env HOME=~ +par_propagate_env ** test_csh +par_propagate_env FOO=test_csh +par_propagate_env HOME=~ +par_propagate_env ** test_csh_filter +par_propagate_env FOO=test_csh_filter +par_propagate_env HOME=~ +par_propagate_env ** bug #41805 done par_ksh_embed --embed par_ksh_embed Redirect the output to a file and add your changes at the end: par_ksh_embed /usr/local/bin/parallel --embed > new_script @@ -66,6 +80,9 @@ par_ksh_embed code par_ksh_embed here par_ksh_embed Put your code here par_fish_embed Not implemented +par_env_parallel_big_env ### bug #54128: command too long when exporting big env +par_env_parallel_big_env should not fail +par_env_parallel_big_env OK par_csh_embed Not implemented par_bash_embed --embed par_bash_embed Redirect the output to a file and add your changes at the end: diff --git a/testsuite/wanted-results/parallel-local19 b/testsuite/wanted-results/parallel-local19 index a23f0ef2..8da77eaa 100644 --- a/testsuite/wanted-results/parallel-local19 +++ b/testsuite/wanted-results/parallel-local19 @@ -135,7 +135,7 @@ a13.gifb13c13 a14.gifb14c14 a15.gifb15c15 a1.gif 2.gif 3.gif 4.gif 5.gif 6.gif 7.gifb1 2 3 4 5 6 7c1 2 3 4 5 6 7 a8.gif 9.gif 10.gif 11.gif 12.gif 13.gifb8 9 10 11 12 13c8 9 10 11 12 13 a14.gif 15.gifb14 15c14 15 - echo '### Test -I with shell meta chars' +echo '### Test -I with shell meta chars' ### Test -I with shell meta chars seq 10000 | parallel -j1 -I :: -X echo a::b::c:: | wc -l 2 @@ -145,7 +145,7 @@ a14.gif 15.gifb14 15c14 15 2 seq 10000 | parallel -j1 -I '>' -X echo 'a>b>c>' | wc -l 2 - echo '### Test {.}' +echo '### Test {.}' ### Test {.} echo a | parallel -qX echo "'"{.}"' " 'a' diff --git a/testsuite/wanted-results/parallel-local23 b/testsuite/wanted-results/parallel-local23 index c3511d92..ebf388d5 100644 --- a/testsuite/wanted-results/parallel-local23 +++ b/testsuite/wanted-results/parallel-local23 @@ -61,8 +61,8 @@ echo '### Check that 4 processes are really used' echo '### --version must have higher priority than retired options' ### --version must have higher priority than retired options $NICEPAR --version -g -Y -U -W -T | tail -Copyright (C) 2007,2008,2009,2010,2011,2012,2013,2014,2015,2016,2017,2018 -Ole Tange and Free Software Foundation, Inc. +GNU parallel 20180916 +Copyright (C) 2007-2018 Ole Tange and Free Software Foundation, Inc. License GPLv3+: GNU GPL version 3 or later This is free software: you are free to change and redistribute it. GNU parallel comes with no warranty. diff --git a/testsuite/wanted-results/parallel-polarhome b/testsuite/wanted-results/parallel-polarhome index 77b604e4..ce8fb5e5 100644 --- a/testsuite/wanted-results/parallel-polarhome +++ b/testsuite/wanted-results/parallel-polarhome @@ -1,17 +1,38 @@ -MAXTIME=25 RETRIES=4 MAXPROC=150 +MAXTIME=50 RETRIES=2 MAXPROC=11 MAXINNERPROC=3 ### Filter out working servers +1d0 +< openstep +4d2 +< alpha +7d4 +< syllable +10d6 +< openindiana +13d8 +< debian-ppc +17d11 +< ubuntu +28,32d21 +< dragonfly +< vax +< ultrix +< minix +< irix +34,35d22 +< beaglebone +< cubieboard2 ### Copy commands to servers bin/parallel qnx copy qnx bin/parallel parallel bin/parallel pidora copy pidora bin/parallel parallel +bin/parallel tru64 copy tru64 bin/parallel parallel +bin/parallel hpux-ia64 copy hpux-ia64 bin/parallel parallel bin/parallel raspbian copy raspbian bin/parallel parallel bin/parallel solaris copy solaris bin/parallel parallel -bin/parallel openindiana copy openindiana bin/parallel parallel bin/parallel aix copy aix bin/parallel parallel bin/parallel hpux copy hpux bin/parallel parallel bin/parallel suse copy suse bin/parallel parallel bin/parallel solaris-x86 copy solaris-x86 bin/parallel parallel bin/parallel mandriva copy mandriva bin/parallel parallel -bin/parallel ubuntu copy ubuntu bin/parallel parallel bin/parallel scosysv copy scosysv bin/parallel parallel bin/parallel unixware copy unixware bin/parallel parallel bin/parallel centos copy centos bin/parallel parallel @@ -23,17 +44,19 @@ bin/parallel openbsd copy openbsd bin/parallel parallel bin/parallel freebsd copy freebsd bin/parallel parallel bin/parallel debian copy debian bin/parallel parallel bin/parallel hurd copy hurd bin/parallel parallel +bin/parallel minix copy minix bin/parallel parallel +bin/parallel minix cat: stdout: no space left on device bin/env_parallel qnx copy qnx bin/env_parallel env_parallel bin/env_parallel pidora copy pidora bin/env_parallel env_parallel +bin/env_parallel tru64 copy tru64 bin/env_parallel env_parallel +bin/env_parallel hpux-ia64 copy hpux-ia64 bin/env_parallel env_parallel bin/env_parallel raspbian copy raspbian bin/env_parallel env_parallel bin/env_parallel solaris copy solaris bin/env_parallel env_parallel -bin/env_parallel openindiana copy openindiana bin/env_parallel env_parallel bin/env_parallel aix copy aix bin/env_parallel env_parallel bin/env_parallel hpux copy hpux bin/env_parallel env_parallel bin/env_parallel suse copy suse bin/env_parallel env_parallel bin/env_parallel solaris-x86 copy solaris-x86 bin/env_parallel env_parallel bin/env_parallel mandriva copy mandriva bin/env_parallel env_parallel -bin/env_parallel ubuntu copy ubuntu bin/env_parallel env_parallel bin/env_parallel scosysv copy scosysv bin/env_parallel env_parallel bin/env_parallel unixware copy unixware bin/env_parallel env_parallel bin/env_parallel centos copy centos bin/env_parallel env_parallel @@ -45,17 +68,19 @@ bin/env_parallel openbsd copy openbsd bin/env_parallel env_parallel bin/env_parallel freebsd copy freebsd bin/env_parallel env_parallel bin/env_parallel debian copy debian bin/env_parallel env_parallel bin/env_parallel hurd copy hurd bin/env_parallel env_parallel +bin/env_parallel minix copy minix bin/env_parallel env_parallel +bin/env_parallel minix cat: stdout: no space left on device bin/env_parallel.ash qnx copy qnx bin/env_parallel.ash env_parallel.ash bin/env_parallel.ash pidora copy pidora bin/env_parallel.ash env_parallel.ash +bin/env_parallel.ash tru64 copy tru64 bin/env_parallel.ash env_parallel.ash +bin/env_parallel.ash hpux-ia64 copy hpux-ia64 bin/env_parallel.ash env_parallel.ash bin/env_parallel.ash raspbian copy raspbian bin/env_parallel.ash env_parallel.ash bin/env_parallel.ash solaris copy solaris bin/env_parallel.ash env_parallel.ash -bin/env_parallel.ash openindiana copy openindiana bin/env_parallel.ash env_parallel.ash bin/env_parallel.ash aix copy aix bin/env_parallel.ash env_parallel.ash bin/env_parallel.ash hpux copy hpux bin/env_parallel.ash env_parallel.ash bin/env_parallel.ash suse copy suse bin/env_parallel.ash env_parallel.ash bin/env_parallel.ash solaris-x86 copy solaris-x86 bin/env_parallel.ash env_parallel.ash bin/env_parallel.ash mandriva copy mandriva bin/env_parallel.ash env_parallel.ash -bin/env_parallel.ash ubuntu copy ubuntu bin/env_parallel.ash env_parallel.ash bin/env_parallel.ash scosysv copy scosysv bin/env_parallel.ash env_parallel.ash bin/env_parallel.ash unixware copy unixware bin/env_parallel.ash env_parallel.ash bin/env_parallel.ash centos copy centos bin/env_parallel.ash env_parallel.ash @@ -67,17 +92,19 @@ bin/env_parallel.ash openbsd copy openbsd bin/env_parallel.ash env_parallel.ash bin/env_parallel.ash freebsd copy freebsd bin/env_parallel.ash env_parallel.ash bin/env_parallel.ash debian copy debian bin/env_parallel.ash env_parallel.ash bin/env_parallel.ash hurd copy hurd bin/env_parallel.ash env_parallel.ash +bin/env_parallel.ash minix copy minix bin/env_parallel.ash env_parallel.ash +bin/env_parallel.ash minix cat: stdout: no space left on device bin/env_parallel.bash qnx copy qnx bin/env_parallel.bash env_parallel.bash bin/env_parallel.bash pidora copy pidora bin/env_parallel.bash env_parallel.bash +bin/env_parallel.bash tru64 copy tru64 bin/env_parallel.bash env_parallel.bash +bin/env_parallel.bash hpux-ia64 copy hpux-ia64 bin/env_parallel.bash env_parallel.bash bin/env_parallel.bash raspbian copy raspbian bin/env_parallel.bash env_parallel.bash bin/env_parallel.bash solaris copy solaris bin/env_parallel.bash env_parallel.bash -bin/env_parallel.bash openindiana copy openindiana bin/env_parallel.bash env_parallel.bash bin/env_parallel.bash aix copy aix bin/env_parallel.bash env_parallel.bash bin/env_parallel.bash hpux copy hpux bin/env_parallel.bash env_parallel.bash bin/env_parallel.bash suse copy suse bin/env_parallel.bash env_parallel.bash bin/env_parallel.bash solaris-x86 copy solaris-x86 bin/env_parallel.bash env_parallel.bash bin/env_parallel.bash mandriva copy mandriva bin/env_parallel.bash env_parallel.bash -bin/env_parallel.bash ubuntu copy ubuntu bin/env_parallel.bash env_parallel.bash bin/env_parallel.bash scosysv copy scosysv bin/env_parallel.bash env_parallel.bash bin/env_parallel.bash unixware copy unixware bin/env_parallel.bash env_parallel.bash bin/env_parallel.bash centos copy centos bin/env_parallel.bash env_parallel.bash @@ -89,17 +116,19 @@ bin/env_parallel.bash openbsd copy openbsd bin/env_parallel.bash env_parallel.ba bin/env_parallel.bash freebsd copy freebsd bin/env_parallel.bash env_parallel.bash bin/env_parallel.bash debian copy debian bin/env_parallel.bash env_parallel.bash bin/env_parallel.bash hurd copy hurd bin/env_parallel.bash env_parallel.bash +bin/env_parallel.bash minix copy minix bin/env_parallel.bash env_parallel.bash +bin/env_parallel.bash minix cat: stdout: no space left on device bin/env_parallel.csh qnx copy qnx bin/env_parallel.csh env_parallel.csh bin/env_parallel.csh pidora copy pidora bin/env_parallel.csh env_parallel.csh +bin/env_parallel.csh tru64 copy tru64 bin/env_parallel.csh env_parallel.csh +bin/env_parallel.csh hpux-ia64 copy hpux-ia64 bin/env_parallel.csh env_parallel.csh bin/env_parallel.csh raspbian copy raspbian bin/env_parallel.csh env_parallel.csh bin/env_parallel.csh solaris copy solaris bin/env_parallel.csh env_parallel.csh -bin/env_parallel.csh openindiana copy openindiana bin/env_parallel.csh env_parallel.csh bin/env_parallel.csh aix copy aix bin/env_parallel.csh env_parallel.csh bin/env_parallel.csh hpux copy hpux bin/env_parallel.csh env_parallel.csh bin/env_parallel.csh suse copy suse bin/env_parallel.csh env_parallel.csh bin/env_parallel.csh solaris-x86 copy solaris-x86 bin/env_parallel.csh env_parallel.csh bin/env_parallel.csh mandriva copy mandriva bin/env_parallel.csh env_parallel.csh -bin/env_parallel.csh ubuntu copy ubuntu bin/env_parallel.csh env_parallel.csh bin/env_parallel.csh scosysv copy scosysv bin/env_parallel.csh env_parallel.csh bin/env_parallel.csh unixware copy unixware bin/env_parallel.csh env_parallel.csh bin/env_parallel.csh centos copy centos bin/env_parallel.csh env_parallel.csh @@ -111,17 +140,19 @@ bin/env_parallel.csh openbsd copy openbsd bin/env_parallel.csh env_parallel.csh bin/env_parallel.csh freebsd copy freebsd bin/env_parallel.csh env_parallel.csh bin/env_parallel.csh debian copy debian bin/env_parallel.csh env_parallel.csh bin/env_parallel.csh hurd copy hurd bin/env_parallel.csh env_parallel.csh +bin/env_parallel.csh minix copy minix bin/env_parallel.csh env_parallel.csh +bin/env_parallel.csh minix cat: stdout: no space left on device bin/env_parallel.dash qnx copy qnx bin/env_parallel.dash env_parallel.dash bin/env_parallel.dash pidora copy pidora bin/env_parallel.dash env_parallel.dash +bin/env_parallel.dash tru64 copy tru64 bin/env_parallel.dash env_parallel.dash +bin/env_parallel.dash hpux-ia64 copy hpux-ia64 bin/env_parallel.dash env_parallel.dash bin/env_parallel.dash raspbian copy raspbian bin/env_parallel.dash env_parallel.dash bin/env_parallel.dash solaris copy solaris bin/env_parallel.dash env_parallel.dash -bin/env_parallel.dash openindiana copy openindiana bin/env_parallel.dash env_parallel.dash bin/env_parallel.dash aix copy aix bin/env_parallel.dash env_parallel.dash bin/env_parallel.dash hpux copy hpux bin/env_parallel.dash env_parallel.dash bin/env_parallel.dash suse copy suse bin/env_parallel.dash env_parallel.dash bin/env_parallel.dash solaris-x86 copy solaris-x86 bin/env_parallel.dash env_parallel.dash bin/env_parallel.dash mandriva copy mandriva bin/env_parallel.dash env_parallel.dash -bin/env_parallel.dash ubuntu copy ubuntu bin/env_parallel.dash env_parallel.dash bin/env_parallel.dash scosysv copy scosysv bin/env_parallel.dash env_parallel.dash bin/env_parallel.dash unixware copy unixware bin/env_parallel.dash env_parallel.dash bin/env_parallel.dash centos copy centos bin/env_parallel.dash env_parallel.dash @@ -133,17 +164,19 @@ bin/env_parallel.dash openbsd copy openbsd bin/env_parallel.dash env_parallel.da bin/env_parallel.dash freebsd copy freebsd bin/env_parallel.dash env_parallel.dash bin/env_parallel.dash debian copy debian bin/env_parallel.dash env_parallel.dash bin/env_parallel.dash hurd copy hurd bin/env_parallel.dash env_parallel.dash +bin/env_parallel.dash minix copy minix bin/env_parallel.dash env_parallel.dash +bin/env_parallel.dash minix cat: stdout: no space left on device bin/env_parallel.fish qnx copy qnx bin/env_parallel.fish env_parallel.fish bin/env_parallel.fish pidora copy pidora bin/env_parallel.fish env_parallel.fish +bin/env_parallel.fish tru64 copy tru64 bin/env_parallel.fish env_parallel.fish +bin/env_parallel.fish hpux-ia64 copy hpux-ia64 bin/env_parallel.fish env_parallel.fish bin/env_parallel.fish raspbian copy raspbian bin/env_parallel.fish env_parallel.fish bin/env_parallel.fish solaris copy solaris bin/env_parallel.fish env_parallel.fish -bin/env_parallel.fish openindiana copy openindiana bin/env_parallel.fish env_parallel.fish bin/env_parallel.fish aix copy aix bin/env_parallel.fish env_parallel.fish bin/env_parallel.fish hpux copy hpux bin/env_parallel.fish env_parallel.fish bin/env_parallel.fish suse copy suse bin/env_parallel.fish env_parallel.fish bin/env_parallel.fish solaris-x86 copy solaris-x86 bin/env_parallel.fish env_parallel.fish bin/env_parallel.fish mandriva copy mandriva bin/env_parallel.fish env_parallel.fish -bin/env_parallel.fish ubuntu copy ubuntu bin/env_parallel.fish env_parallel.fish bin/env_parallel.fish scosysv copy scosysv bin/env_parallel.fish env_parallel.fish bin/env_parallel.fish unixware copy unixware bin/env_parallel.fish env_parallel.fish bin/env_parallel.fish centos copy centos bin/env_parallel.fish env_parallel.fish @@ -155,17 +188,19 @@ bin/env_parallel.fish openbsd copy openbsd bin/env_parallel.fish env_parallel.fi bin/env_parallel.fish freebsd copy freebsd bin/env_parallel.fish env_parallel.fish bin/env_parallel.fish debian copy debian bin/env_parallel.fish env_parallel.fish bin/env_parallel.fish hurd copy hurd bin/env_parallel.fish env_parallel.fish +bin/env_parallel.fish minix copy minix bin/env_parallel.fish env_parallel.fish +bin/env_parallel.fish minix cat: stdout: no space left on device bin/env_parallel.ksh qnx copy qnx bin/env_parallel.ksh env_parallel.ksh bin/env_parallel.ksh pidora copy pidora bin/env_parallel.ksh env_parallel.ksh +bin/env_parallel.ksh tru64 copy tru64 bin/env_parallel.ksh env_parallel.ksh +bin/env_parallel.ksh hpux-ia64 copy hpux-ia64 bin/env_parallel.ksh env_parallel.ksh bin/env_parallel.ksh raspbian copy raspbian bin/env_parallel.ksh env_parallel.ksh bin/env_parallel.ksh solaris copy solaris bin/env_parallel.ksh env_parallel.ksh -bin/env_parallel.ksh openindiana copy openindiana bin/env_parallel.ksh env_parallel.ksh bin/env_parallel.ksh aix copy aix bin/env_parallel.ksh env_parallel.ksh bin/env_parallel.ksh hpux copy hpux bin/env_parallel.ksh env_parallel.ksh bin/env_parallel.ksh suse copy suse bin/env_parallel.ksh env_parallel.ksh bin/env_parallel.ksh solaris-x86 copy solaris-x86 bin/env_parallel.ksh env_parallel.ksh bin/env_parallel.ksh mandriva copy mandriva bin/env_parallel.ksh env_parallel.ksh -bin/env_parallel.ksh ubuntu copy ubuntu bin/env_parallel.ksh env_parallel.ksh bin/env_parallel.ksh scosysv copy scosysv bin/env_parallel.ksh env_parallel.ksh bin/env_parallel.ksh unixware copy unixware bin/env_parallel.ksh env_parallel.ksh bin/env_parallel.ksh centos copy centos bin/env_parallel.ksh env_parallel.ksh @@ -177,17 +212,19 @@ bin/env_parallel.ksh openbsd copy openbsd bin/env_parallel.ksh env_parallel.ksh bin/env_parallel.ksh freebsd copy freebsd bin/env_parallel.ksh env_parallel.ksh bin/env_parallel.ksh debian copy debian bin/env_parallel.ksh env_parallel.ksh bin/env_parallel.ksh hurd copy hurd bin/env_parallel.ksh env_parallel.ksh +bin/env_parallel.ksh minix copy minix bin/env_parallel.ksh env_parallel.ksh +bin/env_parallel.ksh minix cat: stdout: no space left on device bin/env_parallel.mksh qnx copy qnx bin/env_parallel.mksh env_parallel.mksh bin/env_parallel.mksh pidora copy pidora bin/env_parallel.mksh env_parallel.mksh +bin/env_parallel.mksh tru64 copy tru64 bin/env_parallel.mksh env_parallel.mksh +bin/env_parallel.mksh hpux-ia64 copy hpux-ia64 bin/env_parallel.mksh env_parallel.mksh bin/env_parallel.mksh raspbian copy raspbian bin/env_parallel.mksh env_parallel.mksh bin/env_parallel.mksh solaris copy solaris bin/env_parallel.mksh env_parallel.mksh -bin/env_parallel.mksh openindiana copy openindiana bin/env_parallel.mksh env_parallel.mksh bin/env_parallel.mksh aix copy aix bin/env_parallel.mksh env_parallel.mksh bin/env_parallel.mksh hpux copy hpux bin/env_parallel.mksh env_parallel.mksh bin/env_parallel.mksh suse copy suse bin/env_parallel.mksh env_parallel.mksh bin/env_parallel.mksh solaris-x86 copy solaris-x86 bin/env_parallel.mksh env_parallel.mksh bin/env_parallel.mksh mandriva copy mandriva bin/env_parallel.mksh env_parallel.mksh -bin/env_parallel.mksh ubuntu copy ubuntu bin/env_parallel.mksh env_parallel.mksh bin/env_parallel.mksh scosysv copy scosysv bin/env_parallel.mksh env_parallel.mksh bin/env_parallel.mksh unixware copy unixware bin/env_parallel.mksh env_parallel.mksh bin/env_parallel.mksh centos copy centos bin/env_parallel.mksh env_parallel.mksh @@ -199,17 +236,19 @@ bin/env_parallel.mksh openbsd copy openbsd bin/env_parallel.mksh env_parallel.mk bin/env_parallel.mksh freebsd copy freebsd bin/env_parallel.mksh env_parallel.mksh bin/env_parallel.mksh debian copy debian bin/env_parallel.mksh env_parallel.mksh bin/env_parallel.mksh hurd copy hurd bin/env_parallel.mksh env_parallel.mksh +bin/env_parallel.mksh minix copy minix bin/env_parallel.mksh env_parallel.mksh +bin/env_parallel.mksh minix cat: stdout: no space left on device bin/env_parallel.pdksh qnx copy qnx bin/env_parallel.pdksh env_parallel.pdksh bin/env_parallel.pdksh pidora copy pidora bin/env_parallel.pdksh env_parallel.pdksh +bin/env_parallel.pdksh tru64 copy tru64 bin/env_parallel.pdksh env_parallel.pdksh +bin/env_parallel.pdksh hpux-ia64 copy hpux-ia64 bin/env_parallel.pdksh env_parallel.pdksh bin/env_parallel.pdksh raspbian copy raspbian bin/env_parallel.pdksh env_parallel.pdksh bin/env_parallel.pdksh solaris copy solaris bin/env_parallel.pdksh env_parallel.pdksh -bin/env_parallel.pdksh openindiana copy openindiana bin/env_parallel.pdksh env_parallel.pdksh bin/env_parallel.pdksh aix copy aix bin/env_parallel.pdksh env_parallel.pdksh bin/env_parallel.pdksh hpux copy hpux bin/env_parallel.pdksh env_parallel.pdksh bin/env_parallel.pdksh suse copy suse bin/env_parallel.pdksh env_parallel.pdksh bin/env_parallel.pdksh solaris-x86 copy solaris-x86 bin/env_parallel.pdksh env_parallel.pdksh bin/env_parallel.pdksh mandriva copy mandriva bin/env_parallel.pdksh env_parallel.pdksh -bin/env_parallel.pdksh ubuntu copy ubuntu bin/env_parallel.pdksh env_parallel.pdksh bin/env_parallel.pdksh scosysv copy scosysv bin/env_parallel.pdksh env_parallel.pdksh bin/env_parallel.pdksh unixware copy unixware bin/env_parallel.pdksh env_parallel.pdksh bin/env_parallel.pdksh centos copy centos bin/env_parallel.pdksh env_parallel.pdksh @@ -221,17 +260,19 @@ bin/env_parallel.pdksh openbsd copy openbsd bin/env_parallel.pdksh env_parallel. bin/env_parallel.pdksh freebsd copy freebsd bin/env_parallel.pdksh env_parallel.pdksh bin/env_parallel.pdksh debian copy debian bin/env_parallel.pdksh env_parallel.pdksh bin/env_parallel.pdksh hurd copy hurd bin/env_parallel.pdksh env_parallel.pdksh +bin/env_parallel.pdksh minix copy minix bin/env_parallel.pdksh env_parallel.pdksh +bin/env_parallel.pdksh minix cat: stdout: no space left on device bin/env_parallel.sh qnx copy qnx bin/env_parallel.sh env_parallel.sh bin/env_parallel.sh pidora copy pidora bin/env_parallel.sh env_parallel.sh +bin/env_parallel.sh tru64 copy tru64 bin/env_parallel.sh env_parallel.sh +bin/env_parallel.sh hpux-ia64 copy hpux-ia64 bin/env_parallel.sh env_parallel.sh bin/env_parallel.sh raspbian copy raspbian bin/env_parallel.sh env_parallel.sh bin/env_parallel.sh solaris copy solaris bin/env_parallel.sh env_parallel.sh -bin/env_parallel.sh openindiana copy openindiana bin/env_parallel.sh env_parallel.sh bin/env_parallel.sh aix copy aix bin/env_parallel.sh env_parallel.sh bin/env_parallel.sh hpux copy hpux bin/env_parallel.sh env_parallel.sh bin/env_parallel.sh suse copy suse bin/env_parallel.sh env_parallel.sh bin/env_parallel.sh solaris-x86 copy solaris-x86 bin/env_parallel.sh env_parallel.sh bin/env_parallel.sh mandriva copy mandriva bin/env_parallel.sh env_parallel.sh -bin/env_parallel.sh ubuntu copy ubuntu bin/env_parallel.sh env_parallel.sh bin/env_parallel.sh scosysv copy scosysv bin/env_parallel.sh env_parallel.sh bin/env_parallel.sh unixware copy unixware bin/env_parallel.sh env_parallel.sh bin/env_parallel.sh centos copy centos bin/env_parallel.sh env_parallel.sh @@ -243,17 +284,19 @@ bin/env_parallel.sh openbsd copy openbsd bin/env_parallel.sh env_parallel.sh bin/env_parallel.sh freebsd copy freebsd bin/env_parallel.sh env_parallel.sh bin/env_parallel.sh debian copy debian bin/env_parallel.sh env_parallel.sh bin/env_parallel.sh hurd copy hurd bin/env_parallel.sh env_parallel.sh +bin/env_parallel.sh minix copy minix bin/env_parallel.sh env_parallel.sh +bin/env_parallel.sh minix cat: stdout: no space left on device bin/env_parallel.tcsh qnx copy qnx bin/env_parallel.tcsh env_parallel.tcsh bin/env_parallel.tcsh pidora copy pidora bin/env_parallel.tcsh env_parallel.tcsh +bin/env_parallel.tcsh tru64 copy tru64 bin/env_parallel.tcsh env_parallel.tcsh +bin/env_parallel.tcsh hpux-ia64 copy hpux-ia64 bin/env_parallel.tcsh env_parallel.tcsh bin/env_parallel.tcsh raspbian copy raspbian bin/env_parallel.tcsh env_parallel.tcsh bin/env_parallel.tcsh solaris copy solaris bin/env_parallel.tcsh env_parallel.tcsh -bin/env_parallel.tcsh openindiana copy openindiana bin/env_parallel.tcsh env_parallel.tcsh bin/env_parallel.tcsh aix copy aix bin/env_parallel.tcsh env_parallel.tcsh bin/env_parallel.tcsh hpux copy hpux bin/env_parallel.tcsh env_parallel.tcsh bin/env_parallel.tcsh suse copy suse bin/env_parallel.tcsh env_parallel.tcsh bin/env_parallel.tcsh solaris-x86 copy solaris-x86 bin/env_parallel.tcsh env_parallel.tcsh bin/env_parallel.tcsh mandriva copy mandriva bin/env_parallel.tcsh env_parallel.tcsh -bin/env_parallel.tcsh ubuntu copy ubuntu bin/env_parallel.tcsh env_parallel.tcsh bin/env_parallel.tcsh scosysv copy scosysv bin/env_parallel.tcsh env_parallel.tcsh bin/env_parallel.tcsh unixware copy unixware bin/env_parallel.tcsh env_parallel.tcsh bin/env_parallel.tcsh centos copy centos bin/env_parallel.tcsh env_parallel.tcsh @@ -265,17 +308,19 @@ bin/env_parallel.tcsh openbsd copy openbsd bin/env_parallel.tcsh env_parallel.tc bin/env_parallel.tcsh freebsd copy freebsd bin/env_parallel.tcsh env_parallel.tcsh bin/env_parallel.tcsh debian copy debian bin/env_parallel.tcsh env_parallel.tcsh bin/env_parallel.tcsh hurd copy hurd bin/env_parallel.tcsh env_parallel.tcsh +bin/env_parallel.tcsh minix copy minix bin/env_parallel.tcsh env_parallel.tcsh +bin/env_parallel.tcsh minix cat: stdout: no space left on device bin/env_parallel.zsh qnx copy qnx bin/env_parallel.zsh env_parallel.zsh bin/env_parallel.zsh pidora copy pidora bin/env_parallel.zsh env_parallel.zsh +bin/env_parallel.zsh tru64 copy tru64 bin/env_parallel.zsh env_parallel.zsh +bin/env_parallel.zsh hpux-ia64 copy hpux-ia64 bin/env_parallel.zsh env_parallel.zsh bin/env_parallel.zsh raspbian copy raspbian bin/env_parallel.zsh env_parallel.zsh bin/env_parallel.zsh solaris copy solaris bin/env_parallel.zsh env_parallel.zsh -bin/env_parallel.zsh openindiana copy openindiana bin/env_parallel.zsh env_parallel.zsh bin/env_parallel.zsh aix copy aix bin/env_parallel.zsh env_parallel.zsh bin/env_parallel.zsh hpux copy hpux bin/env_parallel.zsh env_parallel.zsh bin/env_parallel.zsh suse copy suse bin/env_parallel.zsh env_parallel.zsh bin/env_parallel.zsh solaris-x86 copy solaris-x86 bin/env_parallel.zsh env_parallel.zsh bin/env_parallel.zsh mandriva copy mandriva bin/env_parallel.zsh env_parallel.zsh -bin/env_parallel.zsh ubuntu copy ubuntu bin/env_parallel.zsh env_parallel.zsh bin/env_parallel.zsh scosysv copy scosysv bin/env_parallel.zsh env_parallel.zsh bin/env_parallel.zsh unixware copy unixware bin/env_parallel.zsh env_parallel.zsh bin/env_parallel.zsh centos copy centos bin/env_parallel.zsh env_parallel.zsh @@ -287,17 +332,19 @@ bin/env_parallel.zsh openbsd copy openbsd bin/env_parallel.zsh env_parallel.zsh bin/env_parallel.zsh freebsd copy freebsd bin/env_parallel.zsh env_parallel.zsh bin/env_parallel.zsh debian copy debian bin/env_parallel.zsh env_parallel.zsh bin/env_parallel.zsh hurd copy hurd bin/env_parallel.zsh env_parallel.zsh +bin/env_parallel.zsh minix copy minix bin/env_parallel.zsh env_parallel.zsh +bin/env_parallel.zsh minix cat: stdout: no space left on device bin/parcat qnx copy qnx bin/parcat parcat bin/parcat pidora copy pidora bin/parcat parcat +bin/parcat tru64 copy tru64 bin/parcat parcat +bin/parcat hpux-ia64 copy hpux-ia64 bin/parcat parcat bin/parcat raspbian copy raspbian bin/parcat parcat bin/parcat solaris copy solaris bin/parcat parcat -bin/parcat openindiana copy openindiana bin/parcat parcat bin/parcat aix copy aix bin/parcat parcat bin/parcat hpux copy hpux bin/parcat parcat bin/parcat suse copy suse bin/parcat parcat bin/parcat solaris-x86 copy solaris-x86 bin/parcat parcat bin/parcat mandriva copy mandriva bin/parcat parcat -bin/parcat ubuntu copy ubuntu bin/parcat parcat bin/parcat scosysv copy scosysv bin/parcat parcat bin/parcat unixware copy unixware bin/parcat parcat bin/parcat centos copy centos bin/parcat parcat @@ -309,17 +356,19 @@ bin/parcat openbsd copy openbsd bin/parcat parcat bin/parcat freebsd copy freebsd bin/parcat parcat bin/parcat debian copy debian bin/parcat parcat bin/parcat hurd copy hurd bin/parcat parcat +bin/parcat minix copy minix bin/parcat parcat +bin/parcat minix cat: stdout: no space left on device bin/stdout qnx copy qnx bin/stdout stdout bin/stdout pidora copy pidora bin/stdout stdout +bin/stdout tru64 copy tru64 bin/stdout stdout +bin/stdout hpux-ia64 copy hpux-ia64 bin/stdout stdout bin/stdout raspbian copy raspbian bin/stdout stdout bin/stdout solaris copy solaris bin/stdout stdout -bin/stdout openindiana copy openindiana bin/stdout stdout bin/stdout aix copy aix bin/stdout stdout bin/stdout hpux copy hpux bin/stdout stdout bin/stdout suse copy suse bin/stdout stdout bin/stdout solaris-x86 copy solaris-x86 bin/stdout stdout bin/stdout mandriva copy mandriva bin/stdout stdout -bin/stdout ubuntu copy ubuntu bin/stdout stdout bin/stdout scosysv copy scosysv bin/stdout stdout bin/stdout unixware copy unixware bin/stdout stdout bin/stdout centos copy centos bin/stdout stdout @@ -331,8 +380,15 @@ bin/stdout openbsd copy openbsd bin/stdout stdout bin/stdout freebsd copy freebsd bin/stdout stdout bin/stdout debian copy debian bin/stdout stdout bin/stdout hurd copy hurd bin/stdout stdout +bin/stdout minix copy minix bin/stdout stdout +bin/stdout minix cat: stdout: no space left on device Done copying + echo + echo '### Works on ...' + echo + par_nonall parallel echo Works on {} ::: '`hostname`' 2>&1 + ### Works on ... aix Works on aix7 @@ -340,6 +396,7 @@ centos Works on centos.polarhome.com debian Works on debian freebsd Syntax error: Bad fd number hpux Works on hpux64 +hpux-ia64 Works on hpux-ia64 hurd Works on hurd macosx Works on macosx.polarhome.com mandriva Works on mandriva.polarhome.com @@ -356,50 +413,130 @@ redhat Works on redhat.polarhome.com scosysv Works on scosysv.polarhome.com solaris Works on solaris solaris-x86 Works on solaris-x86 +sshminix minix Works on minix suse Works on suse +tru64 Works on tru64.polarhome.com ubuntu Works on ubuntu unixware Works on unixware.polarhome.com unixware UX:sh (/bin/sh): ERROR: source: Not found -aix ### Test if empty command in process list causes problems -aix OK_with_empty_cmd -centos ### Test if empty command in process list causes problems -centos OK_with_empty_cmd -debian ### Test if empty command in process list causes problems -debian OK_with_empty_cmd -freebsd export: Illegal option -f -hpux ### Test if empty command in process list causes problems -hpux OK_with_empty_cmd -hurd ### Test if empty command in process list causes problems -hurd OK_with_empty_cmd -macosx ### Test if empty command in process list causes problems -macosx OK_with_empty_cmd -mandriva ### Test if empty command in process list causes problems -mandriva OK_with_empty_cmd -miros ### Test if empty command in process list causes problems -miros OK_with_empty_cmd -netbsd CSH/TCSH DO NOT SUPPORT newlines IN VARIABLES/FUNCTIONS. Unset test_empty_cmd test_empty_cmd test_empty_cmd test_empty_cmd -openbsd ### Test if empty command in process list causes problems -openbsd OK_with_empty_cmd -openindiana ### Test if empty command in process list causes problems -openindiana OK_with_empty_cmd -pidora ### Test if empty command in process list causes problems -pidora OK_with_empty_cmd -qnx /bin/sh: export: -f: unknown option -raspbian ### Test if empty command in process list causes problems -raspbian OK_with_empty_cmd -redhat ### Test if empty command in process list causes problems -redhat OK_with_empty_cmd -scosysv ### Test if empty command in process list causes problems -scosysv OK_with_empty_cmd -solaris ### Test if empty command in process list causes problems -solaris OK_with_empty_cmd -solaris-x86 ### Test if empty command in process list causes problems -solaris-x86 OK_with_empty_cmd -suse ### Test if empty command in process list causes problems -suse OK_with_empty_cmd -ubuntu ### Test if empty command in process list causes problems -ubuntu OK_with_empty_cmd -unixware UX:sh (/bin/sh): ERROR: -f: is not an identifier + echo + echo '### --number-of-cores/--number-of-cpus should work with no error' + echo + par_nonall 'parallel --number-of-sockets; parallel --number-of-cores' 2>&1 + par_nonall 'parallel --number-of-threads; parallel --number-of-cpus' 2>&1 + +### --number-of-cores/--number-of-cpus should work with no error + +aix 1 +aix 1 +centos 1 +centos 1 +debian 1 +debian 2 +freebsd Syntax error: Bad fd number +hpux 1 +hpux 1 +hpux-ia64 1 +hpux-ia64 1 +hurd +hurd 1 +macosx 2 +macosx 2 +mandriva 1 +mandriva 1 +miros 1 +miros 1 +netbsd 1 +netbsd 1 +openbsd 1 +openbsd 1 +pidora 1 +pidora 1 +qnx 1 +qnx 1 +qnx /bin/sh: >&/dev/null : illegal file descriptor name +qnx parallel: Warning: Cannot figure out number of cpus. Using 1. +qnx parallel: Warning: Cannot figure out number of cpus. Using 1. +raspbian 1 +raspbian 1 +redhat 1 +redhat 2 +scosysv 1 +scosysv 1 +solaris +solaris 2 +solaris-x86 +solaris-x86 1 +sshminix minix 1 +sshminix minix 1 +suse 1 +suse 1 +tru64 1 +tru64 1 +ubuntu 1 +ubuntu 2 +unixware 1 +unixware 1 +unixware UX:sh (/bin/sh): ERROR: source: Not found +aix 1 +aix 1 +centos 1 +centos 1 +debian 2 +debian 2 +freebsd Syntax error: Bad fd number +hpux 1 +hpux 1 +hpux-ia64 2 +hpux-ia64 1 +hurd 1 +hurd 1 +macosx 2 +macosx 2 +mandriva 1 +mandriva 1 +miros 1 +miros 1 +netbsd 1 +netbsd 1 +openbsd 1 +openbsd 1 +openindiana 1 +openindiana 1 +pidora 1 +pidora 1 +qnx 1 +qnx 1 +qnx /bin/sh: >&/dev/null : illegal file descriptor name +qnx parallel: Warning: Cannot figure out number of cpus. Using 1. +qnx parallel: Warning: Cannot figure out number of cpus. Using 1. +raspbian 1 +raspbian 1 +redhat 2 +redhat 2 +scosysv 1 +scosysv 1 +solaris 2 +solaris 2 +solaris-x86 1 +solaris-x86 1 +sshminix minix 1 +sshminix minix 1 +suse 1 +suse 1 +tru64 1 +tru64 1 +ubuntu 2 +ubuntu 2 +unixware 1 +unixware 1 +unixware UX:sh (/bin/sh): ERROR: source: Not found + echo + echo '### Fails if tmpdir is R/O' + echo + par_nonall "stdout parallel --tmpdir / echo ::: test read-only tmp | + perl -pe '\$exit += s:/[a-z0-9_]+.arg:/XXXXXXXX.arg:gi; \$exit += s/[0-9][0-9][0-9][0-9]/0000/gi; END { exit not \$exit }' && + echo OK readonly tmp" 2>&1 ### Fails if tmpdir is R/O @@ -413,6 +550,9 @@ debian OK readonly tmp freebsd Syntax error: Bad fd number hpux Error in tempfile() using /XXXXXXXX.arg: Could not create temp file /XXXXXXXX.arg: Permission denied at /home/t/tange/bin/parallel line 0000 hpux OK readonly tmp +hpux-ia64 Error in tempfile() using /XXXXXXXX.arg: Parent directory (/) is not writable +hpux-ia64 at /home/t/tange/bin/parallel line 0000 +hpux-ia64 OK readonly tmp hurd Error in tempfile() using template /XXXXXXXX.arg: Could not create temp file /XXXXXXXX.arg: Permission denied at /home/t/tange/bin/parallel line 0000. hurd OK readonly tmp macosx Error in tempfile() using /XXXXXXXX.arg: Could not create temp file /XXXXXXXX.arg: Permission denied at /Users/tange/bin/parallel line 0000 @@ -422,7 +562,9 @@ mandriva OK readonly tmp miros Error in tempfile() using /XXXXXXXX.arg: Parent directory (/) is not writable miros at /home/t/tange/bin/parallel line 0000 miros OK readonly tmp -netbsd Error in tempfile() using template /XXXXXXXX.arg: Could not create temp file /XXXXXXXX.arg: Permission denied at /home/t/tange/bin/parallel line 0000. +netbsd OK readonly tmp +netbsd Invalid null command. +netbsd Invalid null command. openbsd Error in tempfile() using /XXXXXXXX.arg: Could not create temp file /XXXXXXXX.arg: Permission denied at /home/t/tange/bin/parallel line 0000 openbsd OK readonly tmp openindiana Error in tempfile() using /XXXXXXXX.arg: Could not create temp file /XXXXXXXX.arg: Permission denied at /u/t/tange/bin/parallel line 0000 @@ -446,161 +588,151 @@ solaris-x86 Error in tempfile() using /XXXXXXXX.arg: Could not create temp file solaris-x86 OK readonly tmp suse Error in tempfile() using /XXXXXXXX.arg: Could not create temp file /XXXXXXXX.arg: Permission denied at /home/t/tange/bin/parallel line 0000. suse OK readonly tmp +tru64 Error in tempfile() using /XXXXXXXX.arg: Parent directory (/) is not writable +tru64 at /home/t/tange/bin/parallel line 0000 +tru64 OK readonly tmp ubuntu Error in tempfile() using template /XXXXXXXX.arg: Could not create temp file /XXXXXXXX.arg: Permission denied at /home/t/tange/bin/parallel line 0000. ubuntu OK readonly tmp unixware UX:sh (/bin/sh): ERROR: source: Not found unixware UX:env: ERROR: No such file or directory: bash + echo + echo '### Does exporting a bash function make parallel fail?' + echo 'If login shell is not bash compatible it fails' + echo + # http://zmwangx.github.io/blog/2015-11-25-bash-function-exporting-fiasco.html + par_nonall 'echo test funcA + funcA() { + cat <(echo bash only A) + } + export -f funcA; + bin/parallel funcA ::: 1' 2>&1 -### --number-of-cores/--number-of-cpus should work with no error - -aix 1 -centos 1 -debian 1 -freebsd Syntax error: Bad fd number -hpux 1 -hurd -macosx 2 -mandriva 1 -miros 1 -netbsd 1 -openbsd 1 -openindiana 1 -pidora 1 -qnx 1 -qnx /bin/sh: >&/dev/null : illegal file descriptor name -qnx parallel: Warning: Cannot figure out number of cpus. Using 1. -raspbian 1 -redhat 1 -scosysv 1 -solaris -solaris-x86 -suse 1 -ubuntu 1 -unixware 1 -unixware UX:sh (/bin/sh): ERROR: source: Not found -aix 1 -centos 1 -debian 2 -freebsd Syntax error: Bad fd number -hpux 1 -hurd 1 -macosx 2 -mandriva 1 -miros 1 -netbsd 1 -openbsd 1 -openindiana 1 -pidora 1 -qnx 1 -qnx /bin/sh: >&/dev/null : illegal file descriptor name -qnx parallel: Warning: Cannot figure out number of cpus. Using 1. -raspbian 1 -redhat 2 -scosysv 1 -solaris 2 -solaris-x86 1 -suse 1 -ubuntu 2 -unixware 1 -unixware UX:sh (/bin/sh): ERROR: source: Not found -aix 1 -centos 1 -debian 2 -freebsd Syntax error: Bad fd number -hpux 1 -hurd 1 -macosx 2 -mandriva 1 -miros 1 -netbsd 1 -openbsd 1 -openindiana 1 -pidora 1 -qnx 1 -qnx /bin/sh: >&/dev/null : illegal file descriptor name -qnx parallel: Warning: Cannot figure out number of cpus. Using 1. -raspbian 1 -redhat 2 -scosysv 1 -solaris 2 -solaris-x86 1 -suse 1 -ubuntu 2 -unixware 1 -unixware UX:sh (/bin/sh): ERROR: source: Not found -aix 1 -centos 1 -debian 2 -freebsd Syntax error: Bad fd number -hpux 1 -hurd 1 -macosx 2 -mandriva 1 -miros 1 -netbsd 1 -openbsd 1 -openindiana 1 -pidora 1 -qnx 1 -qnx /bin/sh: >&/dev/null : illegal file descriptor name -qnx parallel: Warning: Cannot figure out number of cpus. Using 1. -raspbian 1 -redhat 2 -scosysv 1 -solaris 2 -solaris-x86 1 -suse 1 -ubuntu 2 -unixware 1 -unixware UX:sh (/bin/sh): ERROR: source: Not found - -### Does exporting a bash function kill parallel +### Does exporting a bash function make parallel fail? +If login shell is not bash compatible it fails +aix test funcA aix bash only A +centos test funcA centos bash only A +debian test funcA debian bash only A -freebsd Syntax error: "(" unexpected (expecting word) +freebsd Syntax error: Bad fd number +hpux test funcA hpux bash only A +hpux-ia64 test funcA +hpux-ia64 bash only A +hurd test funcA hurd bash only A +macosx test funcA macosx bash only A +mandriva test funcA mandriva bash only A +miros test funcA miros bash only A +netbsd test funcA +netbsd Badly placed ()'s. netbsd Missing name for redirect. +netbsd }: Command not found. +netbsd export: Command not found. +netbsd funcA: Command not found. +openbsd test funcA openbsd bash only A -openindiana bash only A +pidora test funcA pidora bash only A qnx /bin/sh: syntax error: `(' unexpected +raspbian test funcA raspbian bash only A +redhat test funcA redhat bash only A +scosysv test funcA scosysv bash only A +solaris test funcA solaris bash only A +solaris-x86 test funcA solaris-x86 bash only A -suse /home/t/tange/bin/bash: func: command not found -ubuntu bash only A -unixware UX:sh (/bin/sh): ERROR: /bin/sh: Syntax error at line 1: `(' unexpected +sshminix minix test funcA +suse test funcA +suse bash only A +tru64 test funcA +tru64 bash only A +unixware test funcA +unixware UX:sh (/bin/sh): ERROR: source: Not found +unixware UX:sh (/bin/sh): ERROR: /bin/sh: Syntax error at line 3: `(' unexpected + echo + echo '### Does PARALLEL_SHELL help exporting a bash function' + echo 'If login shell is not bash compatible it should work' + echo + mkdir -p tmp/bin + cp /bin/bash tmp/bin + cd tmp + export PARALLEL_SHELL=bin/bash + par_nonall 'echo test funcB + funcB() { + cat <(echo bash only B) + } + export -f funcB + export PARALLEL_SHELL=bin/bash + bin/parallel funcB ::: 1' 2>&1 -### Does PARALLEL_SHELL help exporting a bash function not kill parallel +### Does PARALLEL_SHELL help exporting a bash function +If login shell is not bash compatible it should work +aix test funcB aix bash only B +centos test funcB centos bash only B +debian test funcB debian bash only B +freebsd test funcB freebsd bash only B +hpux test funcB +hpux bash only B +hpux-ia64 test funcB +hpux-ia64 bash only B +hurd test funcB +hurd bash only B +macosx test funcB macosx bash only B +mandriva test funcB mandriva bash only B +miros test funcB miros bash only B +netbsd test funcB netbsd bash only B +netbsd /home/t/tange/setupenv: line 9: syntax error: unexpected end of file +openbsd test funcB openbsd bash only B -openindiana bash only B +pidora test funcB pidora bash only B +qnx test funcB qnx bash only B +qnx parallel: Warning: Cannot figure out number of cpus. Using 1. +raspbian test funcB raspbian bash only B +redhat test funcB redhat bash only B +scosysv test funcB scosysv bash only B +solaris test funcB solaris bash only B +solaris-x86 test funcB solaris-x86 bash only B +sshminix minix test funcB +suse test funcB suse bash only B -ubuntu bash only B +tru64 test funcB +tru64 bash only B +unixware test funcB unixware bash only B + echo + echo '### env_parallel echo :::: <(echo OK)' + echo '(bash ksh mksh zsh only)' + echo + par_nonall 'bin/env_parallel --install && echo install-OK' 2>&1 + par_nonall 'env_parallel echo env_parallel ::: run-OK' 2>&1 + # csh on NetBSD does not support process substitution + par_nonall 'env_parallel echo reading from process substitution :::: <(echo OK)' 2>&1 | + grep -v ': /tmp/.*: No such file or directory' ### env_parallel echo :::: <(echo OK) (bash ksh mksh zsh only) @@ -650,6 +782,18 @@ hpux /home/t/tange/.profile hpux /home/t/tange/.cshrc hpux /home/t/tange/.tcshrc hpux install-OK +hpux-ia64 Installed env_parallel in: +hpux-ia64 /home/t/tange/.bashrc +hpux-ia64 /home/t/tange/.shrc +hpux-ia64 /home/t/tange/.zshenv +hpux-ia64 /home/t/tange/.config/fish/config.fish +hpux-ia64 /home/t/tange/.kshrc +hpux-ia64 /home/t/tange/.mkshrc +hpux-ia64 /home/t/tange/.profile +hpux-ia64 /home/t/tange/.cshrc +hpux-ia64 /home/t/tange/.tcshrc +hpux-ia64 install-OK +hpux-ia64 bin/env_parallel: line 33: /home/t/tange/.cshrc: Permission denied hurd Installed env_parallel in: hurd /home/t/tange/.bashrc hurd /home/t/tange/.shrc @@ -805,6 +949,17 @@ solaris-x86 /home/tange/.profile solaris-x86 /home/tange/.cshrc solaris-x86 /home/tange/.tcshrc solaris-x86 install-OK +sshminix minix Installed env_parallel in: +sshminix minix /home/t/tange/.bashrc +sshminix minix /home/t/tange/.shrc +sshminix minix /home/t/tange/.zshenv +sshminix minix /home/t/tange/.config/fish/config.fish +sshminix minix /home/t/tange/.kshrc +sshminix minix /home/t/tange/.mkshrc +sshminix minix /home/t/tange/.profile +sshminix minix /home/t/tange/.cshrc +sshminix minix /home/t/tange/.tcshrc +sshminix minix install-OK suse Installed env_parallel in: suse /home/t/tange/.bashrc suse /home/t/tange/.shrc @@ -816,6 +971,17 @@ suse /home/t/tange/.profile suse /home/t/tange/.cshrc suse /home/t/tange/.tcshrc suse install-OK +tru64 Installed env_parallel in: +tru64 /home/t/tange/.bashrc +tru64 /home/t/tange/.shrc +tru64 /home/t/tange/.zshenv +tru64 /home/t/tange/.config/fish/config.fish +tru64 /home/t/tange/.kshrc +tru64 /home/t/tange/.mkshrc +tru64 /home/t/tange/.profile +tru64 /home/t/tange/.cshrc +tru64 /home/t/tange/.tcshrc +tru64 install-OK ubuntu Installed env_parallel in: ubuntu /home/t/tange/.bashrc ubuntu /home/t/tange/.shrc @@ -834,6 +1000,7 @@ centos env_parallel run-OK debian env_parallel run-OK freebsd Syntax error: Bad fd number hpux env_parallel run-OK +hpux-ia64 env_parallel run-OK hurd env_parallel run-OK macosx env_parallel run-OK mandriva env_parallel run-OK @@ -844,34 +1011,118 @@ openindiana env_parallel run-OK pidora env_parallel run-OK qnx env_parallel run-OK qnx /bin/sh: >&/dev/null : illegal file descriptor name +qnx /bin/sh: compgen: cannot execute - No such file or directory +qnx /bin/sh: compgen: cannot execute - No such file or directory +qnx /bin/sh: compgen: cannot execute - No such file or directory qnx parallel: Warning: Cannot figure out number of cpus. Using 1. +qnx /bin/sh: shopt: cannot execute - No such file or directory raspbian env_parallel run-OK redhat env_parallel run-OK scosysv env_parallel run-OK solaris env_parallel run-OK solaris-x86 env_parallel run-OK +sshminix minix env_parallel run-OK suse env_parallel run-OK +tru64 env_parallel run-OK ubuntu env_parallel run-OK unixware env_parallel run-OK unixware UX:sh (/bin/sh): ERROR: source: Not found aix reading from process substitution OK centos reading from process substitution OK debian reading from process substitution OK +freebsd Syntax error: "(" unexpected (expecting word) hpux reading from process substitution OK +hpux-ia64 reading from process substitution OK hurd reading from process substitution OK macosx reading from process substitution OK mandriva reading from process substitution OK miros reading from process substitution OK +netbsd Missing name for redirect. +netbsd Missing name for redirect. +netbsd parallel: Error: Cannot open input file `<': No such file or directory. openbsd reading from process substitution OK openindiana reading from process substitution OK pidora reading from process substitution OK +qnx /bin/sh: syntax error: `(' unexpected raspbian reading from process substitution OK redhat reading from process substitution OK scosysv reading from process substitution OK solaris reading from process substitution OK solaris-x86 reading from process substitution OK +sshminix minix reading from process substitution OK suse reading from process substitution OK +tru64 reading from process substitution OK ubuntu reading from process substitution OK +unixware UX:sh (/bin/sh): ERROR: /bin/sh: Syntax error at line 1: `(' unexpected + # Test empty command name in process list + test_empty_cmd() { + echo '### Test if empty command name in process list causes problems' + perl -e '$0=" ";sleep 1000' & + pid=$! + parallel echo ::: OK_with_empty_cmd + kill $pid + } + export -f test_empty_cmd + export PARALLEL_SHELL=bin/bash + PARALLEL='--env test_empty_cmd' par_nonall test_empty_cmd 2>&1 +aix ### Test if empty command name in process list causes problems +aix OK_with_empty_cmd +centos ### Test if empty command name in process list causes problems +centos OK_with_empty_cmd +debian ### Test if empty command name in process list causes problems +debian OK_with_empty_cmd +freebsd ### Test if empty command name in process list causes problems +freebsd OK_with_empty_cmd +hpux ### Test if empty command name in process list causes problems +hpux OK_with_empty_cmd +hpux-ia64 ### Test if empty command name in process list causes problems +hpux-ia64 OK_with_empty_cmd +hurd ### Test if empty command name in process list causes problems +hurd OK_with_empty_cmd +macosx ### Test if empty command name in process list causes problems +macosx OK_with_empty_cmd +mandriva ### Test if empty command name in process list causes problems +mandriva OK_with_empty_cmd +miros ### Test if empty command name in process list causes problems +miros OK_with_empty_cmd +netbsd ### Test if empty command name in process list causes problems +netbsd OK_with_empty_cmd +netbsd /home/t/tange/setupenv: line 9: syntax error: unexpected end of file +openbsd ### Test if empty command name in process list causes problems +openbsd OK_with_empty_cmd +openindiana ### Test if empty command name in process list causes problems +openindiana OK_with_empty_cmd +pidora ### Test if empty command name in process list causes problems +pidora OK_with_empty_cmd +qnx ### Test if empty command name in process list causes problems +qnx OK_with_empty_cmd +qnx parallel: Warning: Cannot figure out number of cpus. Using 1. +raspbian ### Test if empty command name in process list causes problems +raspbian OK_with_empty_cmd +redhat ### Test if empty command name in process list causes problems +redhat OK_with_empty_cmd +scosysv ### Test if empty command name in process list causes problems +scosysv OK_with_empty_cmd +solaris ### Test if empty command name in process list causes problems +solaris OK_with_empty_cmd +solaris-x86 ### Test if empty command name in process list causes problems +solaris-x86 OK_with_empty_cmd +sshminix minix ### Test if empty command name in process list causes problems +suse ### Test if empty command name in process list causes problems +suse OK_with_empty_cmd +tru64 ### Test if empty command name in process list causes problems +tru64 OK_with_empty_cmd +ubuntu ### Test if empty command name in process list causes problems +ubuntu OK_with_empty_cmd +unixware ### Test if empty command name in process list causes problems +unixware OK_with_empty_cmd + echo + echo '### parset arr seq ::: 2 3 4' + echo '(bash ksh mksh zsh only)' + echo + par_nonall 'parset arr seq ::: 2 3 4; echo ${arr[*]}' 2>&1 + echo '### env_parset arr seq ::: 2 3 4' + par_nonall 'start=2;env_parset arr seq \$start ::: 2 3 4; echo ${arr[*]}' 2>&1 ### parset arr seq ::: 2 3 4 (bash ksh mksh zsh only) @@ -881,6 +1132,7 @@ centos 1 2 1 2 3 1 2 3 4 debian 1 2 1 2 3 1 2 3 4 freebsd Syntax error: Bad fd number hpux +hpux-ia64 1 2 1 2 3 1 2 3 4 hurd 1 2 1 2 3 1 2 3 4 macosx 1 2 1 2 3 1 2 3 4 mandriva 1 2 1 2 3 1 2 3 4 @@ -899,7 +1151,9 @@ redhat 1 2 1 2 3 1 2 3 4 scosysv 1 2 1 2 3 1 2 3 4 solaris 1 2 1 2 3 1 2 3 4 solaris-x86 1 2 1 2 3 1 2 3 4 +sshminix minix 1 2 1 2 3 1 2 3 4 suse 1 2 1 2 3 1 2 3 4 +tru64 1 2 1 2 3 1 2 3 4 ubuntu 1 2 1 2 3 1 2 3 4 unixware UX:sh (/bin/sh): ERROR: source: Not found unixware UX:sh (/bin/sh): ERROR: /bin/sh: Syntax error at line 1: `arr=' unexpected @@ -909,6 +1163,7 @@ centos 2 2 3 2 3 4 debian 2 2 3 2 3 4 freebsd Syntax error: Bad fd number hpux +hpux-ia64 1 2 1 2 1 2 hurd 2 2 3 2 3 4 macosx 2 2 3 2 3 4 mandriva 2 2 3 2 3 4 @@ -921,6 +1176,9 @@ openindiana 2 2 3 2 3 4 pidora 2 2 3 2 3 4 qnx qnx /bin/sh: >&/dev/null : illegal file descriptor name +qnx /bin/sh: compgen: cannot execute - No such file or directory +qnx /bin/sh: compgen: cannot execute - No such file or directory +qnx /bin/sh: compgen: cannot execute - No such file or directory qnx parallel: Warning: Cannot figure out number of cpus. Using 1. qnx /bin/sh: syntax error: `(' unexpected raspbian 2 2 3 2 3 4 @@ -928,10 +1186,20 @@ redhat 2 2 3 2 3 4 scosysv 2 2 3 2 3 4 solaris 1 2 1 2 1 2 solaris-x86 2 2 3 2 3 4 +sshminix minix 2 2 3 2 3 4 suse 2 2 3 2 3 4 +tru64 2 2 3 2 3 4 ubuntu 2 2 3 2 3 4 unixware UX:sh (/bin/sh): ERROR: source: Not found unixware UX:sh (/bin/sh): ERROR: /bin/sh: Syntax error at line 1: `arr=' unexpected + echo + echo '### parset var1,var2,var3 seq ::: 2 3 4' + echo '(bash ksh mksh zsh ash dash only)' + echo + par_nonall 'parset var1,var2,var3 seq ::: 2 3 4; echo $var1,$var2,$var3' 2>&1 + echo '### env_parset var1,var2,var3 seq ::: 2 3 4' + par_nonall 'start=2; env_parset var1,var2,var3 seq \$start ::: 2 3 4; echo $var1,$var2,$var3' 2>&1 + ### parset var1,var2,var3 seq ::: 2 3 4 (bash ksh mksh zsh ash dash only) @@ -941,6 +1209,7 @@ centos 1 2,1 2 3,1 2 3 4 debian 1 2,1 2 3,1 2 3 4 freebsd Syntax error: Bad fd number hpux ,, +hpux-ia64 1 2,1 2 3,1 2 3 4 hurd 1 2,1 2 3,1 2 3 4 macosx 1 2,1 2 3,1 2 3 4 mandriva 1 2,1 2 3,1 2 3 4 @@ -959,7 +1228,9 @@ redhat 1 2,1 2 3,1 2 3 4 scosysv 1 2,1 2 3,1 2 3 4 solaris 1 2,1 2 3,1 2 3 4 solaris-x86 1 2,1 2 3,1 2 3 4 +sshminix minix 1 2,1 2 3,1 2 3 4 suse 1 2,1 2 3,1 2 3 4 +tru64 1 2,1 2 3,1 2 3 4 ubuntu 1 2,1 2 3,1 2 3 4 unixware ,, unixware UX:sh (/bin/sh): ERROR: source: Not found @@ -969,6 +1240,7 @@ centos 2,2 3,2 3 4 debian 2,2 3,2 3 4 freebsd Syntax error: Bad fd number hpux ,, +hpux-ia64 1 2,1 2,1 2 hurd 2,2 3,2 3 4 macosx 2,2 3,2 3 4 mandriva 2,2 3,2 3 4 @@ -981,6 +1253,9 @@ openindiana 2,2 3,2 3 4 pidora 2,2 3,2 3 4 qnx ,, qnx /bin/sh: >&/dev/null : illegal file descriptor name +qnx /bin/sh: compgen: cannot execute - No such file or directory +qnx /bin/sh: compgen: cannot execute - No such file or directory +qnx /bin/sh: compgen: cannot execute - No such file or directory qnx parallel: Warning: Cannot figure out number of cpus. Using 1. qnx parallel: Warning: Cannot figure out number of cpus. Using 1. raspbian 2,2 3,2 3 4 @@ -988,7 +1263,9 @@ redhat 2,2 3,2 3 4 scosysv 2,2 3,2 3 4 solaris 1 2,1 2,1 2 solaris-x86 2,2 3,2 3 4 +sshminix minix 2,2 3,2 3 4 suse 2,2 3,2 3 4 +tru64 2,2 3,2 3 4 ubuntu 2,2 3,2 3 4 unixware 2,2 3,2 3 4 unixware UX:sh (/bin/sh): ERROR: source: Not found