From 426ea9b3728396b0cc41b248e50d4eb85931a94f Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Mon, 17 Apr 2017 16:25:58 +0200 Subject: [PATCH] env_parallel.{ash,bash,dash,ksh,sh}: Fixed bug #50815: env_parallel should warn if the environment is too big. parallel --tmux --fg: Connect to tmux socket more aggressively. parallel --joblog: Fixed bug #50781: joblog format with multiline commands. Irix's Perl does not have decode_base64 and is no longer supported. --- src/Makefile.am | 4 +- src/Makefile.in | 4 +- src/env_parallel.ash | 24 +- src/env_parallel.bash | 24 +- src/env_parallel.dash | 24 +- src/env_parallel.ksh | 23 +- src/env_parallel.pod | 13 +- src/env_parallel.sh | 24 +- src/parallel | 23 +- src/parallel.pod | 81 ++++-- src/parset.pod | 10 +- testsuite/tests-to-run/parallel-local-1s.sh | 6 +- testsuite/tests-to-run/parallel-local-3s.sh | 9 + testsuite/tests-to-run/parallel-local-ssh7.sh | 233 +++++++++++++++++- testsuite/tests-to-run/parallel-local-ssh8.sh | 4 + testsuite/tests-to-run/parallel-polarhome.sh | 7 +- testsuite/wanted-results/parallel-install | 52 +++- testsuite/wanted-results/parallel-local-0.3s | 4 +- testsuite/wanted-results/parallel-local-100s | 3 +- testsuite/wanted-results/parallel-local-10s | 7 +- testsuite/wanted-results/parallel-local-1s | 13 +- testsuite/wanted-results/parallel-local-3s | 7 + testsuite/wanted-results/parallel-local-ssh2 | 6 +- testsuite/wanted-results/parallel-local-ssh7 | 197 ++++++++++++++- testsuite/wanted-results/parallel-local-ssh8 | 16 ++ testsuite/wanted-results/parallel-local13 | 2 +- testsuite/wanted-results/parallel-local164 | 3 +- testsuite/wanted-results/parallel-local19 | 2 +- testsuite/wanted-results/parallel-local22 | 6 +- testsuite/wanted-results/parallel-local23 | 6 +- testsuite/wanted-results/parallel-local4 | 2 + testsuite/wanted-results/parallel-local9 | 2 +- testsuite/wanted-results/parallel-polarhome | 46 +++- testsuite/wanted-results/parallel-tutorial | 2 +- 34 files changed, 761 insertions(+), 128 deletions(-) diff --git a/src/Makefile.am b/src/Makefile.am index bfc68f8c..6c8bb90a 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -23,8 +23,8 @@ doc_DATA = parallel.html env_parallel.html sem.html sql.html niceload.html \ parcat.pdf parset.pdf endif -parset: parset.pod - cp parset.pod parset +parset.pod: parset + cp parset parset.pod # Build documentation file if the tool to build exists. # Otherwise: Use the distributed version diff --git a/src/Makefile.in b/src/Makefile.in index a3731cc6..bc2479aa 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -611,8 +611,8 @@ install-exec-hook: rm $(DESTDIR)$(bindir)/sem || true $(LN_S) parallel $(DESTDIR)$(bindir)/sem -parset: parset.pod - cp parset.pod parset +parset.pod: parset + cp parset parset.pod # Build documentation file if the tool to build exists. # Otherwise: Use the distributed version diff --git a/src/env_parallel.ash b/src/env_parallel.ash index 9bdd41c4..0587ef06 100755 --- a/src/env_parallel.ash +++ b/src/env_parallel.ash @@ -105,13 +105,13 @@ env_parallel() { if which parallel | grep 'no parallel in' >/dev/null; then echo 'env_parallel: Error: parallel must be in $PATH.' >&2 - return 1 + return 255 fi if which parallel >/dev/null; then true which on linux else echo 'env_parallel: Error: parallel must be in $PATH.' >&2 - return 1 + return 255 fi # Grep regexp for vars given by --env @@ -167,8 +167,20 @@ env_parallel() { unset _list_alias_BODIES unset _list_variable_VALUES unset _list_function_BODIES - `which parallel` "$@"; - _parallel_exit_CODE=$? - unset PARALLEL_ENV; - return $_parallel_exit_CODE + # Test if environment is too big + if `which true` >/dev/null ; then + `which parallel` "$@"; + _parallel_exit_CODE=$? + unset PARALLEL_ENV; + return $_parallel_exit_CODE + else + unset PARALLEL_ENV; + echo "env_parallel: Error: Your environment is too big." >&2 + echo "env_parallel: Error: Try running this in a clean environment once:" >&2 + echo "env_parallel: Error: env_parallel --record-env" >&2 + echo "env_parallel: Error: And the use '--env _'" >&2 + echo "env_parallel: Error: For details see: man env_parallel" >&2 + + return 255 + fi } diff --git a/src/env_parallel.bash b/src/env_parallel.bash index ad15e659..cdb2b1c2 100755 --- a/src/env_parallel.bash +++ b/src/env_parallel.bash @@ -99,13 +99,13 @@ env_parallel() { if which parallel | grep 'no parallel in' >/dev/null; then echo 'env_parallel: Error: parallel must be in $PATH.' >&2 - return 1 + return 255 fi if which parallel >/dev/null; then true which on linux else echo 'env_parallel: Error: parallel must be in $PATH.' >&2 - return 1 + return 255 fi # Grep regexp for vars given by --env @@ -164,10 +164,22 @@ env_parallel() { unset _list_alias_BODIES unset _list_variable_VALUES unset _list_function_BODIES - `which parallel` "$@"; - _parallel_exit_CODE=$? - unset PARALLEL_ENV; - return $_parallel_exit_CODE + # Test if environment is too big + if `which true` >/dev/null ; then + `which parallel` "$@"; + _parallel_exit_CODE=$? + unset PARALLEL_ENV; + return $_parallel_exit_CODE + else + unset PARALLEL_ENV; + echo "env_parallel: Error: Your environment is too big." >&2 + echo "env_parallel: Error: Try running this in a clean environment once:" >&2 + echo "env_parallel: Error: env_parallel --record-env" >&2 + echo "env_parallel: Error: And the use '--env _'" >&2 + echo "env_parallel: Error: For details see: man env_parallel" >&2 + + return 255 + fi } _parset() { diff --git a/src/env_parallel.dash b/src/env_parallel.dash index 40f46d91..3594b477 100755 --- a/src/env_parallel.dash +++ b/src/env_parallel.dash @@ -105,13 +105,13 @@ env_parallel() { if which parallel | grep 'no parallel in' >/dev/null; then echo 'env_parallel: Error: parallel must be in $PATH.' >&2 - return 1 + return 255 fi if which parallel >/dev/null; then true which on linux else echo 'env_parallel: Error: parallel must be in $PATH.' >&2 - return 1 + return 255 fi # Grep regexp for vars given by --env @@ -167,8 +167,20 @@ env_parallel() { unset _list_alias_BODIES unset _list_variable_VALUES unset _list_function_BODIES - `which parallel` "$@"; - _parallel_exit_CODE=$? - unset PARALLEL_ENV; - return $_parallel_exit_CODE + # Test if environment is too big + if `which true` >/dev/null ; then + `which parallel` "$@"; + _parallel_exit_CODE=$? + unset PARALLEL_ENV; + return $_parallel_exit_CODE + else + unset PARALLEL_ENV; + echo "env_parallel: Error: Your environment is too big." >&2 + echo "env_parallel: Error: Try running this in a clean environment once:" >&2 + echo "env_parallel: Error: env_parallel --record-env" >&2 + echo "env_parallel: Error: And the use '--env _'" >&2 + echo "env_parallel: Error: For details see: man env_parallel" >&2 + + return 255 + fi } diff --git a/src/env_parallel.ksh b/src/env_parallel.ksh index bb8e9d16..7901906f 100755 --- a/src/env_parallel.ksh +++ b/src/env_parallel.ksh @@ -95,13 +95,13 @@ env_parallel() { if which parallel | grep 'no parallel in' >/dev/null; then echo 'env_parallel: Error: parallel must be in $PATH.' >&2 - return 1 + return 255 fi if which parallel >/dev/null; then true which on linux else echo 'env_parallel: Error: parallel must be in $PATH.' >&2 - return 1 + return 255 fi # Grep regexp for vars given by --env @@ -157,9 +157,20 @@ env_parallel() { unset _list_alias_BODIES unset _list_variable_VALUES unset _list_function_BODIES + # Test if environment is too big + if `which true` >/dev/null ; then + `which parallel` "$@"; + _parallel_exit_CODE=$? + unset PARALLEL_ENV; + return $_parallel_exit_CODE + else + unset PARALLEL_ENV; + echo "env_parallel: Error: Your environment is too big." >&2 + echo "env_parallel: Error: Try running this in a clean environment once:" >&2 + echo "env_parallel: Error: env_parallel --record-env" >&2 + echo "env_parallel: Error: And the use '--env _'" >&2 + echo "env_parallel: Error: For details see: man env_parallel" >&2 - `which parallel` "$@"; - _parallel_exit_CODE=$? - unset PARALLEL_ENV; - return $_parallel_exit_CODE + return 255 + fi } diff --git a/src/env_parallel.pod b/src/env_parallel.pod index faac372e..4b5deb5f 100644 --- a/src/env_parallel.pod +++ b/src/env_parallel.pod @@ -29,9 +29,11 @@ B, and takes up to 15 ms to start a job. Due to the problem with environment space (see below) the recommended usage is: - # Run once to record the "empty" environment + # Record the "clean" environment (this only needs to be run once) env_parallel --record-env + # Optionally edit ~/.parallel/ignored_vars (only needed once) + # Define whatever you want to use alias myalias=echo myvar=it @@ -42,9 +44,11 @@ usage is: In B: - # Run once to record the "empty" environment + # Record the "clean" environment (this only needs to be run once) env_parallel --record-env + # Optionally edit ~/.parallel/ignored_vars (only needed once) + # Define whatever you want to use alias myalias 'echo \!* $myvar works.' set myvar=it @@ -62,7 +66,7 @@ individual shells below). But this only works if the size of the current environment is smaller than the maximal length of a command and smaller than half of the max if running remotely. E.g. The max size of Bash's command is 128 KB, so -B will fail if B is bigger than 128 +B will fail if 'B' is bigger than 128 KB. Technically the limit is in execve(1) which IPC::open3 uses. Bash completion functions are well-known for taking up well over 128 @@ -71,7 +75,8 @@ B to fail. Instead you can use B<--env> to specify which variables, arrays, aliases and functions to export as this will only export those with -the given name. +the given name. Or follow the recommended usage in shown in +DESCRIPTION. =head1 OPTIONS diff --git a/src/env_parallel.sh b/src/env_parallel.sh index 33f0f085..0c9bb377 100755 --- a/src/env_parallel.sh +++ b/src/env_parallel.sh @@ -105,13 +105,13 @@ env_parallel() { if which parallel | grep 'no parallel in' >/dev/null; then echo 'env_parallel: Error: parallel must be in $PATH.' >&2 - return 1 + return 255 fi if which parallel >/dev/null; then true which on linux else echo 'env_parallel: Error: parallel must be in $PATH.' >&2 - return 1 + return 255 fi # Grep regexp for vars given by --env @@ -167,8 +167,20 @@ env_parallel() { unset _list_alias_BODIES unset _list_variable_VALUES unset _list_function_BODIES - `which parallel` "$@"; - _parallel_exit_CODE=$? - unset PARALLEL_ENV; - return $_parallel_exit_CODE + # Test if environment is too big + if `which true` >/dev/null ; then + `which parallel` "$@"; + _parallel_exit_CODE=$? + unset PARALLEL_ENV; + return $_parallel_exit_CODE + else + unset PARALLEL_ENV; + echo "env_parallel: Error: Your environment is too big." >&2 + echo "env_parallel: Error: Try running this in a clean environment once:" >&2 + echo "env_parallel: Error: env_parallel --record-env" >&2 + echo "env_parallel: Error: And the use '--env _'" >&2 + echo "env_parallel: Error: For details see: man env_parallel" >&2 + + return 255 + fi } diff --git a/src/parallel b/src/parallel index c159a797..2a5a3f4d 100755 --- a/src/parallel +++ b/src/parallel @@ -1362,7 +1362,7 @@ sub check_invalid_option_combinations { sub init_globals { # Defaults: - $Global::version = 20170404; + $Global::version = 20170417; $Global::progname = 'parallel'; $Global::infinity = 2**31; $Global::debug = 0; @@ -5485,7 +5485,8 @@ sub compute_number_of_processes { # The child takes one process slot # It will be killed later $SIG{'TERM'} = $Global::original_sig{'TERM'}; - sleep 10101010; + # 'exec sleep' takes less RAM than sleeping in perl + exec 'sleep', 10101; exit(0); } else { # Failed to spawn @@ -5619,7 +5620,7 @@ sub compute_number_of_processes { # The system_limit is less than the wanted_processes if($system_limit < 1 and not $Global::JobQueue->empty()) { ::warning("Cannot spawn any jobs. ". - "Raising ulimit -u or /etc/security/limits.conf", + "Raising ulimit -u or 'nproc' in /etc/security/limits.conf", "or /proc/sys/kernel/pid_max may help."); ::wait_and_exit(255); } @@ -5627,7 +5628,8 @@ sub compute_number_of_processes { ::warning("Only enough file handles to run ". $system_limit. " jobs in parallel.", "Running 'parallel -j0 -N $system_limit --pipe parallel -j0' or", - "raising ulimit -n or /etc/security/limits.conf may help."); + "raising 'ulimit -n' or 'nofile' in /etc/security/limits.conf", + "or /proc/sys/fs/file-max may help."); } if($max_system_proc_reached) { ::warning("Only enough available processes to run ". @@ -8135,8 +8137,8 @@ sub print_dryrun_and_verbose { } if($opt::tmuxpane) { # Move the command into a pane in window 0 - $actual_command = 'tmux joinp -t :0 ; '. - 'tmux select-layout -t :0 tiled ; '. + $actual_command = $ENV{'PARALLEL_TMUX'}.' joinp -t :0 ; '. + $ENV{'PARALLEL_TMUX'}.' select-layout -t :0 tiled ; '. $actual_command; } my $visual_command = $self->replaced(); @@ -8147,7 +8149,7 @@ sub print_dryrun_and_verbose { } # ; causes problems # ascii 194-245 annoys tmux - $title =~ tr/[\011-\016;\302-\365]//d; + $title =~ tr/[\011-\016;\302-\365]/ /s; $title = ::shell_quote_scalar($title); my $l_act = length($actual_command); @@ -8182,8 +8184,9 @@ sub print_dryrun_and_verbose { if($opt::fg) { if(not fork) { # Run tmux in the foreground - # Give it 1 second to get started - `sleep 1;$ENV{'PARALLEL_TMUX'} -S $tmuxsocket attach`; + # Wait for the socket to appear + while (not -e $tmuxsocket) { } + `$ENV{'PARALLEL_TMUX'} -S $tmuxsocket attach`; exit; } } @@ -8690,7 +8693,7 @@ sub print_joblog { # Verbose level > 1: Print the rsync and stuff $cmd = "@command"; } - # Newlines makes it hard to parse the joblog + # Newlines make it hard to parse the joblog $cmd =~ s/\n/\0/g; print $Global::joblog join("\t", $self->seq(), $self->sshlogin()->string(), diff --git a/src/parallel.pod b/src/parallel.pod index 9f70b98a..da65af49 100644 --- a/src/parallel.pod +++ b/src/parallel.pod @@ -84,24 +84,14 @@ If I is given, GNU B solve the same tasks as B. If I is not given GNU B will behave similar to B. -The I must be an executable, a script, a composed command, or -a function. +The I must be an executable, a script, a composed command, an +alias, or a function. B: B the function first or use B. -B: Use B. +B: Use B. -B: Use B. - -B: Use B. - -B: Use B. - -B: Use B. - -B: Use B. - -B: Use B. +B: Use B. The command cannot contain the character \257 (macron: ¯). @@ -117,6 +107,10 @@ The replacement string B<{}> can be changed with B<-I>. If the command line contains no replacement strings then B<{}> will be appended to the command line. +Replacement strings are normally quoted, so special characters are not +parsed by the shell. The exception is if the command starts with a +replacement string; then the string is not quoted. + =item B<{.}> @@ -732,11 +726,11 @@ may not be used. B<--gnu> is kept for compatibility. Group output. Output from each jobs is grouped together and is only printed when the command is finished. stdout (standard output) first -followed by stderr (standard error). This takes some CPU time. In rare -situations GNU B takes up lots of CPU time and if it is -acceptable that the outputs from different commands are mixed -together, then disabling grouping with B<-u> can speedup GNU -B by a factor of 10. +followed by stderr (standard error). + +This takes in the order of 0.5ms per job and depends on the speed of +your disk for larger output. It can be disabled with B<-u>, but this +means output from different commands can get mixed. B<--group> is the default. Can be reversed with B<-u>. @@ -3045,25 +3039,40 @@ this technique with no problems. To raise the 32000 jobs limit raise =head1 EXAMPLE: Using shell variables When using shell variables you need to quote them correctly as they -may otherwise be split on spaces. +may otherwise be interpreted by the shell. Notice the difference between: - V=("My brother's 12\" records are worth <\$\$\$>"'!' Foo Bar) - parallel echo ::: ${V[@]} # This is probably not what you want + ARR=("My brother's 12\" records are worth <\$\$\$>"'!' Foo Bar) + parallel echo ::: ${ARR[@]} # This is probably not what you want and: - V=("My brother's 12\" records are worth <\$\$\$>"'!' Foo Bar) - parallel echo ::: "${V[@]}" + ARR=("My brother's 12\" records are worth <\$\$\$>"'!' Foo Bar) + parallel echo ::: "${ARR[@]}" When using variables in the actual command that contains special characters (e.g. space) you can quote them using B<'"$VAR"'> or using "'s and B<-q>: - V="Here are two " - parallel echo "'$V'" ::: spaces - parallel -q echo "$V" ::: spaces + VAR="My brother's 12\" records are worth <\$\$\$>" + parallel -q echo "$VAR" ::: '!' + export VAR + parallel echo '"$VAR"' ::: '!' + +If B<$VAR> does not contain ' then B<"'$VAR'"> will also work +(and does not need B): + + VAR="My 12\" records are worth <\$\$\$>" + parallel echo "'$VAR'" ::: '!' + +If you use them in a function you just quote as you normally would do: + + VAR="My brother's 12\" records are worth <\$\$\$>" + export VAR + myfunc() { echo "$VAR" "$1"; } + export -f myfunc + parallel myfunc ::: '!' =head1 EXAMPLE: Group output lines @@ -3961,6 +3970,24 @@ Using GNU Parallel as dir processor has the same limitations as using GNU Parallel as queue system/batch manager. +=head1 EXAMPLE: Locate the missing package + +If you have downloaded source and tried compiling it, you may have seen: + + $ ./configure + [...] + checking for something.h... no + configure: error: "libsomething not found" + +Often it is not obvious which package you should install to get that +file. Debian has `apt-file` to search for a file. `tracefile` from +https://gitlab.com/ole.tange/tangetools can tell which files a program +tried to access. In this case we are interested in one of the last +files: + + $ tracefile -un ./configure | tail | parallel -j0 apt-file search + + =head1 QUOTING GNU B is very liberal in quoting. You only need to quote diff --git a/src/parset.pod b/src/parset.pod index 9a98b419..8b428c0c 100644 --- a/src/parset.pod +++ b/src/parset.pod @@ -28,7 +28,8 @@ the names will be the destination variables. =head1 OPTIONS -Same as GNU B. +Same as GNU B, but they are put I the destination +variable. =head1 SUPPORTED SHELLS @@ -217,15 +218,14 @@ A copy of the full license is included in the file as cc-by-sa.txt. =head1 DEPENDENCIES -B uses GNU B. +B uses GNU B. =head1 SEE ALSO -B(1), +B(1), B(1), +B(1). -B(1), B(1), B(1), B(1), B(1) B(1), -B(1). =cut diff --git a/testsuite/tests-to-run/parallel-local-1s.sh b/testsuite/tests-to-run/parallel-local-1s.sh index 25f155da..9a47b171 100644 --- a/testsuite/tests-to-run/parallel-local-1s.sh +++ b/testsuite/tests-to-run/parallel-local-1s.sh @@ -201,14 +201,14 @@ par_parset() { . `which env_parallel.bash` echo 'Put output into $myarray' - parset myarray seq 3 ::: 4 5 6 + parset myarray seq 10 ::: 14 15 16 echo "${myarray[1]}" - echo 'Put output into vars $seq, $pwd, $ls' + echo 'Put output into vars "$seq, $pwd, $ls"' parset "seq pwd ls" ::: "seq 10" pwd ls echo "$seq" - echo 'Put output into vars $seq, $pwd, $ls': + echo 'Put output into vars ($seq, $pwd, $ls)': into_vars=(seq pwd ls) parset "${into_vars[*]}" ::: "seq 5" pwd ls echo "$seq" diff --git a/testsuite/tests-to-run/parallel-local-3s.sh b/testsuite/tests-to-run/parallel-local-3s.sh index cd08097b..af09b441 100644 --- a/testsuite/tests-to-run/parallel-local-3s.sh +++ b/testsuite/tests-to-run/parallel-local-3s.sh @@ -105,5 +105,14 @@ par_keeporder_roundrobin() { fi } +par_multiline_commands() { + echo 'bug #50781: joblog format with multiline commands' + seq 1 3 | parallel --jl jl --timeout 2 'sleep {}; echo {}; +echo finish {}' + seq 1 3 | parallel --jl jl --timeout 4 --retry-failed 'sleep {}; echo {}; +echo finish {}' +} + + export -f $(compgen -A function | grep par_) compgen -A function | grep par_ | sort | parallel -j6 --tag -k '{} 2>&1' diff --git a/testsuite/tests-to-run/parallel-local-ssh7.sh b/testsuite/tests-to-run/parallel-local-ssh7.sh index 7bd0ae23..3ce509b0 100755 --- a/testsuite/tests-to-run/parallel-local-ssh7.sh +++ b/testsuite/tests-to-run/parallel-local-ssh7.sh @@ -308,7 +308,7 @@ par_bash_underscore() { alias not_copied_alias="echo BAD" not_copied_func() { echo BAD; }; not_copied_var=BAD - not_copied_array=(BAD BAD BAD); + not_copied_array=(BAD BAD BAD); env_parallel --record-env; alias myecho="echo \$myvar aliases in"; myfunc() { myecho ${myarray[@]} functions $*; }; @@ -357,7 +357,7 @@ par_zsh_underscore() { alias not_copied_alias="echo BAD" not_copied_func() { echo BAD; }; not_copied_var=BAD - not_copied_array=(BAD BAD BAD); + not_copied_array=(BAD BAD BAD); env_parallel --record-env; alias myecho="echo \$myvar aliases in"; eval `cat <<"_EOS"; @@ -407,7 +407,7 @@ par_ksh_underscore() { alias not_copied_alias="echo BAD" not_copied_func() { echo BAD; }; not_copied_var=BAD - not_copied_array=(BAD BAD BAD); + not_copied_array=(BAD BAD BAD); . `which env_parallel.ksh`; env_parallel --record-env; alias myecho="echo \$myvar aliases in"; @@ -455,7 +455,7 @@ _disabled_pdksh_underscore() { alias not_copied_alias="echo BAD" not_copied_func() { echo BAD; }; not_copied_var=BAD - not_copied_array=(BAD BAD BAD); + not_copied_array=(BAD BAD BAD); . `which env_parallel.pdksh`; env_parallel --record-env; alias myecho="echo \$myvar aliases in"; @@ -827,6 +827,8 @@ _EOF par_tcsh_funky() { myscript=$(cat <<'_EOF' + # funky breaks with different LANG + setenv LANG C set myvar = "myvar works" set funky = "`perl -e 'print pack q(c*), 2..255'`" set myarray = ('' 'array_val2' '3' '' '5' ' space 6 ') @@ -935,6 +937,229 @@ _EOF ssh tcsh@lo "$myscript" 2>&1 | sort } +par_bash_environment_too_big() { + myscript=$(cat <<'_EOF' + echo 'bug #50815: env_parallel should warn if the environment is too big' + . `which env_parallel.bash`; + bigvar="$(yes | head -c 119k)" + env_parallel echo ::: OK + env_parallel -S lo echo ::: OK + + bigvar="$(yes \"| head -c 79k)" + env_parallel echo ::: OK + env_parallel -S lo echo ::: OK + + bigvar=u + eval 'bigfunc() { a="'"$(yes a| head -c 118k)"'"; };' + env_parallel echo ::: OK + env_parallel -S lo echo ::: OK + + bigvar="$(yes | head -c 120k)" + env_parallel echo ::: fail + env_parallel -S lo echo ::: fail + + bigvar="$(yes \"| head -c 80k)" + env_parallel echo ::: fail + env_parallel -S lo echo ::: fail + + bigvar=u + eval 'bigfunc() { a="'"$(yes a| head -c 119k)"'"; };' + env_parallel echo ::: fail + env_parallel -S lo echo ::: fail +_EOF + ) + ssh bash@lo "$myscript" +} + +par_dash_environment_too_big() { + myscript=$(cat <<'_EOF' + echo 'bug #50815: env_parallel should warn if the environment is too big' + . `which env_parallel.dash`; + bigvar="$(perl -e 'print "x"x130000')" + env_parallel echo ::: OK + env_parallel -S lo echo ::: OK + + bigvar="$(perl -e 'print "\""x65000')" + env_parallel echo ::: OK + env_parallel -S lo echo ::: OK + +# Functions not supported om ash +# bigvar=u +# eval 'bigfunc() { a="'"$(perl -e 'print "\""x126000')"'"; };' +# env_parallel echo ::: OK +# env_parallel -S lo echo ::: OK + + bigvar="$(perl -e 'print "x"x131000')" + env_parallel echo ::: fail + env_parallel -S lo echo ::: fail + + bigvar="$(perl -e 'print "\""x66000')" + env_parallel echo ::: fail + env_parallel -S lo echo ::: fail + +# Functions not supported om ash +# bigvar=u +# eval 'bigfunc() { a="'"$(perl -e 'print "\""x126000')"'"; };' +# env_parallel echo ::: OK +# env_parallel -S lo echo ::: OK +_EOF + ) + ssh dash@lo "$myscript" +} + +par_ash_environment_too_big() { + myscript=$(cat <<'_EOF' + echo 'bug #50815: env_parallel should warn if the environment is too big' + . `which env_parallel.ash`; + bigvar="$(perl -e 'print "x"x130000')" + env_parallel echo ::: OK + env_parallel -S lo echo ::: OK + + bigvar="$(perl -e 'print "\""x65000')" + env_parallel echo ::: OK + env_parallel -S lo echo ::: OK + +# Functions not supported in ash +# bigvar=u +# eval 'bigfunc() { a="'"$(perl -e 'print "\""x126000')"'"; };' +# env_parallel echo ::: OK +# env_parallel -S lo echo ::: OK + + bigvar="$(perl -e 'print "x"x131000')" + env_parallel echo ::: fail + env_parallel -S lo echo ::: fail + + bigvar="$(perl -e 'print "\""x66000')" + env_parallel echo ::: fail + env_parallel -S lo echo ::: fail + +# Functions not supported in ash +# bigvar=u +# eval 'bigfunc() { a="'"$(perl -e 'print "\""x126000')"'"; };' +# env_parallel echo ::: OK +# env_parallel -S lo echo ::: OK +_EOF + ) + ssh ash@lo "$myscript" +} + +par_sh_environment_too_big() { + myscript=$(cat <<'_EOF' + echo 'bug #50815: env_parallel should warn if the environment is too big' + . `which env_parallel.sh`; + bigvar="$(perl -e 'print "x"x130000')" + env_parallel echo ::: OK + env_parallel -S lo echo ::: OK + + bigvar="$(perl -e 'print "\""x65000')" + env_parallel echo ::: OK + env_parallel -S lo echo ::: OK + +# Functions not supported on GNU/Linux +# bigvar=u +# eval 'bigfunc() { a="'"$(perl -e 'print "\\\""x133000')"'"; };' +# env_parallel echo ::: OK +# env_parallel -S lo echo ::: OK + + bigvar="$(perl -e 'print "x"x131000')" + env_parallel echo ::: fail + env_parallel -S lo echo ::: fail + + bigvar="$(perl -e 'print "\""x66000')" + env_parallel echo ::: fail + env_parallel -S lo echo ::: fail + +# Functions not supported on GNU/Linux +# bigvar=u +# eval 'bigfunc() { a="'"$(perl -e 'print "\""x132000')"'"; };' +# env_parallel echo ::: fail +# env_parallel -S lo echo ::: fail +_EOF + ) + ssh sh@lo "$myscript" +} + +par_zsh_environment_too_big() { + myscript=$(cat <<'_EOF' + echo 'bug #50815: env_parallel should warn if the environment is too big' + . `which env_parallel.zsh`; + bigvar="$(perl -e 'print "x"x122000')" + env_parallel echo ::: OK + env_parallel -S lo echo ::: OK + + bigvar="$(perl -e 'print "\""x122000')" + env_parallel echo ::: OK + env_parallel -S lo echo ::: OK + + bigvar=u + eval 'bigfunc() { a="'"$(perl -e 'print "x"x122000')"'"; };' + env_parallel echo ::: OK + env_parallel -S lo echo ::: OK + + bigvar="$(perl -e 'print "x"x123000')" + env_parallel echo ::: fail + env_parallel -S lo echo ::: fail + + bigvar="$(perl -e 'print "\""x123000')" + env_parallel echo ::: fail + env_parallel -S lo echo ::: fail + + bigvar=u + eval 'bigfunc() { a="'"$(perl -e 'print "x"x123000')"'"; };' + env_parallel echo ::: fail + env_parallel -S lo echo ::: fail +_EOF + ) + ssh zsh@lo "$myscript" +} + +par_ksh_environment_too_big() { + myscript=$(cat <<'_EOF' + echo 'bug #50815: env_parallel should warn if the environment is too big' + . `which env_parallel.ksh`; + bigvar="$(perl -e 'print "x"x125000')" + env_parallel echo ::: OK + bigvar="$(perl -e 'print "x"x124000')" + env_parallel -S lo echo ::: OK + + bigvar="$(perl -e 'print "\""x124000')" + env_parallel echo ::: OK + env_parallel -S lo echo ::: OK + + bigvar=u + eval 'bigfunc() { a="'"$(perl -e 'print "\""x124000')"'"; };' + env_parallel echo ::: OK + env_parallel -S lo echo ::: OK + + bigvar="$(perl -e 'print "x"x126000')" + env_parallel echo ::: fail + env_parallel -S lo echo ::: fail + + bigvar="$(perl -e 'print "\""x125000')" + env_parallel echo ::: fail + env_parallel -S lo echo ::: fail + + bigvar=u + eval 'bigfunc() { a="'"$(perl -e 'print "\""x125000')"'"; };' + env_parallel echo ::: fail + env_parallel -S lo echo ::: fail +_EOF + ) + ssh ksh@lo "$myscript" +} + +par_fish_environment_too_big() { + echo Not implemented +} + +par_csh_environment_too_big() { + echo Not implemented +} + +par_tcsh_environment_too_big() { + echo Not implemented +} + export -f $(compgen -A function | grep par_) #compgen -A function | grep par_ | sort | parallel --delay $D -j$P --tag -k '{} 2>&1' compgen -A function | grep par_ | sort | diff --git a/testsuite/tests-to-run/parallel-local-ssh8.sh b/testsuite/tests-to-run/parallel-local-ssh8.sh index 2e86b47c..d06f528a 100644 --- a/testsuite/tests-to-run/parallel-local-ssh8.sh +++ b/testsuite/tests-to-run/parallel-local-ssh8.sh @@ -59,6 +59,10 @@ par_keeporder() { parallel --keeporder -j100% -S 1/:,2/parallel@lo -q perl -e 'sleep 1;print "job{}\n";exit({})' } +par_load_csh() { + echo '### Gave Word too long.' + parallel --load 100% -S csh@lo echo ::: a +} export -f $(compgen -A function | grep par_) #compgen -A function | grep par_ | sort | parallel --delay $D -j$P --tag -k '{} 2>&1' diff --git a/testsuite/tests-to-run/parallel-polarhome.sh b/testsuite/tests-to-run/parallel-polarhome.sh index d81e87c4..40079a76 100755 --- a/testsuite/tests-to-run/parallel-polarhome.sh +++ b/testsuite/tests-to-run/parallel-polarhome.sh @@ -2,7 +2,7 @@ # Check servers up on http://www.polarhome.com/service/status/ -P_ALL="vax freebsd solaris openbsd netbsd debian alpha aix redhat hpux ultrix minix qnx irix tru64 openindiana suse solaris-x86 mandriva ubuntu scosysv unixware dragonfly centos miros hurd raspbian macosx hpux-ia64 syllable pidora" +P_ALL="tru64 syllable pidora raspbian solaris openindiana aix hpux qnx debian-ppc suse solaris-x86 mandriva ubuntu scosysv unixware centos miros macosx redhat netbsd openbsd freebsd debian dragonfly hpux-ia64 vax alpha ultrix minix irix hurd beaglebone cubieboard2" P_NOTWORKING="vax alpha openstep" P_NOTWORKING_YET="ultrix irix" @@ -10,6 +10,7 @@ P_WORKING="tru64 syllable pidora raspbian solaris openindiana aix hpux qnx debia P_TEMPORARILY_BROKEN="minix hurd hpux-ia64 dragonfly" P="$P_WORKING" +P="$P_ALL" POLAR=`parallel -k echo {}.polarhome.com ::: $P` S_POLAR=`parallel -k echo -S 1/{}.polarhome.com ::: $P` @@ -58,12 +59,12 @@ echo echo '### Does exporting a bash function kill parallel' echo # http://zmwangx.github.io/blog/2015-11-25-bash-function-exporting-fiasco.html -parallel --onall -j0 -k --tag --timeout $TIMEOUT $S_POLAR 'func() { cat <(echo bash only A); };export -f func; bin/parallel func ::: ' ::: 1 2>&1 +parallel --onall -j0 -k --tag --timeout $TIMEOUT $S_POLAR 'func() { cat <(echo bash only A); };export -f func; bin/parallel func ::: ' ::: 1 2>&1 | sort echo echo '### Does PARALLEL_SHELL help exporting a bash function not kill parallel' echo -PARALLEL_SHELL=/bin/bash parallel --retries $RETRIES --onall -j0 -k --tag --timeout $TIMEOUT $S_POLAR 'func() { cat <(echo bash only B); };export -f func; bin/parallel func ::: ' ::: 1 2>&1 +PARALLEL_SHELL=/bin/bash parallel --retries $RETRIES --onall -j0 -k --tag --timeout $TIMEOUT $S_POLAR 'func() { cat <(echo bash only B); };export -f func; bin/parallel func ::: ' ::: 1 2>&1 | sort # Started earlier - therefore wait wait; cat /tmp/test_empty_cmd diff --git a/testsuite/wanted-results/parallel-install b/testsuite/wanted-results/parallel-install index 55cc22be..818636d6 100644 --- a/testsuite/wanted-results/parallel-install +++ b/testsuite/wanted-results/parallel-install @@ -7,6 +7,26 @@ test -d "parallel-00000000" || mkdir "parallel-00000000" (cd src && make top_distdir=../parallel-00000000 distdir=../parallel-00000000/src \ am__remove_distdir=: am__skip_length_check=: am__skip_mode_fix=: distdir) make[0]: Entering directory '~/privat/parallel/src' +pod2man --release='00000000' --center='parallel' \ + --section=1 ./parset > ./parset.1n \ +&& mv ./parset.1n ./parset.1 \ +|| echo "Warning: pod2man not found. Using old parset.1" +/bin/bash: pod2man: command not found +Warning: pod2man not found. Using old parset.1 +pod2html --title "GNU parset" ./parset > ./parset.htmln \ +&& mv ./parset.htmln ./parset.html \ +|| echo "Warning: pod2html not found. Using old parset.html" +/bin/bash: pod2html: command not found +Warning: pod2html not found. Using old parset.html +rm -f ./pod2htm* +pod2texi --output=./parset.texi ./parset \ +|| echo "Warning: pod2texi not found. Using old parset.texi" +/bin/bash: pod2texi: command not found +Warning: pod2texi not found. Using old parset.texi +pod2pdf --output-file ./parset.pdf ./parset --title "GNU parset" \ +|| echo "Warning: pod2pdf not found. Using old parset.pdf" +/bin/bash: pod2pdf: command not found +Warning: pod2pdf not found. Using old parset.pdf make[0]: Leaving directory '~/privat/parallel/src' test -n "" \ || find "parallel-00000000" -type d ! -perm -755 \ @@ -81,6 +101,12 @@ pod2man --release='00000000' --center='parallel' \ || echo "Warning: pod2man not found. Using old parallel_alternatives.7" /bin/bash: pod2man: command not found Warning: pod2man not found. Using old parallel_alternatives.7 +pod2man --release='00000000' --center='parallel' \ + --section=1 ./parset > ./parset.1n \ +&& mv ./parset.1n ./parset.1 \ +|| echo "Warning: pod2man not found. Using old parset.1" +/bin/bash: pod2man: command not found +Warning: pod2man not found. Using old parset.1 pod2html --title "GNU Parallel" ./parallel.pod > ./parallel.htmln \ && mv ./parallel.htmln ./parallel.html \ || echo "Warning: pod2html not found. Using old parallel.html" @@ -161,6 +187,10 @@ pod2texi --output=./parallel_alternatives.texi ./parallel_alternatives.pod \ || echo "Warning: pod2texi not found. Using old parallel_alternatives.texi" /bin/bash: pod2texi: command not found Warning: pod2texi not found. Using old parallel_alternatives.texi +pod2texi --output=./parset.texi ./parset \ +|| echo "Warning: pod2texi not found. Using old parset.texi" +/bin/bash: pod2texi: command not found +Warning: pod2texi not found. Using old parset.texi pod2pdf --output-file ./parallel.pdf ./parallel.pod --title "GNU Parallel" \ || echo "Warning: pod2pdf not found. Using old parallel.pdf" /bin/bash: pod2pdf: command not found @@ -193,6 +223,10 @@ pod2pdf --output-file ./parallel_alternatives.pdf ./parallel_alternatives.pod -- || echo "Warning: pod2pdf not found. Using old parallel_alternatives.pdf" /bin/bash: pod2pdf: command not found Warning: pod2pdf not found. Using old parallel_alternatives.pdf +pod2pdf --output-file ./parset.pdf ./parset --title "GNU parset" \ +|| echo "Warning: pod2pdf not found. Using old parset.pdf" +/bin/bash: pod2pdf: command not found +Warning: pod2pdf not found. Using old parset.pdf make[0]: Entering directory '/tmp/parallel-00000000/src' /bin/mkdir -p '/usr/local/bin' /usr/bin/install -c parallel sql niceload parcat env_parallel env_parallel.ash env_parallel.bash env_parallel.csh env_parallel.dash env_parallel.fish env_parallel.ksh env_parallel.pdksh env_parallel.sh env_parallel.tcsh env_parallel.zsh '/usr/local/bin' @@ -281,6 +315,10 @@ pod2texi --output=./parallel_alternatives.texi ./parallel_alternatives.pod \ || echo "Warning: pod2texi not found. Using old parallel_alternatives.texi" /bin/bash: pod2texi: command not found Warning: pod2texi not found. Using old parallel_alternatives.texi +pod2texi --output=./parset.texi ./parset \ +|| echo "Warning: pod2texi not found. Using old parset.texi" +/bin/bash: pod2texi: command not found +Warning: pod2texi not found. Using old parset.texi pod2pdf --output-file ./parallel.pdf ./parallel.pod --title "GNU Parallel" \ || echo "Warning: pod2pdf not found. Using old parallel.pdf" /bin/bash: pod2pdf: command not found @@ -313,8 +351,12 @@ pod2pdf --output-file ./parallel_alternatives.pdf ./parallel_alternatives.pod -- || echo "Warning: pod2pdf not found. Using old parallel_alternatives.pdf" /bin/bash: pod2pdf: command not found Warning: pod2pdf not found. Using old parallel_alternatives.pdf +pod2pdf --output-file ./parset.pdf ./parset --title "GNU parset" \ +|| echo "Warning: pod2pdf not found. Using old parset.pdf" +/bin/bash: pod2pdf: command not found +Warning: pod2pdf not found. Using old parset.pdf /bin/mkdir -p '/usr/local/share/doc/parallel' - /usr/bin/install -c -m 644 parallel.html env_parallel.html sem.html sql.html niceload.html parallel_tutorial.html parallel_design.html parallel_alternatives.html parcat.html parallel.texi env_parallel.texi sem.texi sql.texi niceload.texi parallel_tutorial.texi parallel_design.texi parallel_alternatives.texi parcat.texi parallel.pdf env_parallel.pdf sem.pdf sql.pdf niceload.pdf parallel_tutorial.pdf parallel_design.pdf parallel_alternatives.pdf parcat.pdf '/usr/local/share/doc/parallel' + /usr/bin/install -c -m 644 parallel.html env_parallel.html sem.html sql.html niceload.html parallel_tutorial.html parallel_design.html parallel_alternatives.html parcat.html parallel.texi env_parallel.texi sem.texi sql.texi niceload.texi parallel_tutorial.texi parallel_design.texi parallel_alternatives.texi parcat.texi parset.texi parallel.pdf env_parallel.pdf sem.pdf sql.pdf niceload.pdf parallel_tutorial.pdf parallel_design.pdf parallel_alternatives.pdf parcat.pdf parset.pdf '/usr/local/share/doc/parallel' pod2man --release='00000000' --center='parallel' \ --section=1 ./parallel.pod > ./parallel.1n \ && mv ./parallel.1n ./parallel.1 \ @@ -363,8 +405,14 @@ pod2man --release='00000000' --center='parallel' \ || echo "Warning: pod2man not found. Using old parallel_alternatives.7" /bin/bash: pod2man: command not found Warning: pod2man not found. Using old parallel_alternatives.7 +pod2man --release='00000000' --center='parallel' \ + --section=1 ./parset > ./parset.1n \ +&& mv ./parset.1n ./parset.1 \ +|| echo "Warning: pod2man not found. Using old parset.1" +/bin/bash: pod2man: command not found +Warning: pod2man not found. Using old parset.1 /bin/mkdir -p '/usr/local/share/man/man1' - /usr/bin/install -c -m 644 parallel.1 env_parallel.1 sem.1 sql.1 niceload.1 parcat.1 '/usr/local/share/man/man1' + /usr/bin/install -c -m 644 parallel.1 env_parallel.1 sem.1 sql.1 niceload.1 parcat.1 parset.1 '/usr/local/share/man/man1' /bin/mkdir -p '/usr/local/share/man/man7' /usr/bin/install -c -m 644 parallel_tutorial.7 parallel_design.7 parallel_alternatives.7 '/usr/local/share/man/man7' make[0]: Leaving directory '/tmp/parallel-00000000/src' diff --git a/testsuite/wanted-results/parallel-local-0.3s b/testsuite/wanted-results/parallel-local-0.3s index db830573..f27f4f5c 100644 --- a/testsuite/wanted-results/parallel-local-0.3s +++ b/testsuite/wanted-results/parallel-local-0.3s @@ -594,7 +594,7 @@ e echo '### test too long args' ### test too long args perl -e 'print "z"x1000000' | parallel echo 2>&1 -parallel: Error: Command line too long (1000005 >= 65528) at input 0: zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz... +parallel: Error: Command line too long (1000005 >= 65524) at input 0: zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz... perl -e 'print "z"x1000000' | xargs echo 2>&1 xargs: argument line too long (seq 1 10; perl -e 'print "z"x1000000'; seq 12 15) | stdsort parallel -j1 -km -s 10 echo @@ -861,7 +861,7 @@ echo far ### Test --show-limits (echo b; echo c; echo f) | parallel -k --show-limits echo {}ar Maximal size of command: 131049 -Maximal used size of command: 65528 +Maximal used size of command: 65524 Execution of will continue now, and it will try to read its input and run commands; if this is not what you wanted to happen, please diff --git a/testsuite/wanted-results/parallel-local-100s b/testsuite/wanted-results/parallel-local-100s index a478f736..126ce0e1 100644 --- a/testsuite/wanted-results/parallel-local-100s +++ b/testsuite/wanted-results/parallel-local-100s @@ -5,7 +5,8 @@ par_outside_file_handle_limit 2>&1 par_outside_file_handle_limit ### Test Force outside the file handle limit, 2009-02-17 Gave fork error par_outside_file_handle_limit parallel: Warning: Only enough file handles to run 999 jobs in parallel. par_outside_file_handle_limit parallel: Warning: Running 'parallel -j0 -N 999 --pipe parallel -j0' or -par_outside_file_handle_limit parallel: Warning: raising ulimit -n or /etc/security/limits.conf may help. +par_outside_file_handle_limit parallel: Warning: raising 'ulimit -n' or 'nofile' in /etc/security/limits.conf +par_outside_file_handle_limit parallel: Warning: or /proc/sys/fs/file-max may help. par_outside_file_handle_limit Start par_outside_file_handle_limit end par_over_4GB 2>&1 diff --git a/testsuite/wanted-results/parallel-local-10s b/testsuite/wanted-results/parallel-local-10s index 66eea1d5..b7e45a45 100644 --- a/testsuite/wanted-results/parallel-local-10s +++ b/testsuite/wanted-results/parallel-local-10s @@ -273,7 +273,8 @@ par_interactive par_k ### Test -k par_k parallel: Warning: Only enough file handles to run 9 jobs in parallel. par_k parallel: Warning: Running 'parallel -j0 -N 9 --pipe parallel -j0' or -par_k parallel: Warning: raising ulimit -n or /etc/security/limits.conf may help. +par_k parallel: Warning: raising 'ulimit -n' or 'nofile' in /etc/security/limits.conf +par_k parallel: Warning: or /proc/sys/fs/file-max may help. par_k parallel: Warning: No more file handles. par_k parallel: Warning: Raising ulimit -n or /etc/security/limits.conf may help. par_k begin @@ -482,10 +483,10 @@ par_kill_children_timeout 0 0 0 par_kill_children_timeout 2 par_kill_children_timeout 0 0 0 par_maxlinelen_X_I ### Test max line length -X -I -par_maxlinelen_X_I 81c0a85162c989c07f666b827a30ce52 - +par_maxlinelen_X_I 47ec7550232044dc1e7a504705a45184 - par_maxlinelen_X_I Chars per line (817788/13): 62906 par_maxlinelen_m_I ### Test max line length -m -I -par_maxlinelen_m_I 14bacad229d8b0d32be0a2339c2a6af7 - +par_maxlinelen_m_I d10d5c84fc4716b21d90afa92cf44d73 - par_maxlinelen_m_I Chars per line (697810/11): 63437 par_memleak ### Test memory consumption stays (almost) the same for 30 and 300 jobs par_memleak should give 1 == true diff --git a/testsuite/wanted-results/parallel-local-1s b/testsuite/wanted-results/parallel-local-1s index 1ac65c0c..8e759e9f 100644 --- a/testsuite/wanted-results/parallel-local-1s +++ b/testsuite/wanted-results/parallel-local-1s @@ -455,10 +455,13 @@ par_failing_compressor C=cat;false,D=cat;false par_failing_compressor parallel: Error: cat;false failed. par_parset ### test parset par_parset Put output into $myarray -par_parset 3 -par_parset 4 -par_parset 5 -par_parset Put output into vars $seq, $pwd, $ls +par_parset 10 +par_parset 11 +par_parset 12 +par_parset 13 +par_parset 14 +par_parset 15 +par_parset Put output into vars "$seq, $pwd, $ls" par_parset 1 par_parset 2 par_parset 3 @@ -469,7 +472,7 @@ par_parset 7 par_parset 8 par_parset 9 par_parset 10 -par_parset Put output into vars $seq, $pwd, $ls: +par_parset Put output into vars ($seq, $pwd, $ls): par_parset 1 par_parset 2 par_parset 3 diff --git a/testsuite/wanted-results/parallel-local-3s b/testsuite/wanted-results/parallel-local-3s index 6483d5e6..66f5c6e6 100644 --- a/testsuite/wanted-results/parallel-local-3s +++ b/testsuite/wanted-results/parallel-local-3s @@ -19,6 +19,13 @@ par_kill_term_twice parallel: bash -c sleep\ 120\ \&\ pid\=\$\!\;\ wait\ \$pid 1 par_kill_term_twice bash-+-perl---bash---sleep par_kill_term_twice `-pstree par_kill_term_twice bash---pstree +par_multiline_commands bug #50781: joblog format with multiline commands +par_multiline_commands 1 +par_multiline_commands finish 1 +par_multiline_commands 2 +par_multiline_commands finish 2 +par_multiline_commands 3 +par_multiline_commands finish 3 par_pipepart_block ### --pipepart --block -# (# < 0) par_pipepart_block 1 par_pipepart_block 2 diff --git a/testsuite/wanted-results/parallel-local-ssh2 b/testsuite/wanted-results/parallel-local-ssh2 index 828abba6..43d38e4b 100644 --- a/testsuite/wanted-results/parallel-local-ssh2 +++ b/testsuite/wanted-results/parallel-local-ssh2 @@ -90,14 +90,14 @@ par_tee_ssh 2 par_tee_ssh 3 par_tee_ssh 3 par_wd_no_such_dir ### --wd no-such-dir - csh -par_wd_no_such_dir mkdir: cannot create directory '/no-such-dir': Permission denied +par_wd_no_such_dir mkdir: cannot create directory ‘/no-such-dir’: Permission denied par_wd_no_such_dir parallel: Cannot chdir to /no-such-dir par_wd_no_such_dir Exit code 1 par_wd_no_such_dir ### --wd no-such-dir - tcsh -par_wd_no_such_dir mkdir: cannot create directory '/no-such-dir': Permission denied +par_wd_no_such_dir mkdir: cannot create directory ‘/no-such-dir’: Permission denied par_wd_no_such_dir parallel: Cannot chdir to /no-such-dir par_wd_no_such_dir Exit code 1 par_wd_no_such_dir ### --wd no-such-dir - bash -par_wd_no_such_dir mkdir: cannot create directory '/no-such-dir': Permission denied +par_wd_no_such_dir mkdir: cannot create directory ‘/no-such-dir’: Permission denied par_wd_no_such_dir parallel: Cannot chdir to /no-such-dir par_wd_no_such_dir Exit code 1 diff --git a/testsuite/wanted-results/parallel-local-ssh7 b/testsuite/wanted-results/parallel-local-ssh7 index 58270572..19990b43 100644 --- a/testsuite/wanted-results/parallel-local-ssh7 +++ b/testsuite/wanted-results/parallel-local-ssh7 @@ -1,11 +1,83 @@ ### test --env _ Both test that variables are copied, but also that they are NOT copied, if ignored +par_ash_environment_too_big bug #50815: env_parallel should warn if the environment is too big +par_ash_environment_too_big OK +par_ash_environment_too_big OK +par_ash_environment_too_big OK +par_ash_environment_too_big OK +par_ash_environment_too_big ash: 1: which: Argument list too long +par_ash_environment_too_big env_parallel: Error: Your environment is too big. +par_ash_environment_too_big env_parallel: Error: Try running this in a clean environment once: +par_ash_environment_too_big env_parallel: Error: env_parallel --record-env +par_ash_environment_too_big env_parallel: Error: And the use '--env _' +par_ash_environment_too_big env_parallel: Error: For details see: man env_parallel +par_ash_environment_too_big ash: 1: which: Argument list too long +par_ash_environment_too_big env_parallel: Error: Your environment is too big. +par_ash_environment_too_big env_parallel: Error: Try running this in a clean environment once: +par_ash_environment_too_big env_parallel: Error: env_parallel --record-env +par_ash_environment_too_big env_parallel: Error: And the use '--env _' +par_ash_environment_too_big env_parallel: Error: For details see: man env_parallel +par_ash_environment_too_big ash: 1: which: Argument list too long +par_ash_environment_too_big env_parallel: Error: Your environment is too big. +par_ash_environment_too_big env_parallel: Error: Try running this in a clean environment once: +par_ash_environment_too_big env_parallel: Error: env_parallel --record-env +par_ash_environment_too_big env_parallel: Error: And the use '--env _' +par_ash_environment_too_big env_parallel: Error: For details see: man env_parallel +par_ash_environment_too_big ash: 1: which: Argument list too long +par_ash_environment_too_big env_parallel: Error: Your environment is too big. +par_ash_environment_too_big env_parallel: Error: Try running this in a clean environment once: +par_ash_environment_too_big env_parallel: Error: env_parallel --record-env +par_ash_environment_too_big env_parallel: Error: And the use '--env _' +par_ash_environment_too_big env_parallel: Error: For details see: man env_parallel par_bash_env_parallel_fifo OK par_bash_env_parallel_fifo OK par_bash_env_parallel_fifo bug #50435: Remote fifo broke in 20150522 par_bash_env_parallel_fifo data from stdin par_bash_env_parallel_fifo data from stdin +par_bash_environment_too_big bug #50815: env_parallel should warn if the environment is too big +par_bash_environment_too_big OK +par_bash_environment_too_big OK +par_bash_environment_too_big OK +par_bash_environment_too_big OK +par_bash_environment_too_big OK +par_bash_environment_too_big OK +par_bash_environment_too_big /usr/local/bin/env_parallel.bash: line 168: /usr/bin/which: Argument list too long +par_bash_environment_too_big env_parallel: Error: Your environment is too big. +par_bash_environment_too_big env_parallel: Error: Try running this in a clean environment once: +par_bash_environment_too_big env_parallel: Error: env_parallel --record-env +par_bash_environment_too_big env_parallel: Error: And the use '--env _' +par_bash_environment_too_big env_parallel: Error: For details see: man env_parallel +par_bash_environment_too_big /usr/local/bin/env_parallel.bash: line 168: /usr/bin/which: Argument list too long +par_bash_environment_too_big env_parallel: Error: Your environment is too big. +par_bash_environment_too_big env_parallel: Error: Try running this in a clean environment once: +par_bash_environment_too_big env_parallel: Error: env_parallel --record-env +par_bash_environment_too_big env_parallel: Error: And the use '--env _' +par_bash_environment_too_big env_parallel: Error: For details see: man env_parallel +par_bash_environment_too_big /usr/local/bin/env_parallel.bash: line 168: /usr/bin/which: Argument list too long +par_bash_environment_too_big env_parallel: Error: Your environment is too big. +par_bash_environment_too_big env_parallel: Error: Try running this in a clean environment once: +par_bash_environment_too_big env_parallel: Error: env_parallel --record-env +par_bash_environment_too_big env_parallel: Error: And the use '--env _' +par_bash_environment_too_big env_parallel: Error: For details see: man env_parallel +par_bash_environment_too_big /usr/local/bin/env_parallel.bash: line 168: /usr/bin/which: Argument list too long +par_bash_environment_too_big env_parallel: Error: Your environment is too big. +par_bash_environment_too_big env_parallel: Error: Try running this in a clean environment once: +par_bash_environment_too_big env_parallel: Error: env_parallel --record-env +par_bash_environment_too_big env_parallel: Error: And the use '--env _' +par_bash_environment_too_big env_parallel: Error: For details see: man env_parallel +par_bash_environment_too_big /usr/local/bin/env_parallel.bash: line 168: /usr/bin/which: Argument list too long +par_bash_environment_too_big env_parallel: Error: Your environment is too big. +par_bash_environment_too_big env_parallel: Error: Try running this in a clean environment once: +par_bash_environment_too_big env_parallel: Error: env_parallel --record-env +par_bash_environment_too_big env_parallel: Error: And the use '--env _' +par_bash_environment_too_big env_parallel: Error: For details see: man env_parallel +par_bash_environment_too_big /usr/local/bin/env_parallel.bash: line 168: /usr/bin/which: Argument list too long +par_bash_environment_too_big env_parallel: Error: Your environment is too big. +par_bash_environment_too_big env_parallel: Error: Try running this in a clean environment once: +par_bash_environment_too_big env_parallel: Error: env_parallel --record-env +par_bash_environment_too_big env_parallel: Error: And the use '--env _' +par_bash_environment_too_big env_parallel: Error: For details see: man env_parallel par_bash_funky par_bash_funky par_bash_funky @@ -87,6 +159,7 @@ par_csh_env_parallel_fifo data from stdin par_csh_env_parallel_fifo OK par_csh_env_parallel_fifo data from stdin par_csh_env_parallel_fifo OK +par_csh_environment_too_big Not implemented par_csh_funky 3 arg alias_works par_csh_funky myvar works par_csh_funky space special chars problem @@ -178,6 +251,35 @@ par_csh_underscore OK ^^^^^^^^^^^^^^^^^ if no myecho par_csh_underscore myecho: Command not found. par_csh_underscore OK ^^^^^^^^^^^^^^^^^ if no myecho par_csh_underscore Connection to lo closed. +par_dash_environment_too_big bug #50815: env_parallel should warn if the environment is too big +par_dash_environment_too_big OK +par_dash_environment_too_big OK +par_dash_environment_too_big OK +par_dash_environment_too_big OK +par_dash_environment_too_big dash: 1: which: Argument list too long +par_dash_environment_too_big env_parallel: Error: Your environment is too big. +par_dash_environment_too_big env_parallel: Error: Try running this in a clean environment once: +par_dash_environment_too_big env_parallel: Error: env_parallel --record-env +par_dash_environment_too_big env_parallel: Error: And the use '--env _' +par_dash_environment_too_big env_parallel: Error: For details see: man env_parallel +par_dash_environment_too_big dash: 1: which: Argument list too long +par_dash_environment_too_big env_parallel: Error: Your environment is too big. +par_dash_environment_too_big env_parallel: Error: Try running this in a clean environment once: +par_dash_environment_too_big env_parallel: Error: env_parallel --record-env +par_dash_environment_too_big env_parallel: Error: And the use '--env _' +par_dash_environment_too_big env_parallel: Error: For details see: man env_parallel +par_dash_environment_too_big dash: 1: which: Argument list too long +par_dash_environment_too_big env_parallel: Error: Your environment is too big. +par_dash_environment_too_big env_parallel: Error: Try running this in a clean environment once: +par_dash_environment_too_big env_parallel: Error: env_parallel --record-env +par_dash_environment_too_big env_parallel: Error: And the use '--env _' +par_dash_environment_too_big env_parallel: Error: For details see: man env_parallel +par_dash_environment_too_big dash: 1: which: Argument list too long +par_dash_environment_too_big env_parallel: Error: Your environment is too big. +par_dash_environment_too_big env_parallel: Error: Try running this in a clean environment once: +par_dash_environment_too_big env_parallel: Error: env_parallel --record-env +par_dash_environment_too_big env_parallel: Error: And the use '--env _' +par_dash_environment_too_big env_parallel: Error: For details see: man env_parallel par_fish_env_parallel_fifo bug #50435: Remote fifo broke in 20150522 par_fish_env_parallel_fifo data from stdin par_fish_env_parallel_fifo OK @@ -186,6 +288,7 @@ par_fish_env_parallel_fifo OK par_fish_env_parallel_fifo $? is not the exit status. In fish, please use $status. par_fish_env_parallel_fifo fish: cat $PARALLEL_TMP; and echo $OK;perl -e '$bash=shift;$csh=shift;for(@ARGV){unlink;rmdir;}if($bash=~s/h//){exit$bash;}exit$csh;' "$?h" "$status" $PARALLEL_TMP par_fish_env_parallel_fifo ^ +par_fish_environment_too_big Not implemented par_fish_funky env_parallel: Warning: ASCII value 1 in variables is not supported par_fish_funky 3 arg alias_works par_fish_funky env_parallel: Warning: ASCII value 1 in variables is not supported @@ -264,15 +367,15 @@ par_fish_underscore aliases and arrays in functions work par_fish_underscore aliases functions work par_fish_underscore aliases functions work par_fish_underscore ^ -par_fish_underscore in function 'myfunc' +par_fish_underscore in function “myfunc” par_fish_underscore called on standard input -par_fish_underscore with parameter list 'work' +par_fish_underscore with parameter list “work” par_fish_underscore par_fish_underscore OK if ^^^^^^^^^^^^^^^^^ no myecho par_fish_underscore ^ -par_fish_underscore in function 'myfunc' +par_fish_underscore in function “myfunc” par_fish_underscore called on standard input -par_fish_underscore with parameter list 'work' +par_fish_underscore with parameter list “work” par_fish_underscore par_fish_underscore OK if ^^^^^^^^^^^^^^^^^ no myecho par_fish_underscore ^ @@ -284,6 +387,49 @@ par_ksh_env_parallel_fifo OK par_ksh_env_parallel_fifo bug #50435: Remote fifo broke in 20150522 par_ksh_env_parallel_fifo data from stdin par_ksh_env_parallel_fifo data from stdin +par_ksh_environment_too_big bug #50815: env_parallel should warn if the environment is too big +par_ksh_environment_too_big OK +par_ksh_environment_too_big OK +par_ksh_environment_too_big OK +par_ksh_environment_too_big OK +par_ksh_environment_too_big OK +par_ksh_environment_too_big OK +par_ksh_environment_too_big ksh[161]: which: /usr/bin/which: cannot execute [Argument list too long] +par_ksh_environment_too_big env_parallel: Error: Your environment is too big. +par_ksh_environment_too_big env_parallel: Error: Try running this in a clean environment once: +par_ksh_environment_too_big env_parallel: Error: env_parallel --record-env +par_ksh_environment_too_big env_parallel: Error: And the use '--env _' +par_ksh_environment_too_big env_parallel: Error: For details see: man env_parallel +par_ksh_environment_too_big ksh[161]: which: /usr/bin/which: cannot execute [Argument list too long] +par_ksh_environment_too_big env_parallel: Error: Your environment is too big. +par_ksh_environment_too_big env_parallel: Error: Try running this in a clean environment once: +par_ksh_environment_too_big env_parallel: Error: env_parallel --record-env +par_ksh_environment_too_big env_parallel: Error: And the use '--env _' +par_ksh_environment_too_big env_parallel: Error: For details see: man env_parallel +par_ksh_environment_too_big ksh[161]: which: /usr/bin/which: cannot execute [Argument list too long] +par_ksh_environment_too_big env_parallel: Error: Your environment is too big. +par_ksh_environment_too_big env_parallel: Error: Try running this in a clean environment once: +par_ksh_environment_too_big env_parallel: Error: env_parallel --record-env +par_ksh_environment_too_big env_parallel: Error: And the use '--env _' +par_ksh_environment_too_big env_parallel: Error: For details see: man env_parallel +par_ksh_environment_too_big ksh[161]: which: /usr/bin/which: cannot execute [Argument list too long] +par_ksh_environment_too_big env_parallel: Error: Your environment is too big. +par_ksh_environment_too_big env_parallel: Error: Try running this in a clean environment once: +par_ksh_environment_too_big env_parallel: Error: env_parallel --record-env +par_ksh_environment_too_big env_parallel: Error: And the use '--env _' +par_ksh_environment_too_big env_parallel: Error: For details see: man env_parallel +par_ksh_environment_too_big ksh[161]: which: /usr/bin/which: cannot execute [Argument list too long] +par_ksh_environment_too_big env_parallel: Error: Your environment is too big. +par_ksh_environment_too_big env_parallel: Error: Try running this in a clean environment once: +par_ksh_environment_too_big env_parallel: Error: env_parallel --record-env +par_ksh_environment_too_big env_parallel: Error: And the use '--env _' +par_ksh_environment_too_big env_parallel: Error: For details see: man env_parallel +par_ksh_environment_too_big ksh[161]: which: /usr/bin/which: cannot execute [Argument list too long] +par_ksh_environment_too_big env_parallel: Error: Your environment is too big. +par_ksh_environment_too_big env_parallel: Error: Try running this in a clean environment once: +par_ksh_environment_too_big env_parallel: Error: env_parallel --record-env +par_ksh_environment_too_big env_parallel: Error: And the use '--env _' +par_ksh_environment_too_big env_parallel: Error: For details see: man env_parallel par_ksh_funky par_ksh_funky \\\\\\\\ par_ksh_funky  !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~-funky @@ -354,11 +500,41 @@ par_ksh_underscore /usr/bin/ksh: line 2: myfunc: not found par_ksh_underscore OK if no myfunc ^^^^^^^^^^^^^^^^^ par_ksh_underscore /usr/bin/ksh: line 2: myfunc: not found par_ksh_underscore OK if no myfunc ^^^^^^^^^^^^^^^^^ +par_sh_environment_too_big bug #50815: env_parallel should warn if the environment is too big +par_sh_environment_too_big OK +par_sh_environment_too_big OK +par_sh_environment_too_big OK +par_sh_environment_too_big OK +par_sh_environment_too_big sh: 1: which: Argument list too long +par_sh_environment_too_big env_parallel: Error: Your environment is too big. +par_sh_environment_too_big env_parallel: Error: Try running this in a clean environment once: +par_sh_environment_too_big env_parallel: Error: env_parallel --record-env +par_sh_environment_too_big env_parallel: Error: And the use '--env _' +par_sh_environment_too_big env_parallel: Error: For details see: man env_parallel +par_sh_environment_too_big sh: 1: which: Argument list too long +par_sh_environment_too_big env_parallel: Error: Your environment is too big. +par_sh_environment_too_big env_parallel: Error: Try running this in a clean environment once: +par_sh_environment_too_big env_parallel: Error: env_parallel --record-env +par_sh_environment_too_big env_parallel: Error: And the use '--env _' +par_sh_environment_too_big env_parallel: Error: For details see: man env_parallel +par_sh_environment_too_big sh: 1: which: Argument list too long +par_sh_environment_too_big env_parallel: Error: Your environment is too big. +par_sh_environment_too_big env_parallel: Error: Try running this in a clean environment once: +par_sh_environment_too_big env_parallel: Error: env_parallel --record-env +par_sh_environment_too_big env_parallel: Error: And the use '--env _' +par_sh_environment_too_big env_parallel: Error: For details see: man env_parallel +par_sh_environment_too_big sh: 1: which: Argument list too long +par_sh_environment_too_big env_parallel: Error: Your environment is too big. +par_sh_environment_too_big env_parallel: Error: Try running this in a clean environment once: +par_sh_environment_too_big env_parallel: Error: env_parallel --record-env +par_sh_environment_too_big env_parallel: Error: And the use '--env _' +par_sh_environment_too_big env_parallel: Error: For details see: man env_parallel par_tcsh_env_parallel_fifo OK par_tcsh_env_parallel_fifo OK par_tcsh_env_parallel_fifo bug #50435: Remote fifo broke in 20150522 par_tcsh_env_parallel_fifo data from stdin par_tcsh_env_parallel_fifo data from stdin +par_tcsh_environment_too_big Not implemented par_tcsh_funky par_tcsh_funky par_tcsh_funky @@ -461,6 +637,19 @@ par_zsh_env_parallel_fifo OK par_zsh_env_parallel_fifo bug #50435: Remote fifo broke in 20150522 par_zsh_env_parallel_fifo data from stdin par_zsh_env_parallel_fifo data from stdin +par_zsh_environment_too_big bug #50815: env_parallel should warn if the environment is too big +par_zsh_environment_too_big OK +par_zsh_environment_too_big OK +par_zsh_environment_too_big OK +par_zsh_environment_too_big OK +par_zsh_environment_too_big OK +par_zsh_environment_too_big OK +par_zsh_environment_too_big env_parallel:132: argument list too long: /usr/local/bin/parallel +par_zsh_environment_too_big env_parallel:132: argument list too long: /usr/local/bin/parallel +par_zsh_environment_too_big env_parallel:132: argument list too long: /usr/local/bin/parallel +par_zsh_environment_too_big env_parallel:132: argument list too long: /usr/local/bin/parallel +par_zsh_environment_too_big env_parallel:132: argument list too long: /usr/local/bin/parallel +par_zsh_environment_too_big env_parallel:132: argument list too long: /usr/local/bin/parallel par_zsh_funky par_zsh_funky \\\\\\\\ par_zsh_funky  diff --git a/testsuite/wanted-results/parallel-local-ssh8 b/testsuite/wanted-results/parallel-local-ssh8 index c8b51d71..e535bd3e 100644 --- a/testsuite/wanted-results/parallel-local-ssh8 +++ b/testsuite/wanted-results/parallel-local-ssh8 @@ -6,13 +6,29 @@ par_keeporder ### Test --keeporder par_keeporder job0 par_keeporder job1 par_keeporder job2 +par_load_csh ### Gave Word too long. +par_load_csh a par_path_remote_bash bug #47695: How to set $PATH on remote? Bash +par_path_remote_bash +par_path_remote_bash * Documentation: https://help.ubuntu.com +par_path_remote_bash * Management: https://landscape.canonical.com +par_path_remote_bash * Support: https://ubuntu.com/advantage +par_path_remote_bash +par_path_remote_bash 0 updates are security updates. +par_path_remote_bash par_path_remote_bash BASH Path before: /bin:/usr/bin with no parallel par_path_remote_bash -bash: line 2: parallel: command not found par_path_remote_bash ^^^^^^^^ Not found is OK par_path_remote_bash /bin:/usr/bin:/tmp OK par_path_remote_bash par_path_remote_csh bug #47695: How to set $PATH on remote? csh +par_path_remote_csh +par_path_remote_csh * Documentation: https://help.ubuntu.com +par_path_remote_csh * Management: https://landscape.canonical.com +par_path_remote_csh * Support: https://ubuntu.com/advantage +par_path_remote_csh +par_path_remote_csh 0 updates are security updates. +par_path_remote_csh par_path_remote_csh Warning: no access to tty (Bad file descriptor). par_path_remote_csh Thus no job control in this shell. par_path_remote_csh CSH Path before: /bin:/usr/bin with no parallel diff --git a/testsuite/wanted-results/parallel-local13 b/testsuite/wanted-results/parallel-local13 index 2d82d1e9..73fc9036 100644 --- a/testsuite/wanted-results/parallel-local13 +++ b/testsuite/wanted-results/parallel-local13 @@ -458,7 +458,7 @@ echo '### true < 32767-ys.xi' ### true < 32767-ys.xi stdout xargs true < 32767-ys.xi stdout parallel -k true < 32767-ys.xi -parallel: Error: Command line too long (98306 >= 65528) at input 0: y y y y y y y y y y y y y y y y y y y y y y y y y ... +parallel: Error: Command line too long (98306 >= 65524) at input 0: y y y y y y y y y y y y y y y y y y y y y y y y y ... echo '### true < 16383-ys.xi' ### true < 16383-ys.xi stdout xargs true < 16383-ys.xi diff --git a/testsuite/wanted-results/parallel-local164 b/testsuite/wanted-results/parallel-local164 index f272a4d8..17498aa1 100644 --- a/testsuite/wanted-results/parallel-local164 +++ b/testsuite/wanted-results/parallel-local164 @@ -339,7 +339,8 @@ bug #38439: "open files" with --files --pipe blocks after a while 21 of 21 parallel: Warning: Only enough file handles to run 1 jobs in parallel. parallel: Warning: Running 'parallel -j0 -N 1 --pipe parallel -j0' or -parallel: Warning: raising ulimit -n or /etc/security/limits.conf may help. +parallel: Warning: raising 'ulimit -n' or 'nofile' in /etc/security/limits.conf +parallel: Warning: or /proc/sys/fs/file-max may help. echo 'bug #34241: --pipe should not spawn unneeded processes - part 2' bug #34241: --pipe should not spawn unneeded processes - part 2 seq 500 | parallel --tmpdir . -j10 --pipe --block 1k --files wc >/dev/null; ls *.par | wc -l; rm *.par; seq 500 | parallel --tmpdir . -j10 --pipe --block 1k --files --dry-run wc >/dev/null; echo No .par should exist; stdout ls *.par diff --git a/testsuite/wanted-results/parallel-local19 b/testsuite/wanted-results/parallel-local19 index fe9f911a..6144b49e 100644 --- a/testsuite/wanted-results/parallel-local19 +++ b/testsuite/wanted-results/parallel-local19 @@ -111,7 +111,7 @@ echo '### Test -m with 60000 args'; seq 1 60000 | perl -pe 's/$/.gif/' | par 20 179960 1286702 echo '### Test -X with 60000 args'; seq 1 60000 | perl -pe 's/$/.gif/' | parallel -j1 -kX echo a{}b{.}c{.} | tee >(wc; sleep 1) >(md5sum; sleep 1) >/dev/null; wait; sleep 1 ### Test -X with 60000 args -97715240fa65309902932877d24273db - +8646e0f7fd5f8795e4b1a831ede50e85 - 21 60000 1346682 echo '### Test -X with 60000 args and 5 expansions' ### Test -X with 60000 args and 5 expansions diff --git a/testsuite/wanted-results/parallel-local22 b/testsuite/wanted-results/parallel-local22 index b83255a0..c0b7cc29 100644 --- a/testsuite/wanted-results/parallel-local22 +++ b/testsuite/wanted-results/parallel-local22 @@ -1,16 +1,16 @@ echo '### Test of xargs -m command lines > 130k'; seq 1 60000 | parallel -m -j1 echo a{}b{}c | tee >(wc >/tmp/awc$$) >(sort | md5sum) >/tmp/a$$; wait; CHAR=$(cat /tmp/a$$ | wc -c); LINES=$(cat /tmp/a$$ | wc -l); echo "Chars per line:" $(echo "$CHAR/$LINES" | bc); cat /tmp/awc$$; rm /tmp/a$$ /tmp/awc$$ ### Test of xargs -m command lines > 130k -14bacad229d8b0d32be0a2339c2a6af7 - +d10d5c84fc4716b21d90afa92cf44d73 - Chars per line: 63437 11 119989 697810 echo '### Test of xargs -X command lines > 130k'; seq 1 60000 | parallel -X -j1 echo a{}b{}c | tee >(wc >/tmp/bwc$$) >(sort | (sleep 1; md5sum)) >/tmp/b$$; wait; CHAR=$(cat /tmp/b$$ | wc -c); LINES=$(cat /tmp/b$$ | wc -l); echo "Chars per line:" $(echo "$CHAR/$LINES" | bc); cat /tmp/bwc$$; rm /tmp/b$$ /tmp/bwc$$ ### Test of xargs -X command lines > 130k -81c0a85162c989c07f666b827a30ce52 - +47ec7550232044dc1e7a504705a45184 - Chars per line: 62906 13 60000 817788 echo '### Test of xargs -m command lines > 130k'; seq 1 60000 | parallel -k -j1 -m echo | md5sum ### Test of xargs -m command lines > 130k -4c821f9aa0e42e53b4b5ff8cc809490b - +4d3752b433dc963c466adc859e8d793f - echo '### This causes problems if we kill child processes'; # 2>/dev/null to avoid parallel: Warning: Starting 45 processes took > 2 sec. ### This causes problems if we kill child processes seq 2 40 | parallel -j 0 seq 1 10 2>/dev/null | sort | md5sum diff --git a/testsuite/wanted-results/parallel-local23 b/testsuite/wanted-results/parallel-local23 index 1e0b7b82..d8bbfd7e 100644 --- a/testsuite/wanted-results/parallel-local23 +++ b/testsuite/wanted-results/parallel-local23 @@ -74,9 +74,9 @@ please cite as described in 'parallel --citation'. echo '### bug #39787: --xargs broken' ### bug #39787: --xargs broken nice perl -e 'for(1..30000){print "$_\n"}' | $NICEPAR --xargs -k echo | perl -ne 'print length $_,"\n"' -65520 -65520 -37854 +65514 +65514 +37866 echo '### --delay should grow by 3 sec per arg' ### --delay should grow by 3 sec per arg stdout /usr/bin/time -f %e parallel --delay 3 true ::: 1 2 | perl -ne '$_ >= 3 and $_ <= 8 and print "OK\n"' diff --git a/testsuite/wanted-results/parallel-local4 b/testsuite/wanted-results/parallel-local4 index ab9f72d3..f888b031 100644 --- a/testsuite/wanted-results/parallel-local4 +++ b/testsuite/wanted-results/parallel-local4 @@ -3,12 +3,14 @@ it should burn 1.9 CPU seconds, but no more than that The 5 second sleep will make it be killed by timeout when it fgs 1024 SHA256:SXgag2Z2L91JsrT5WjNBcARD1EpyCCj4JctVJ6Zpzm0 ~/.ssh/id_dsa (DSA) 8192 SHA256:lYn04AefJq/5r0e4FftceviJ7JVnq9NGKY3CW9XMpO8 ~/.ssh/id_rsa (RSA) +1024 SHA256:R8Dy6OsG3j646iO2KeSkCAmBiR5s2720ZYx695+0JHw ~/.ssh/id_rsa_1024 (RSA) 4096 SHA256:jUQ9ysfprs7UOckttjjVb+j3qikUmKWDEWC+eEJkbDQ ~/.ssh/id_rsa_openindiana (RSA) stdout /usr/bin/time -f CPUTIME=%U parallel --timeout 5 burnP6 ::: 1 | grep -q CPUTIME=1 Zero=OK 0 1024 SHA256:SXgag2Z2L91JsrT5WjNBcARD1EpyCCj4JctVJ6Zpzm0 ~/.ssh/id_dsa (DSA) 8192 SHA256:lYn04AefJq/5r0e4FftceviJ7JVnq9NGKY3CW9XMpO8 ~/.ssh/id_rsa (RSA) +1024 SHA256:R8Dy6OsG3j646iO2KeSkCAmBiR5s2720ZYx695+0JHw ~/.ssh/id_rsa_1024 (RSA) 4096 SHA256:jUQ9ysfprs7UOckttjjVb+j3qikUmKWDEWC+eEJkbDQ ~/.ssh/id_rsa_openindiana (RSA) echo 1 | stdout /usr/bin/time -f CPUTIME=%U parallel --timeout 5 burnP6 | grep -q CPUTIME=1 diff --git a/testsuite/wanted-results/parallel-local9 b/testsuite/wanted-results/parallel-local9 index d5ad4865..c0c10e67 100644 --- a/testsuite/wanted-results/parallel-local9 +++ b/testsuite/wanted-results/parallel-local9 @@ -400,7 +400,7 @@ echo "### BUG: empty lines with --show-limit" ### BUG: empty lines with --show-limit echo | $NICEPAR --show-limits Maximal size of command: 131049 -Maximal used size of command: 65528 +Maximal used size of command: 65524 Execution of will continue now, and it will try to read its input and run commands; if this is not what you wanted to happen, please diff --git a/testsuite/wanted-results/parallel-polarhome b/testsuite/wanted-results/parallel-polarhome index 7c76b668..aa144a6c 100644 --- a/testsuite/wanted-results/parallel-polarhome +++ b/testsuite/wanted-results/parallel-polarhome @@ -8,7 +8,7 @@ tru64.polarhome.com at bin/parallel line 0000 tru64.polarhome.com OK copy_and_test syllable.polarhome.com syllable.polarhome.com ### Run the test on syllable.polarhome.com -syllable.polarhome.com chmod: changing permissions of `bin/p.tmp': Function not implemented +syllable.polarhome.com Connection to gate.polarhome.com closed by remote host. copy_and_test pidora.polarhome.com pidora.polarhome.com ### Run the test on pidora.polarhome.com pidora.polarhome.com Works on pidora.polarhome.com @@ -42,10 +42,6 @@ hpux.polarhome.com Error in tempfile() using /XXXXXXXX.arg: Could not create tem hpux.polarhome.com OK copy_and_test qnx.polarhome.com qnx.polarhome.com ### Run the test on qnx.polarhome.com -qnx.polarhome.com parallel: Warning: Cannot figure out number of CPU cores. Using 1. -qnx.polarhome.com Works on qnx.polarhome.com -qnx.polarhome.com Error in tempfile() using /XXXXXXXX.arg: Could not create temp file /XXXXXXXX.arg: Permission denied at bin/parallel line 0000 -qnx.polarhome.com OK copy_and_test debian-ppc.polarhome.com debian-ppc.polarhome.com ### Run the test on debian-ppc.polarhome.com debian-ppc.polarhome.com Works on debian-ppc.polarhome.com @@ -124,6 +120,34 @@ debian.polarhome.com ### Run the test on debian.polarhome.com debian.polarhome.com Works on debian.polarhome.com debian.polarhome.com Error in tempfile() using template /XXXXXXXX.arg: Could not create temp file /XXXXXXXX.arg: Permission denied at bin/parallel line 0000. debian.polarhome.com OK +copy_and_test dragonfly.polarhome.com +dragonfly.polarhome.com ### Run the test on dragonfly.polarhome.com +dragonfly.polarhome.com Works on dragonfly.polarhome.com +dragonfly.polarhome.com Error in tempfile() using /XXXXXXXX.arg: Could not create temp file /XXXXXXXX.arg: Permission denied at bin/parallel line 0000 +dragonfly.polarhome.com OK +copy_and_test hpux-ia64.polarhome.com +hpux-ia64.polarhome.com ### Run the test on hpux-ia64.polarhome.com +hpux-ia64.polarhome.com Works on hpux-ia64.polarhome.com +hpux-ia64.polarhome.com Error in tempfile() using /XXXXXXXX.arg: Parent directory (/) is not writable +hpux-ia64.polarhome.com at bin/parallel line 0000 +hpux-ia64.polarhome.com OK +copy_and_test vax.polarhome.com +vax.polarhome.com ### Run the test on vax.polarhome.com +copy_and_test alpha.polarhome.com +alpha.polarhome.com ### Run the test on alpha.polarhome.com +copy_and_test ultrix.polarhome.com +ultrix.polarhome.com ### Run the test on ultrix.polarhome.com +copy_and_test minix.polarhome.com +minix.polarhome.com ### Run the test on minix.polarhome.com +copy_and_test irix.polarhome.com +irix.polarhome.com ### Run the test on irix.polarhome.com +irix.polarhome.com Unknown open() mode '>&' at bin/parallel line 2214. +copy_and_test hurd.polarhome.com +hurd.polarhome.com ### Run the test on hurd.polarhome.com +copy_and_test beaglebone.polarhome.com +beaglebone.polarhome.com ### Run the test on beaglebone.polarhome.com +copy_and_test cubieboard2.polarhome.com +cubieboard2.polarhome.com ### Run the test on cubieboard2.polarhome.com ### Test remote wrapper working on all platforms @@ -131,7 +155,9 @@ aix7 centos.polarhome.com debian-ppc debian +dragonfly.polarhome.com freebsd.polarhome.com +hpux-ia64 hpux64 macosx mandriva.polarhome.com @@ -140,7 +166,6 @@ netbsd.polarhome.com openbsd.polarhome.com openindiana pidora -qnx raspbian redhat.polarhome.com scosysv.polarhome.com @@ -157,8 +182,12 @@ aix.polarhome.com bash only A centos.polarhome.com bash only A debian-ppc.polarhome.com bash only A debian.polarhome.com bash only A +dragonfly.polarhome.com Syntax error: "(" unexpected freebsd.polarhome.com Syntax error: "(" unexpected (expecting word) +hpux-ia64.polarhome.com bash only A hpux.polarhome.com bash only A +irix.polarhome.com Undefined subroutine &main::decode_base64 called at -e line 1. +irix.polarhome.com open3: exec of bzip2 -dc failed at -e line 1 macosx.polarhome.com bash only A mandriva.polarhome.com bash only A miros.polarhome.com bash only A @@ -166,13 +195,13 @@ netbsd.polarhome.com Missing name for redirect. openbsd.polarhome.com bash only A openindiana.polarhome.com bash only A pidora.polarhome.com bash only A -qnx.polarhome.com /bin/sh: syntax error: `(' unexpected raspbian.polarhome.com bash only A redhat.polarhome.com bash only A scosysv.polarhome.com bash only A solaris-x86.polarhome.com bash only A solaris.polarhome.com bash only A suse.polarhome.com bash only A +syllable.polarhome.com Connection to gate.polarhome.com closed by remote host. tru64.polarhome.com bash only A ubuntu.polarhome.com bash only A unixware.polarhome.com UX:sh (/bin/sh): ERROR: /bin/sh: Syntax error at line 1: `(' unexpected @@ -184,6 +213,8 @@ centos.polarhome.com bash only B debian-ppc.polarhome.com bash only B debian.polarhome.com bash only B freebsd.polarhome.com bash only B +irix.polarhome.com Undefined subroutine &main::decode_base64 called at -e line 1. +irix.polarhome.com open3: exec of bzip2 -dc failed at -e line 1 macosx.polarhome.com bash only B mandriva.polarhome.com bash only B miros.polarhome.com bash only B @@ -197,6 +228,7 @@ scosysv.polarhome.com bash only B solaris-x86.polarhome.com bash only B solaris.polarhome.com bash only B suse.polarhome.com bash only B +syllable.polarhome.com Connection to gate.polarhome.com closed by remote host. ubuntu.polarhome.com bash only B macosx.polarhome.com macosx.polarhome.com ### Test if empty command in process list causes problems diff --git a/testsuite/wanted-results/parallel-tutorial b/testsuite/wanted-results/parallel-tutorial index a51fc7b9..3877ebcc 100644 --- a/testsuite/wanted-results/parallel-tutorial +++ b/testsuite/wanted-results/parallel-tutorial @@ -613,7 +613,7 @@ B D seq 10 20 | parallel --tmux 'echo start {}; sleep {}; echo done {}' See output with: tmux -S /tmp/parallel-tutorial-tmpdir/tmsXXXXX attach tmux -S /tmp/tmsXXXXX attach -no sessions +no current session parallel --delay 2.5 echo Starting {}\;date ::: 1 2 3 Starting 1 DATE OUTPUT