From 5b51b19bb16114672b49c29c392159344bee3621 Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Sun, 16 Aug 2015 02:53:47 +0200 Subject: [PATCH] testsuite: Most env_parallel works for different shells. --- doc/release_new_version | 6 +- src/parallel | 2 +- testsuite/tests-to-run/parallel-local-ssh1.sh | 23 ----- testsuite/tests-to-run/parallel-local-ssh4.sh | 90 ++++++++++++++++++- testsuite/tests-to-run/parallel-local4.sh | 2 +- testsuite/wanted-results/parallel-local-ssh1 | 26 ------ testsuite/wanted-results/parallel-local-ssh4 | 65 +++++++++++++- testsuite/wanted-results/parallel-local4 | 2 +- 8 files changed, 159 insertions(+), 57 deletions(-) diff --git a/doc/release_new_version b/doc/release_new_version index a2de8d2a..56343a5a 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -209,9 +209,9 @@ cc:Tim Cuthbertson , Ryoichiro Suzuki , Jesse Alama -Subject: GNU Parallel 20150822 ('') released <<[stable]>> +Subject: GNU Parallel 20150822 ('天津') released <<[stable]>> -GNU Parallel 20150822 ('') <<[stable]>> has been released. It is available for download at: http://ftp.gnu.org/gnu/parallel/ +GNU Parallel 20150822 ('天津') <<[stable]>> has been released. It is available for download at: http://ftp.gnu.org/gnu/parallel/ <> @@ -224,6 +224,8 @@ Haiku of the month: New in this release: +* If $job->skip() is called in {= =} the job will not be run. + * @arg can now be used in {= =}: parallel echo '{= $arg[2] < 5 and $_="j" =}' ::: 1 2 3 ::: 4 5 6 * Version of GNU Parallel that uses PostgreSQL as backend https://github.com/stephen-fralich/parallel-sql/ diff --git a/src/parallel b/src/parallel index 13567b1c..b4e47c51 100755 --- a/src/parallel +++ b/src/parallel @@ -1090,7 +1090,7 @@ sub parse_options { sub init_globals { # Defaults: - $Global::version = 20150723; + $Global::version = 20150813; $Global::progname = 'parallel'; $Global::infinity = 2**31; $Global::debug = 0; diff --git a/testsuite/tests-to-run/parallel-local-ssh1.sh b/testsuite/tests-to-run/parallel-local-ssh1.sh index de1d761b..bdf5d4d9 100644 --- a/testsuite/tests-to-run/parallel-local-ssh1.sh +++ b/testsuite/tests-to-run/parallel-local-ssh1.sh @@ -30,29 +30,6 @@ echo '### Deal with long command lines on remote servers' echo '### Test bug #34241: --pipe should not spawn unneeded processes' seq 5 | ssh csh@lo parallel -k --block 5 --pipe -j10 cat\\\;echo Block_end -echo '### --env _' - fUbAr="OK FUBAR" parallel -S parallel@lo --env _ echo '$fUbAr $DEBEMAIL' ::: test - fUbAr="OK FUBAR" parallel -S csh@lo --env _ echo '$fUbAr $DEBEMAIL' ::: test - -echo '### --env _ with explicit mentioning of normally ignored var $DEBEMAIL' - fUbAr="OK FUBAR" parallel -S parallel@lo --env DEBEMAIL,_ echo '$fUbAr $DEBEMAIL' ::: test - fUbAr="OK FUBAR" parallel -S csh@lo --env DEBEMAIL,_ echo '$fUbAr $DEBEMAIL' ::: test - -echo 'bug #40137: SHELL not bash: Warning when exporting funcs' - . <(printf 'myfunc() {\necho $1\n}'); export -f myfunc; parallel --env myfunc -S lo myfunc ::: no_warning - . <(printf 'myfunc() {\necho $1\n}'); export -f myfunc; SHELL=/bin/sh parallel --env myfunc -S lo myfunc ::: warning - -echo 'env_parallel from man page - transfer non-exported var' - env_parallel() { - export parallel_bash_environment="$(echo "shopt -s expand_aliases 2>/dev/null"; alias;typeset -p | grep -vFf <(readonly; echo GROUPS; echo FUNCNAME; echo DIRSTACK; echo _; echo PIPESTATUS; echo USERNAME) | grep -v BASH_;typeset -f)"; - `which parallel` "$@"; - unset parallel_bash_environment; - }; - var=nonexported env_parallel -S parallel@lo echo '$var' ::: variable - -echo 'compared to parallel - no transfer non-exported var' - var=nonexported parallel -S parallel@lo echo '$var' ::: variable - echo '### bug #40002: --files and --nonall seem not to work together:' parallel --files --nonall -S localhost true | tee >(parallel rm) | wc -l diff --git a/testsuite/tests-to-run/parallel-local-ssh4.sh b/testsuite/tests-to-run/parallel-local-ssh4.sh index 09364144..d3d96b36 100644 --- a/testsuite/tests-to-run/parallel-local-ssh4.sh +++ b/testsuite/tests-to-run/parallel-local-ssh4.sh @@ -3,6 +3,29 @@ # SSH only allowed to localhost/lo # --retries if ssh dies cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | parallel -vj1 --retries 2 -k --joblog /tmp/jl-`basename $0` -L1 +echo '### --env _' + fUbAr="OK FUBAR" parallel -S parallel@lo --env _ echo '$fUbAr $DEBEMAIL' ::: test + fUbAr="OK FUBAR" parallel -S csh@lo --env _ echo '$fUbAr $DEBEMAIL' ::: test + +echo '### --env _ with explicit mentioning of normally ignored var $DEBEMAIL' + fUbAr="OK FUBAR" parallel -S parallel@lo --env DEBEMAIL,_ echo '$fUbAr $DEBEMAIL' ::: test + fUbAr="OK FUBAR" parallel -S csh@lo --env DEBEMAIL,_ echo '$fUbAr $DEBEMAIL' ::: test + +echo 'bug #40137: SHELL not bash: Warning when exporting funcs' + . <(printf 'myfunc() {\necho $1\n}'); export -f myfunc; parallel --env myfunc -S lo myfunc ::: no_warning + . <(printf 'myfunc() {\necho $1\n}'); export -f myfunc; SHELL=/bin/sh parallel --env myfunc -S lo myfunc ::: warning + +echo 'env_parallel from man page - transfer non-exported var' + env_parallel() { + export parallel_bash_environment="$(echo "shopt -s expand_aliases 2>/dev/null"; alias;typeset -p | grep -vFf <(readonly; echo GROUPS; echo FUNCNAME; echo DIRSTACK; echo _; echo PIPESTATUS; echo USERNAME) | grep -v BASH_;typeset -f)"; + `which parallel` "$@"; + unset parallel_bash_environment; + }; + var=nonexported env_parallel -S parallel@lo echo '$var' ::: variable + +echo 'compared to parallel - no transfer non-exported var' + var=nonexported parallel -S parallel@lo echo '$var' ::: variable + echo '### zsh' ssh zsh@lo 'fun="() { echo function from zsh to zsh \$*; }"; export fun; @@ -67,17 +90,62 @@ echo '### bug #45575: -m and multiple hosts repeats first args' EOF echo +echo Test env_parallel: +echo + for each shell +echo + remote, locally +echo + variables, variables with funky content, arrays, functions, aliases +echo +echo "### Bash environment" +#stdout ssh -t lo <<'EOS' +myvar="myvar works" +funky=$(perl -e 'print pack "c*", 1..255') +alias alias_echo=echo; +func_echo() { + echo $*; + echo "$myvar" + echo Funky-"$funky"-funky +} + +env_parallel() { + export PARALLEL_ENV="$(echo "shopt -s expand_aliases 2>/dev/null"; alias;typeset -p | + grep -vFf <(readonly) | + grep -v 'declare .. (GROUPS|FUNCNAME|DIRSTACK|_|PIPESTATUS|USERNAME|BASH_[A-Z_]+) '; + typeset -f)"; + `which parallel` "$@"; + unset PARALLEL_ENV; +} + +env_parallel alias_echo ::: alias_works +env_parallel func_echo ::: function_works +env_parallel -S lo alias_echo ::: alias_works_over_ssh +env_parallel -S lo func_echo ::: function_works_over_ssh +#EOS + + echo "### Fish environment" stdout ssh -q fish@lo <<'EOS' | egrep -v 'Welcome to |packages can be updated.' +set myvar "myvar works" +# ` (ascii 39) is a problem +set funky (perl -e 'print pack "c*", 1..38,40..255') alias alias_echo=echo; function func_echo echo $argv; + echo $myvar; + echo Funky-"$funky"-funky end + function env_parallel - setenv PARALLEL_ENV (functions -n | perl -pe 's/,/\n/g' | while read d; functions $d; end|perl -pe 's/\n/\001/') + setenv PARALLEL_ENV (begin; \ + set -L|perl -ne ' + /^(status|version|history|PWD|SHLVL|USER|_|COLUMNS|FISH_VERSION|HOME|LINES) / + or s/^/set / and print + '; \ + functions -n | perl -pe 's/,/\n/g' | while read d; functions $d; end; \ + end|perl -pe 's/\n/\001/') parallel $argv; set -e PARALLEL_ENV end + env_parallel alias_echo ::: alias_works env_parallel func_echo ::: function_works env_parallel -S fish@lo alias_echo ::: alias_works_over_ssh @@ -87,17 +155,35 @@ EOS echo echo "### Zsh environment" stdout ssh -q zsh@lo <<'EOS' | egrep -v 'Welcome to |packages can be updated.' +myvar="myvar works" +funky=$(perl -e 'print pack "c*", 1..255') alias alias_echo=echo; func_echo() { echo $*; + echo "$myvar" + echo Funky-"$funky"-funky } + + env_parallel() { PARALLEL_ENV="$(typeset -f)"; export PARALLEL_ENV `which parallel` "$@"; unset PARALLEL_ENV; } + +env_parallel() { + unset PARALLEL_ENV; + export PARALLEL_ENV="$(alias | perl -pe 's/^/alias /';typeset -p | + grep -aFvf <(typeset -pr)|egrep -iav 'ZSH_EVAL_CONTEXT|LINENO=| _=|aliases|^typeset [a-z_]+$'| + egrep -av '^(typeset IFS=|..$)|cyan' +; + typeset -f)"; + parallel "$@"; +} + env_parallel alias_echo ::: alias_does_not_work + env_parallel func_echo ::: function_works env_parallel -S zsh@lo alias_echo ::: alias_does_not_work_over_ssh env_parallel -S zsh@lo func_echo ::: function_works_over_ssh @@ -106,9 +192,11 @@ EOS echo echo "### Ksh environment" stdout ssh -q ksh@lo <<'EOS' | egrep -v 'Welcome to |packages can be updated.' +myvar="myvar works" alias alias_echo=echo; func_echo() { echo $*; + echo "$myvar" } env_parallel() { export PARALLEL_ENV="$(alias | perl -pe 's/^/alias /';typeset -p|egrep -v 'typeset( -i)? -r|PIPESTATUS';typeset -f)"; diff --git a/testsuite/tests-to-run/parallel-local4.sh b/testsuite/tests-to-run/parallel-local4.sh index dfa83928..eecfda66 100644 --- a/testsuite/tests-to-run/parallel-local4.sh +++ b/testsuite/tests-to-run/parallel-local4.sh @@ -65,7 +65,7 @@ echo 'env in zsh' echo 'bug #45692: Easy way of cancelling a job in {= =} and' echo 'bug #45691: Accessing multiple arguments in {= =}' - parallel echo {= '$arg[1] eq 2 and $job->skip()' =} ::: {1..5} + parallel -k echo {= '$arg[1] eq 2 and $job->skip()' =} ::: {1..5} EOF diff --git a/testsuite/wanted-results/parallel-local-ssh1 b/testsuite/wanted-results/parallel-local-ssh1 index 02181f7e..fc94f978 100644 --- a/testsuite/wanted-results/parallel-local-ssh1 +++ b/testsuite/wanted-results/parallel-local-ssh1 @@ -47,32 +47,6 @@ Block_end Block_end 5 Block_end -echo '### --env _' -### --env _ - fUbAr="OK FUBAR" parallel -S parallel@lo --env _ echo '$fUbAr $DEBEMAIL' ::: test -OK FUBAR test - fUbAr="OK FUBAR" parallel -S csh@lo --env _ echo '$fUbAr $DEBEMAIL' ::: test -DEBEMAIL: Undefined variable. -echo '### --env _ with explicit mentioning of normally ignored var $DEBEMAIL' -### --env _ with explicit mentioning of normally ignored var $DEBEMAIL - fUbAr="OK FUBAR" parallel -S parallel@lo --env DEBEMAIL,_ echo '$fUbAr $DEBEMAIL' ::: test -OK FUBAR ole@tange.dk test - fUbAr="OK FUBAR" parallel -S csh@lo --env DEBEMAIL,_ echo '$fUbAr $DEBEMAIL' ::: test -OK FUBAR ole@tange.dk test -echo 'bug #40137: SHELL not bash: Warning when exporting funcs' -bug #40137: SHELL not bash: Warning when exporting funcs - . <(printf 'myfunc() {\necho $1\n}'); export -f myfunc; parallel --env myfunc -S lo myfunc ::: no_warning -no_warning - . <(printf 'myfunc() {\necho $1\n}'); export -f myfunc; SHELL=/bin/sh parallel --env myfunc -S lo myfunc ::: warning -warning -echo 'env_parallel from man page - transfer non-exported var' -env_parallel from man page - transfer non-exported var - env_parallel() { export parallel_bash_environment="$(echo "shopt -s expand_aliases 2>/dev/null"; alias;typeset -p | grep -vFf <(readonly; echo GROUPS; echo FUNCNAME; echo DIRSTACK; echo _; echo PIPESTATUS; echo USERNAME) | grep -v BASH_;typeset -f)"; `which parallel` "$@"; unset parallel_bash_environment; }; var=nonexported env_parallel -S parallel@lo echo '$var' ::: variable -nonexported variable -echo 'compared to parallel - no transfer non-exported var' -compared to parallel - no transfer non-exported var - var=nonexported parallel -S parallel@lo echo '$var' ::: variable -variable echo '### bug #40002: --files and --nonall seem not to work together:' ### bug #40002: --files and --nonall seem not to work together: parallel --files --nonall -S localhost true | tee >(parallel rm) | wc -l diff --git a/testsuite/wanted-results/parallel-local-ssh4 b/testsuite/wanted-results/parallel-local-ssh4 index 36b2aa03..8e60fc66 100644 --- a/testsuite/wanted-results/parallel-local-ssh4 +++ b/testsuite/wanted-results/parallel-local-ssh4 @@ -1,3 +1,28 @@ +echo '### --env _' +### --env _ + fUbAr="OK FUBAR" parallel -S parallel@lo --env _ echo '$fUbAr $DEBEMAIL' ::: test +OK FUBAR test + fUbAr="OK FUBAR" parallel -S csh@lo --env _ echo '$fUbAr $DEBEMAIL' ::: test +echo '### --env _ with explicit mentioning of normally ignored var $DEBEMAIL' +### --env _ with explicit mentioning of normally ignored var $DEBEMAIL + fUbAr="OK FUBAR" parallel -S parallel@lo --env DEBEMAIL,_ echo '$fUbAr $DEBEMAIL' ::: test +OK FUBAR ole@tange.dk test + fUbAr="OK FUBAR" parallel -S csh@lo --env DEBEMAIL,_ echo '$fUbAr $DEBEMAIL' ::: test +OK FUBAR ole@tange.dk test +echo 'bug #40137: SHELL not bash: Warning when exporting funcs' +bug #40137: SHELL not bash: Warning when exporting funcs + . <(printf 'myfunc() {\necho $1\n}'); export -f myfunc; parallel --env myfunc -S lo myfunc ::: no_warning +no_warning + . <(printf 'myfunc() {\necho $1\n}'); export -f myfunc; SHELL=/bin/sh parallel --env myfunc -S lo myfunc ::: warning +warning +echo 'env_parallel from man page - transfer non-exported var' +env_parallel from man page - transfer non-exported var + env_parallel() { export parallel_bash_environment="$(echo "shopt -s expand_aliases 2>/dev/null"; alias;typeset -p | grep -vFf <(readonly; echo GROUPS; echo FUNCNAME; echo DIRSTACK; echo _; echo PIPESTATUS; echo USERNAME) | grep -v BASH_;typeset -f)"; `which parallel` "$@"; unset parallel_bash_environment; }; var=nonexported env_parallel -S parallel@lo echo '$var' ::: variable +nonexported variable +echo 'compared to parallel - no transfer non-exported var' +compared to parallel - no transfer non-exported var + var=nonexported parallel -S parallel@lo echo '$var' ::: variable +variable echo '### zsh' ### zsh ssh zsh@lo 'fun="() { echo function from zsh to zsh \$*; }"; export fun; parallel --env fun fun ::: OK' @@ -67,6 +92,28 @@ echo '### bug #45575: -m and multiple hosts repeats first args' 2 3 +Test env_parallel: ++ for each shell ++ remote, locally ++ variables, variables with funky content, arrays, functions, aliases + +### Bash environment +alias_works +function_works +myvar works +Funky- + + +  !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~ +-funky +alias_works_over_ssh +function_works_over_ssh +myvar works +Funky- + + +  !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~ +-funky ### Fish environment * Documentation: http://www.linuxmint.com @@ -75,8 +122,12 @@ echo '### bug #45575: -m and multiple hosts repeats first args' alias_works function_works +myvar works +Funky-   !"#$%&()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~-funky alias_works_over_ssh function_works_over_ssh +myvar works +Funky-   !"#$%&()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~-funky ### Zsh environment @@ -84,10 +135,18 @@ function_works_over_ssh 0 updates are security updates. -zsh:10: command not found: alias_echo +zsh:124: command not found: alias_echo function_works -zsh:10: command not found: alias_echo +myvar works +Funky- + +  !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~-funky +zsh:124: command not found: alias_echo function_works_over_ssh +myvar works +Funky- + +  !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~-funky ### Ksh environment @@ -97,5 +156,7 @@ function_works_over_ssh alias_works function_works +myvar works alias_works_over_ssh function_works_over_ssh +myvar works diff --git a/testsuite/wanted-results/parallel-local4 b/testsuite/wanted-results/parallel-local4 index 608ec825..9d4e70cc 100644 --- a/testsuite/wanted-results/parallel-local4 +++ b/testsuite/wanted-results/parallel-local4 @@ -160,7 +160,7 @@ echo 'bug #45692: Easy way of cancelling a job in {= =} and' bug #45692: Easy way of cancelling a job in {= =} and echo 'bug #45691: Accessing multiple arguments in {= =}' bug #45691: Accessing multiple arguments in {= =} - parallel echo {= '$arg[1] eq 2 and $job->skip()' =} ::: {1..5} + parallel -k echo {= '$arg[1] eq 2 and $job->skip()' =} ::: {1..5} 1 3 4