tange/parallel
1
0
Fork 0
mirror of https://git.savannah.gnu.org/git/parallel.git synced 2024-11-26 07:57:58 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

testsuite: Most env_parallel works for different shells.

Browse source
This commit is contained in:
Ole Tange 2015-08-16 02:53:47 +02:00
parent eebaad1731
commit 5b51b19bb1
8 changed files with 159 additions and 57 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
doc/release_new_version
View file

@ -209,9 +209,9 @@ cc:Tim Cuthbertson <tim3d.junk@gmail.com>,
Ryoichiro Suzuki <ryoichiro.suzuki@gmail.com>, Ryoichiro Suzuki <ryoichiro.suzuki@gmail.com>,
Jesse Alama <jesse.alama@gmail.com> Jesse Alama <jesse.alama@gmail.com>
Subject: GNU Parallel 20150822 ('') released <<[stable]>> Subject: GNU Parallel 20150822 ('天津') released <<[stable]>>
GNU Parallel 20150822 ('') <<[stable]>> has been released. It is available for download at: http://ftp.gnu.org/gnu/parallel/ GNU Parallel 20150822 ('天津') <<[stable]>> has been released. It is available for download at: http://ftp.gnu.org/gnu/parallel/
<<No new functionality was introduced so this is a good candidate for a stable release.>> <<No new functionality was introduced so this is a good candidate for a stable release.>>
@ -224,6 +224,8 @@ Haiku of the month:
New in this release: New in this release:
* If $job->skip() is called in {= =} the job will not be run.
* @arg can now be used in {= =}: parallel echo '{= $arg[2] < 5 and $_="j" =}' ::: 1 2 3 ::: 4 5 6 * @arg can now be used in {= =}: parallel echo '{= $arg[2] < 5 and $_="j" =}' ::: 1 2 3 ::: 4 5 6
* Version of GNU Parallel that uses PostgreSQL as backend https://github.com/stephen-fralich/parallel-sql/ * Version of GNU Parallel that uses PostgreSQL as backend https://github.com/stephen-fralich/parallel-sql/

2
src/parallel
View file

@ -1090,7 +1090,7 @@ sub parse_options {
sub init_globals { sub init_globals {
# Defaults: # Defaults:
$Global::version = 20150723; $Global::version = 20150813;
$Global::progname = 'parallel'; $Global::progname = 'parallel';
$Global::infinity = 2**31; $Global::infinity = 2**31;
$Global::debug = 0; $Global::debug = 0;

23
testsuite/tests-to-run/parallel-local-ssh1.sh
View file

@ -30,29 +30,6 @@ echo '### Deal with long command lines on remote servers'
echo '### Test bug #34241: --pipe should not spawn unneeded processes' echo '### Test bug #34241: --pipe should not spawn unneeded processes'
seq 5 | ssh csh@lo parallel -k --block 5 --pipe -j10 cat\\\;echo Block_end seq 5 | ssh csh@lo parallel -k --block 5 --pipe -j10 cat\\\;echo Block_end
echo '### --env _'
fUbAr="OK FUBAR" parallel -S parallel@lo --env _ echo '$fUbAr $DEBEMAIL' ::: test
fUbAr="OK FUBAR" parallel -S csh@lo --env _ echo '$fUbAr $DEBEMAIL' ::: test
echo '### --env _ with explicit mentioning of normally ignored var $DEBEMAIL'
fUbAr="OK FUBAR" parallel -S parallel@lo --env DEBEMAIL,_ echo '$fUbAr $DEBEMAIL' ::: test
fUbAr="OK FUBAR" parallel -S csh@lo --env DEBEMAIL,_ echo '$fUbAr $DEBEMAIL' ::: test
echo 'bug #40137: SHELL not bash: Warning when exporting funcs'
. <(printf 'myfunc() {\necho $1\n}'); export -f myfunc; parallel --env myfunc -S lo myfunc ::: no_warning
. <(printf 'myfunc() {\necho $1\n}'); export -f myfunc; SHELL=/bin/sh parallel --env myfunc -S lo myfunc ::: warning
echo 'env_parallel from man page - transfer non-exported var'
env_parallel() {
export parallel_bash_environment="$(echo "shopt -s expand_aliases 2>/dev/null"; alias;typeset -p | grep -vFf <(readonly; echo GROUPS; echo FUNCNAME; echo DIRSTACK; echo _; echo PIPESTATUS; echo USERNAME) | grep -v BASH_;typeset -f)";
`which parallel` "$@";
unset parallel_bash_environment;
};
var=nonexported env_parallel -S parallel@lo echo '$var' ::: variable
echo 'compared to parallel - no transfer non-exported var'
var=nonexported parallel -S parallel@lo echo '$var' ::: variable
echo '### bug #40002: --files and --nonall seem not to work together:' echo '### bug #40002: --files and --nonall seem not to work together:'
parallel --files --nonall -S localhost true | tee >(parallel rm) | wc -l parallel --files --nonall -S localhost true | tee >(parallel rm) | wc -l

90
testsuite/tests-to-run/parallel-local-ssh4.sh
View file

@ -3,6 +3,29 @@
# SSH only allowed to localhost/lo # SSH only allowed to localhost/lo
# --retries if ssh dies # --retries if ssh dies
cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | parallel -vj1 --retries 2 -k --joblog /tmp/jl-`basename $0` -L1 cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | parallel -vj1 --retries 2 -k --joblog /tmp/jl-`basename $0` -L1
echo '### --env _'
fUbAr="OK FUBAR" parallel -S parallel@lo --env _ echo '$fUbAr $DEBEMAIL' ::: test
fUbAr="OK FUBAR" parallel -S csh@lo --env _ echo '$fUbAr $DEBEMAIL' ::: test
echo '### --env _ with explicit mentioning of normally ignored var $DEBEMAIL'
fUbAr="OK FUBAR" parallel -S parallel@lo --env DEBEMAIL,_ echo '$fUbAr $DEBEMAIL' ::: test
fUbAr="OK FUBAR" parallel -S csh@lo --env DEBEMAIL,_ echo '$fUbAr $DEBEMAIL' ::: test
echo 'bug #40137: SHELL not bash: Warning when exporting funcs'
. <(printf 'myfunc() {\necho $1\n}'); export -f myfunc; parallel --env myfunc -S lo myfunc ::: no_warning
. <(printf 'myfunc() {\necho $1\n}'); export -f myfunc; SHELL=/bin/sh parallel --env myfunc -S lo myfunc ::: warning
echo 'env_parallel from man page - transfer non-exported var'
env_parallel() {
export parallel_bash_environment="$(echo "shopt -s expand_aliases 2>/dev/null"; alias;typeset -p | grep -vFf <(readonly; echo GROUPS; echo FUNCNAME; echo DIRSTACK; echo _; echo PIPESTATUS; echo USERNAME) | grep -v BASH_;typeset -f)";
`which parallel` "$@";
unset parallel_bash_environment;
};
var=nonexported env_parallel -S parallel@lo echo '$var' ::: variable
echo 'compared to parallel - no transfer non-exported var'
var=nonexported parallel -S parallel@lo echo '$var' ::: variable
echo '### zsh' echo '### zsh'
ssh zsh@lo 'fun="() { echo function from zsh to zsh \$*; }"; ssh zsh@lo 'fun="() { echo function from zsh to zsh \$*; }";
export fun; export fun;
@ -67,17 +90,62 @@ echo '### bug #45575: -m and multiple hosts repeats first args'
EOF EOF
echo echo
echo Test env_parallel:
echo + for each shell
echo + remote, locally
echo + variables, variables with funky content, arrays, functions, aliases
echo
echo "### Bash environment"
#stdout ssh -t lo <<'EOS'
myvar="myvar works"
funky=$(perl -e 'print pack "c*", 1..255')
alias alias_echo=echo;
func_echo() {
echo $*;
echo "$myvar"
echo Funky-"$funky"-funky
}
env_parallel() {
export PARALLEL_ENV="$(echo "shopt -s expand_aliases 2>/dev/null"; alias;typeset -p |
grep -vFf <(readonly) |
grep -v 'declare .. (GROUPS|FUNCNAME|DIRSTACK|_|PIPESTATUS|USERNAME|BASH_[A-Z_]+) ';
typeset -f)";
`which parallel` "$@";
unset PARALLEL_ENV;
}
env_parallel alias_echo ::: alias_works
env_parallel func_echo ::: function_works
env_parallel -S lo alias_echo ::: alias_works_over_ssh
env_parallel -S lo func_echo ::: function_works_over_ssh
#EOS
echo "### Fish environment" echo "### Fish environment"
stdout ssh -q fish@lo <<'EOS' | egrep -v 'Welcome to |packages can be updated.' stdout ssh -q fish@lo <<'EOS' | egrep -v 'Welcome to |packages can be updated.'
set myvar "myvar works"
# ` (ascii 39) is a problem
set funky (perl -e 'print pack "c*", 1..38,40..255')
alias alias_echo=echo; alias alias_echo=echo;
function func_echo function func_echo
echo $argv; echo $argv;
echo $myvar;
echo Funky-"$funky"-funky
end end
function env_parallel function env_parallel
setenv PARALLEL_ENV (functions -n | perl -pe 's/,/\n/g' | while read d; functions $d; end|perl -pe 's/\n/\001/') setenv PARALLEL_ENV (begin; \
set -L|perl -ne '
/^(status|version|history|PWD|SHLVL|USER|_|COLUMNS|FISH_VERSION|HOME|LINES) /
or s/^/set / and print
'; \
functions -n | perl -pe 's/,/\n/g' | while read d; functions $d; end; \
end|perl -pe 's/\n/\001/')
parallel $argv; parallel $argv;
set -e PARALLEL_ENV set -e PARALLEL_ENV
end end
env_parallel alias_echo ::: alias_works env_parallel alias_echo ::: alias_works
env_parallel func_echo ::: function_works env_parallel func_echo ::: function_works
env_parallel -S fish@lo alias_echo ::: alias_works_over_ssh env_parallel -S fish@lo alias_echo ::: alias_works_over_ssh
@ -87,17 +155,35 @@ EOS
echo echo
echo "### Zsh environment" echo "### Zsh environment"
stdout ssh -q zsh@lo <<'EOS' | egrep -v 'Welcome to |packages can be updated.' stdout ssh -q zsh@lo <<'EOS' | egrep -v 'Welcome to |packages can be updated.'
myvar="myvar works"
funky=$(perl -e 'print pack "c*", 1..255')
alias alias_echo=echo; alias alias_echo=echo;
func_echo() { func_echo() {
echo $*; echo $*;
echo "$myvar"
echo Funky-"$funky"-funky
} }
env_parallel() { env_parallel() {
PARALLEL_ENV="$(typeset -f)"; PARALLEL_ENV="$(typeset -f)";
export PARALLEL_ENV export PARALLEL_ENV
`which parallel` "$@"; `which parallel` "$@";
unset PARALLEL_ENV; unset PARALLEL_ENV;
} }
env_parallel() {
unset PARALLEL_ENV;
export PARALLEL_ENV="$(alias | perl -pe 's/^/alias /';typeset -p |
grep -aFvf <(typeset -pr)|egrep -iav 'ZSH_EVAL_CONTEXT|LINENO=| _=|aliases|^typeset [a-z_]+$'|
egrep -av '^(typeset IFS=|..$)|cyan'
;
typeset -f)";
parallel "$@";
}
env_parallel alias_echo ::: alias_does_not_work env_parallel alias_echo ::: alias_does_not_work
env_parallel func_echo ::: function_works env_parallel func_echo ::: function_works
env_parallel -S zsh@lo alias_echo ::: alias_does_not_work_over_ssh env_parallel -S zsh@lo alias_echo ::: alias_does_not_work_over_ssh
env_parallel -S zsh@lo func_echo ::: function_works_over_ssh env_parallel -S zsh@lo func_echo ::: function_works_over_ssh
@ -106,9 +192,11 @@ EOS
echo echo
echo "### Ksh environment" echo "### Ksh environment"
stdout ssh -q ksh@lo <<'EOS' | egrep -v 'Welcome to |packages can be updated.' stdout ssh -q ksh@lo <<'EOS' | egrep -v 'Welcome to |packages can be updated.'
myvar="myvar works"
alias alias_echo=echo; alias alias_echo=echo;
func_echo() { func_echo() {
echo $*; echo $*;
echo "$myvar"
} }
env_parallel() { env_parallel() {
export PARALLEL_ENV="$(alias | perl -pe 's/^/alias /';typeset -p|egrep -v 'typeset( -i)? -r|PIPESTATUS';typeset -f)"; export PARALLEL_ENV="$(alias | perl -pe 's/^/alias /';typeset -p|egrep -v 'typeset( -i)? -r|PIPESTATUS';typeset -f)";

2
testsuite/tests-to-run/parallel-local4.sh
View file

@ -65,7 +65,7 @@ echo 'env in zsh'
echo 'bug #45692: Easy way of cancelling a job in {= =} and' echo 'bug #45692: Easy way of cancelling a job in {= =} and'
echo 'bug #45691: Accessing multiple arguments in {= =}' echo 'bug #45691: Accessing multiple arguments in {= =}'
parallel echo {= '$arg[1] eq 2 and $job->skip()' =} ::: {1..5} parallel -k echo {= '$arg[1] eq 2 and $job->skip()' =} ::: {1..5}
EOF EOF

26
testsuite/wanted-results/parallel-local-ssh1
View file

@ -47,32 +47,6 @@ Block_end
Block_end Block_end
5 5
Block_end Block_end
echo '### --env _'
### --env _
fUbAr="OK FUBAR" parallel -S parallel@lo --env _ echo '$fUbAr $DEBEMAIL' ::: test
OK FUBAR test
fUbAr="OK FUBAR" parallel -S csh@lo --env _ echo '$fUbAr $DEBEMAIL' ::: test
DEBEMAIL: Undefined variable.
echo '### --env _ with explicit mentioning of normally ignored var $DEBEMAIL'
### --env _ with explicit mentioning of normally ignored var $DEBEMAIL
fUbAr="OK FUBAR" parallel -S parallel@lo --env DEBEMAIL,_ echo '$fUbAr $DEBEMAIL' ::: test
OK FUBAR ole@tange.dk test
fUbAr="OK FUBAR" parallel -S csh@lo --env DEBEMAIL,_ echo '$fUbAr $DEBEMAIL' ::: test
OK FUBAR ole@tange.dk test
echo 'bug #40137: SHELL not bash: Warning when exporting funcs'
bug #40137: SHELL not bash: Warning when exporting funcs
. <(printf 'myfunc() {\necho $1\n}'); export -f myfunc; parallel --env myfunc -S lo myfunc ::: no_warning
no_warning
. <(printf 'myfunc() {\necho $1\n}'); export -f myfunc; SHELL=/bin/sh parallel --env myfunc -S lo myfunc ::: warning
warning
echo 'env_parallel from man page - transfer non-exported var'
env_parallel from man page - transfer non-exported var
env_parallel() { export parallel_bash_environment="$(echo "shopt -s expand_aliases 2>/dev/null"; alias;typeset -p | grep -vFf <(readonly; echo GROUPS; echo FUNCNAME; echo DIRSTACK; echo _; echo PIPESTATUS; echo USERNAME) | grep -v BASH_;typeset -f)"; `which parallel` "$@"; unset parallel_bash_environment; }; var=nonexported env_parallel -S parallel@lo echo '$var' ::: variable
nonexported variable
echo 'compared to parallel - no transfer non-exported var'
compared to parallel - no transfer non-exported var
var=nonexported parallel -S parallel@lo echo '$var' ::: variable
variable
echo '### bug #40002: --files and --nonall seem not to work together:' echo '### bug #40002: --files and --nonall seem not to work together:'
### bug #40002: --files and --nonall seem not to work together: ### bug #40002: --files and --nonall seem not to work together:
parallel --files --nonall -S localhost true | tee >(parallel rm) | wc -l parallel --files --nonall -S localhost true | tee >(parallel rm) | wc -l

65
testsuite/wanted-results/parallel-local-ssh4
View file

@ -1,3 +1,28 @@
echo '### --env _'
### --env _
fUbAr="OK FUBAR" parallel -S parallel@lo --env _ echo '$fUbAr $DEBEMAIL' ::: test
OK FUBAR test
fUbAr="OK FUBAR" parallel -S csh@lo --env _ echo '$fUbAr $DEBEMAIL' ::: test
echo '### --env _ with explicit mentioning of normally ignored var $DEBEMAIL'
### --env _ with explicit mentioning of normally ignored var $DEBEMAIL
fUbAr="OK FUBAR" parallel -S parallel@lo --env DEBEMAIL,_ echo '$fUbAr $DEBEMAIL' ::: test
OK FUBAR ole@tange.dk test
fUbAr="OK FUBAR" parallel -S csh@lo --env DEBEMAIL,_ echo '$fUbAr $DEBEMAIL' ::: test
OK FUBAR ole@tange.dk test
echo 'bug #40137: SHELL not bash: Warning when exporting funcs'
bug #40137: SHELL not bash: Warning when exporting funcs
. <(printf 'myfunc() {\necho $1\n}'); export -f myfunc; parallel --env myfunc -S lo myfunc ::: no_warning
no_warning
. <(printf 'myfunc() {\necho $1\n}'); export -f myfunc; SHELL=/bin/sh parallel --env myfunc -S lo myfunc ::: warning
warning
echo 'env_parallel from man page - transfer non-exported var'
env_parallel from man page - transfer non-exported var
env_parallel() { export parallel_bash_environment="$(echo "shopt -s expand_aliases 2>/dev/null"; alias;typeset -p | grep -vFf <(readonly; echo GROUPS; echo FUNCNAME; echo DIRSTACK; echo _; echo PIPESTATUS; echo USERNAME) | grep -v BASH_;typeset -f)"; `which parallel` "$@"; unset parallel_bash_environment; }; var=nonexported env_parallel -S parallel@lo echo '$var' ::: variable
nonexported variable
echo 'compared to parallel - no transfer non-exported var'
compared to parallel - no transfer non-exported var
var=nonexported parallel -S parallel@lo echo '$var' ::: variable
variable
echo '### zsh' echo '### zsh'
### zsh ### zsh
ssh zsh@lo 'fun="() { echo function from zsh to zsh \$*; }"; export fun; parallel --env fun fun ::: OK' ssh zsh@lo 'fun="() { echo function from zsh to zsh \$*; }"; export fun; parallel --env fun fun ::: OK'
@ -67,6 +92,28 @@ echo '### bug #45575: -m and multiple hosts repeats first args'
2 2
3 3
Test env_parallel:
+ for each shell
+ remote, locally
+ variables, variables with funky content, arrays, functions, aliases
### Bash environment
alias_works
function_works
myvar works
Funky-

 !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~
€亗儎厗噲墛媽崕彁憭摂晼棙櫄洔潪煚、￥ウЖ┆<EFBFBD><EFBFBD><EFBFBD>辈炒刀犯购患骄坷谅媚牌侨墒颂臀闲岩釉罩棕仝圮蒉哙徕沅彐玷殛腱眍镳耱篝貊鼬<EFBFBD><EFBFBD><EFBFBD><EFBFBD>-funky
alias_works_over_ssh
function_works_over_ssh
myvar works
Funky-

 !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~
€亗儎厗噲墛媽崕彁憭摂晼棙櫄洔潪煚、￥ウЖ┆<EFBFBD><EFBFBD><EFBFBD>辈炒刀犯购患骄坷谅媚牌侨墒颂臀闲岩釉罩棕仝圮蒉哙徕沅彐玷殛腱眍镳耱篝貊鼬<EFBFBD><EFBFBD><EFBFBD><EFBFBD>-funky
### Fish environment ### Fish environment
* Documentation: http://www.linuxmint.com * Documentation: http://www.linuxmint.com
@ -75,8 +122,12 @@ echo '### bug #45575: -m and multiple hosts repeats first args'
alias_works alias_works
function_works function_works
myvar works
Funky-   !"#$%&()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~€亗儎厗噲墛媽崕彁憭摂晼棙櫄洔潪煚、￥ウЖ┆<D096><E29486><EFBFBD>辈炒刀犯购患骄坷谅媚牌侨墒颂臀闲岩釉罩棕仝圮蒉哙徕沅彐玷殛腱眍镳耱篝貊鼬<E8B28A><E9BCAC><EFBFBD><EFBFBD>-funky
alias_works_over_ssh alias_works_over_ssh
function_works_over_ssh function_works_over_ssh
myvar works
Funky-   !"#$%&()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~€亗儎厗噲墛媽崕彁憭摂晼棙櫄洔潪煚、￥ウЖ┆<D096><E29486><EFBFBD>辈炒刀犯购患骄坷谅媚牌侨墒颂臀闲岩釉罩棕仝圮蒉哙徕沅彐玷殛腱眍镳耱篝貊鼬<E8B28A><E9BCAC><EFBFBD><EFBFBD>-funky
### Zsh environment ### Zsh environment
@ -84,10 +135,18 @@ function_works_over_ssh
0 updates are security updates. 0 updates are security updates.
zsh:10: command not found: alias_echo zsh:124: command not found: alias_echo
function_works function_works
zsh:10: command not found: alias_echo myvar works
Funky-

 !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~€亗儎厗噲墛媽崕彁憭摂晼棙櫄洔潪煚、￥ウЖ┆<D096><E29486><EFBFBD>辈炒刀犯购患骄坷谅媚牌侨墒颂臀闲岩釉罩棕仝圮蒉哙徕沅彐玷殛腱眍镳耱篝貊鼬<E8B28A><E9BCAC><EFBFBD><EFBFBD>-funky
zsh:124: command not found: alias_echo
function_works_over_ssh function_works_over_ssh
myvar works
Funky-

 !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~€亗儎厗噲墛媽崕彁憭摂晼棙櫄洔潪煚、￥ウЖ┆<D096><E29486><EFBFBD>辈炒刀犯购患骄坷谅媚牌侨墒颂臀闲岩釉罩棕仝圮蒉哙徕沅彐玷殛腱眍镳耱篝貊鼬<E8B28A><E9BCAC><EFBFBD><EFBFBD>-funky
### Ksh environment ### Ksh environment
@ -97,5 +156,7 @@ function_works_over_ssh
alias_works alias_works
function_works function_works
myvar works
alias_works_over_ssh alias_works_over_ssh
function_works_over_ssh function_works_over_ssh
myvar works

2
testsuite/wanted-results/parallel-local4
View file

@ -160,7 +160,7 @@ echo 'bug #45692: Easy way of cancelling a job in {= =} and'
bug #45692: Easy way of cancelling a job in {= =} and bug #45692: Easy way of cancelling a job in {= =} and
echo 'bug #45691: Accessing multiple arguments in {= =}' echo 'bug #45691: Accessing multiple arguments in {= =}'
bug #45691: Accessing multiple arguments in {= =} bug #45691: Accessing multiple arguments in {= =}
parallel echo {= '$arg[1] eq 2 and $job->skip()' =} ::: {1..5} parallel -k echo {= '$arg[1] eq 2 and $job->skip()' =} ::: {1..5}
1 1
3 3
4 4