tange/parallel
1
0
Fork 0
mirror of https://git.savannah.gnu.org/git/parallel.git synced 2024-11-22 14:07:55 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

testsuite: Tests for env_parallel for multiple shells. Passes testsuite.

Browse source
This commit is contained in:
Ole Tange 2015-05-21 23:02:55 +02:00
parent 03a69b38ce
commit 6df65fa6ce
9 changed files with 222 additions and 40 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
src/parallel
View file

@ -832,7 +832,7 @@ sub options_hash {
"cat" => \$opt::cat,
"fifo" => \$opt::fifo,
"pipepart|pipe-part" => \$opt::pipepart,
"hgrp|hostgroup|hostgroups" => \$opt::hostgroups,
"hgrp|hostgrp|hostgroup|hostgroups" => \$opt::hostgroups,
);
}
@ -1075,7 +1075,7 @@ sub parse_options {
sub init_globals {
# Defaults:
$Global::version = 20150516;
$Global::version = 20150521;
$Global::progname = 'parallel';
$Global::infinity = 2**31;
$Global::debug = 0;
@ -6448,6 +6448,12 @@ sub wrapped {
' $PARALLEL_TMP'.
';';
}
if($ENV{'PARALLEL_ENV'}) {
# If $PARALLEL_ENV set, put that in front of the command
# Used for importing functions for fish
$ENV{'PARALLEL_ENV'} =~ s/\001/\n/g;
$command = $ENV{'PARALLEL_ENV'}."\n".$command;
}
# Wrap with ssh + tranferring of files
$command = $self->sshlogin_wrap($command);
if(@Global::cat_partials) {

85
src/parallel.pod
View file

@ -80,41 +80,74 @@ similar to B<cat | sh>.
The I<command> must be an executable, a script, a composed command, or
a function.
If it is a Bash function you need to B<export -f> the function
first. To use aliases copy the full environment as described under
B<--env> and use B<env_parallel> instead of B<parallel>.
B<Bash functions>: B<export -f> the function first.
If it is a Ksh function you can encode the function in a variable:
B<Bash aliases>: Copy the full environment using this and use B<env_parallel> instead of B<parallel>.
env_parallel() {
export PARALLEL_ENV="$(echo "shopt -s expand_aliases 2>/dev/null"; alias;typeset -p | grep -vFf <(readonly; echo GROUPS; echo FUNCNAME; echo DIRSTACK; echo _; echo PIPESTATUS; echo USERNAME) | grep -v BASH_;typeset -f)";
`which parallel` "$@";
unset PARALLEL_ENV;
}
# call as:
env_parallel [normal parallel options]
B<Ksh functions>: If it is a Ksh function you can encode the function in a variable:
foo() {
echo $*;
}
export fun=`typeset -f foo`; parallel 'eval "$fun";' foo ::: works
export PARALLEL_ENV=`typeset -f foo`
parallel foo ::: works
To export all functions and make them available when running remote:
To export all functions and also make them available when running remote:
export fun=`typeset -f`; parallel --env fun 'eval "$fun";' foo ::: works
env_parallel() {
export PARALLEL_ENV="$(alias | perl -pe 's/^/alias /';typeset -p;typeset -f)";
`which parallel` "$@";
unset PARALLEL_ENV;
}
# call as:
env_parallel [normal parallel options]
=cut
# ssh ksh@lo 'foo() { echo $* ; }; export fun="`typeset -f`"; parallel -S ksh@lo --env fun "eval \"\$fun\";"foo ::: works'
# ssh zsh@lo 'foo() { echo $* ; }; export fun="`typeset -f`"; parallel -S zsh@lo --env fun "eval \"\$fun\";"foo ::: works'
B<Zsh functions>: If it is a Zsh function you can encode the function in a variable:
# If it is a zsh function you will need to use this helper function
# B<exportf> to export and to set $PARALLEL_SHELL to bash:
#
# function exportf (){
# export $(echo $1)="`whence -f $1 | sed -e "s/$1 //" `"
# }
#
# function my_func(){
# echo $1;
# echo "hello";
# }
#
# exportf my_func
# parallel --env my_func "my_func {}" ::: 1 2
# TODO accept --env without -S
=pod
foo() {
echo $*;
}
PARALLEL_ENV="$(typeset -f foo)";
export PARALLEL_ENV
parallel foo ::: works
To export all functions and also make them available when running remote:
env_parallel() {
PARALLEL_ENV="$(alias | perl -pe 's/^/alias /';typeset -f)";
export PARALLEL_ENV
`which parallel` "$@";
unset PARALLEL_ENV;
}
# call as:
env_parallel [normal parallel options]
B<Fish functions>: If it is a Fish function you can encode the function in a variable:
function foo
echo $argv;
end
setenv PARALLEL_ENV (functions foo|perl -pe 's/\n/\001/')
parallel foo ::: works
To export all functions and also make them available when running remote:
function env_parallel
setenv PARALLEL_ENV (functions -n | perl -pe 's/,/\n/g' | while read d; functions $d; end|perl -pe 's/\n/\001/')
parallel $argv;
set -e PARALLEL_ENV
end
# call as:
env_parallel [normal parallel options]
The command cannot contain the character \257 (macron: ¯).

4
src/parallel_tutorial.html
View file

@ -1434,9 +1434,9 @@
<pre><code> force 4 cpus on server</code></pre>
<p>Servers can be put into groups by prepending &#39;@groupname&#39; to the server and the group can then be selected by appending &#39;@groupname&#39; to the argument if using &#39;--hostgrp&#39;.</p>
<p>Servers can be put into groups by prepending &#39;@groupname&#39; to the server and the group can then be selected by appending &#39;@groupname&#39; to the argument if using &#39;--hostgroup&#39;.</p>
<pre><code> parallel --hostgrp -S @grp1/$SERVER1 -S @grp2/SERVER2 echo {} ::: run_on_grp1@grp1 run_on_grp2@grp2</code></pre>
<pre><code> parallel --hostgroup -S @grp1/$SERVER1 -S @grp2/$SERVER2 echo {} ::: run_on_grp1@grp1 run_on_grp2@grp2</code></pre>
<p>Output:</p>

4
src/parallel_tutorial.pod
View file

@ -1387,9 +1387,9 @@ Output:
Servers can be put into groups by prepending '@groupname' to the
server and the group can then be selected by appending '@groupname' to
the argument if using '--hostgrp'.
the argument if using '--hostgroup'.
parallel --hostgrp -S @grp1/$SERVER1 -S @grp2/SERVER2 echo {} ::: run_on_grp1@grp1 run_on_grp2@grp2
parallel --hostgroup -S @grp1/$SERVER1 -S @grp2/$SERVER2 echo {} ::: run_on_grp1@grp1 run_on_grp2@grp2
Output:

56
testsuite/tests-to-run/parallel-local-ssh4.sh
View file

@ -65,3 +65,59 @@ echo '### bug #44143: csh and nice'
parallel --nice 1 -S csh@lo setenv B {}\; echo '$B' ::: OK
EOF
echo
echo "### Fish environment"
stdout ssh -q fish@lo <<'EOS' | grep -v 'packages can be updated.'
alias alias_echo=echo;
function func_echo
echo $argv;
end
function env_parallel
setenv PARALLEL_ENV (functions -n | perl -pe 's/,/\n/g' | while read d; functions $d; end|perl -pe 's/\n/\001/')
parallel $argv;
set -e PARALLEL_ENV
end
env_parallel alias_echo ::: alias_works
env_parallel func_echo ::: function_works
env_parallel -S fish@lo alias_echo ::: alias_works_over_ssh
env_parallel -S fish@lo func_echo ::: function_works_over_ssh
EOS
echo
echo "### Zsh environment"
stdout ssh -q zsh@lo <<'EOS' | grep -v 'packages can be updated.'
alias alias_echo=echo;
func_echo() {
echo $*;
}
env_parallel() {
PARALLEL_ENV="$(typeset -f)";
export PARALLEL_ENV
`which parallel` "$@";
unset PARALLEL_ENV;
}
env_parallel alias_echo ::: alias_does_not_work
env_parallel func_echo ::: function_works
env_parallel -S zsh@lo alias_echo ::: alias_does_not_work_over_ssh
env_parallel -S zsh@lo func_echo ::: function_works_over_ssh
EOS
echo
echo "### Ksh environment"
stdout ssh -q ksh@lo <<'EOS' | grep -v 'packages can be updated.'
alias alias_echo=echo;
func_echo() {
echo $*;
}
env_parallel() {
export PARALLEL_ENV="$(alias | perl -pe 's/^/alias /';typeset -p;typeset -f)";
`which parallel` "$@";
unset PARALLEL_ENV;
}
env_parallel alias_echo ::: alias_works
env_parallel func_echo ::: function_works
env_parallel -S ksh@lo alias_echo ::: alias_works_over_ssh
env_parallel -S ksh@lo func_echo ::: function_works_over_ssh
EOS

2
testsuite/tests-to-run/parallel-tutorial.sh
View file

@ -24,7 +24,7 @@ perl -ne '$/="\n\n"; /^Output/../^[^O]\S/ and next; /^ / and print;' ../../src/
stdout parallel -j7 -vd'\n\n' |
perl -pe '$|=1;
# --tmux
s:(/tmp\S+)(/tms).....:$1XXXXX$2:;
s:(/tmp\S+)(tms).....:$1$2XXXXX:;
# --files
s:(/tmp\S+par).....(\....):$1XXXXX$2:;
# --eta --progress

39
testsuite/wanted-results/parallel-local-ssh4
View file

@ -69,3 +69,42 @@ echo '### bug #44143: csh and nice'
### bug #44143: csh and nice
parallel --nice 1 -S csh@lo setenv B {}\; echo '$B' ::: OK
OK
### Fish environment
Welcome to Linux Mint 17 Qiana (GNU/Linux 3.16.0-31-lowlatency x86_64)
Welcome to Linux Mint
* Documentation: http://www.linuxmint.com
0 updates are security updates.
alias_works
function_works
alias_works_over_ssh
function_works_over_ssh
### Zsh environment
Welcome to Linux Mint 17 Qiana (GNU/Linux 3.16.0-31-lowlatency x86_64)
Welcome to Linux Mint
* Documentation: http://www.linuxmint.com
0 updates are security updates.
zsh:10: command not found: alias_echo
function_works
zsh:10: command not found: alias_echo
function_works_over_ssh
### Ksh environment
Welcome to Linux Mint 17 Qiana (GNU/Linux 3.16.0-31-lowlatency x86_64)
Welcome to Linux Mint
* Documentation: http://www.linuxmint.com
0 updates are security updates.
alias_works
function_works
alias_works_over_ssh
function_works_over_ssh

2
testsuite/wanted-results/parallel-local22
View file

@ -90,7 +90,7 @@ With script in $PARALLEL /bin/bash=/home/tange/privat/parallel/testsuite
echo '### bug #42892: parallel -a nonexiting --pipepart'
### bug #42892: parallel -a nonexiting --pipepart
parallel --pipepart -a nonexisting wc
parallel: Error: Cannot open input file `nonexisting': No such file or directory.
parallel: Error: nonexisting is not a seekable file.
echo '### bug #42913: Dont use $SHELL but the shell currently running'
### bug #42913: Dont use $SHELL but the shell currently running
echo '## Unknown shell => $SHELL (bash)'

60
testsuite/wanted-results/parallel-tutorial
View file

@ -404,10 +404,38 @@ B D
wait
3
parallel --use-cpus-instead-of-cores -N0 sleep 1 :::: num8
parallel --shuf echo ::: 1 2 3 ::: a b c ::: A B C
123 abc ABC
123 abc ABC
123 abc ABC
123 abc ABC
123 abc ABC
123 abc ABC
123 abc ABC
123 abc ABC
123 abc ABC
123 abc ABC
123 abc ABC
123 abc ABC
123 abc ABC
123 abc ABC
123 abc ABC
123 abc ABC
123 abc ABC
123 abc ABC
123 abc ABC
123 abc ABC
123 abc ABC
123 abc ABC
123 abc ABC
123 abc ABC
123 abc ABC
123 abc ABC
123 abc ABC
seq 10 20 | parallel --tmux 'echo start {}; sleep {}; echo done {}'
See output with: tmux -S /tmp/parallel-tutorialXXXXX/tms attach
tmux -S /tmp/parXXXXX.tms attach
See output with: tmux -S /tmp/parallel-tutorial/tmsXXXXX attach
tmux -S /tmp/tmsXXXXX attach
no sessions
parallel --delay 2.5 echo Starting {}\;date ::: 1 2 3
Starting 1
@ -446,7 +474,7 @@ BASE64 parallel --joblog /tmp/log exit ::: 1 2 3 0
cat /tmp/log;
parallel --resume-failed --joblog /tmp/log exit ::: 1 2 3 0 0 0
cat /tmp/log;
parallel -j2 --halt 1 echo {}\; exit {} ::: 0 0 1 2 3
parallel -j2 --halt soon,fail=1 echo {}\; exit {} ::: 0 0 1 2 3
Seq Host Starttime JobRuntime Send Receive Exitval Signal Command
1 : TIMESTAMP 9.999 0 0 1 0 exit 1
2 : TIMESTAMP 9.999 0 0 2 0 exit 2
@ -483,14 +511,13 @@ echo 1; exit 1
parallel: Starting no more jobs. Waiting for 1 jobs to finish.
parallel: This job failed:
echo 2; exit 2
parallel -j2 --halt 2 echo {}\; exit {} ::: 0 0 1 2 3
parallel -j2 --halt now,fail=1 echo {}\; exit {} ::: 0 0 1 2 3
0
0
1
parallel: This job failed:
echo 1; exit 1
parallel -j2 --halt 20% echo {}\; exit {} ::: 0 0 1 2 3 4 5 6 7
0
parallel -j2 --halt soon,fail=20% echo {}\; exit {} ::: 0 1 2 3 4 5 6 7 8 9
0
1
2
@ -502,6 +529,15 @@ echo 2; exit 2
parallel: Starting no more jobs. Waiting for 1 jobs to finish.
parallel: This job failed:
echo 3; exit 3
parallel -j2 --halt soon,success=1 echo {}\; exit {} ::: 1 2 3 0 4 5 6
1
2
3
0
4
parallel: This job succeeded:
echo 0; exit 0
parallel: Starting no more jobs. Waiting for 1 jobs to finish.
parallel -k --retries 3 'echo tried {} >>/tmp/runs; echo completed {}; exit {}' ::: 1 2 0
cat /tmp/runs
completed 1
@ -540,6 +576,9 @@ more
hosts
parallel -S 4/$SERVER1 echo force {} cpus on server ::: 4
force 4 cpus on server
parallel --hostgroup -S @grp1/$SERVER1 -S @grp2/$SERVER2 echo {} ::: run_on_grp1@grp1 run_on_grp2@grp2
run_on_grp1
run_on_grp2
echo This is input_file > input_file
parallel -S $SERVER1 --transfer cat ::: input_file
This is input_file
@ -895,6 +934,15 @@ The second finished
Fourth started
The third finished
The fourth finished
sem --id foo -u 'echo Slow started; sleep 5; echo Slow ended' &&
sem --id foo --semaphoretimeout 1 'echo Force this running after 1 sec' &&
sem --id foo --semaphoretimeout -2 'echo Give up after 1 sec'
sem --id foo --wait
Slow started
Force this running after 1 sec
Slow ended
parallel: Warning: Semaphore timed out. Stealing the semaphore.
parallel: Warning: Semaphore timed out. Exiting.
parallel --help
Usage: