diff --git a/src/parallel b/src/parallel index 9b005fe1..8e9b2ae4 100755 --- a/src/parallel +++ b/src/parallel @@ -832,7 +832,7 @@ sub options_hash { "cat" => \$opt::cat, "fifo" => \$opt::fifo, "pipepart|pipe-part" => \$opt::pipepart, - "hgrp|hostgroup|hostgroups" => \$opt::hostgroups, + "hgrp|hostgrp|hostgroup|hostgroups" => \$opt::hostgroups, ); } @@ -1075,7 +1075,7 @@ sub parse_options { sub init_globals { # Defaults: - $Global::version = 20150516; + $Global::version = 20150521; $Global::progname = 'parallel'; $Global::infinity = 2**31; $Global::debug = 0; @@ -6448,6 +6448,12 @@ sub wrapped { ' $PARALLEL_TMP'. ';'; } + if($ENV{'PARALLEL_ENV'}) { + # If $PARALLEL_ENV set, put that in front of the command + # Used for importing functions for fish + $ENV{'PARALLEL_ENV'} =~ s/\001/\n/g; + $command = $ENV{'PARALLEL_ENV'}."\n".$command; + } # Wrap with ssh + tranferring of files $command = $self->sshlogin_wrap($command); if(@Global::cat_partials) { diff --git a/src/parallel.pod b/src/parallel.pod index 9c1bc121..24ad9ff9 100644 --- a/src/parallel.pod +++ b/src/parallel.pod @@ -80,41 +80,74 @@ similar to B. The I must be an executable, a script, a composed command, or a function. -If it is a Bash function you need to B the function -first. To use aliases copy the full environment as described under -B<--env> and use B instead of B. +B: B the function first. -If it is a Ksh function you can encode the function in a variable: +B: Copy the full environment using this and use B instead of B. + + env_parallel() { + export PARALLEL_ENV="$(echo "shopt -s expand_aliases 2>/dev/null"; alias;typeset -p | grep -vFf <(readonly; echo GROUPS; echo FUNCNAME; echo DIRSTACK; echo _; echo PIPESTATUS; echo USERNAME) | grep -v BASH_;typeset -f)"; + `which parallel` "$@"; + unset PARALLEL_ENV; + } + # call as: + env_parallel [normal parallel options] + +B: If it is a Ksh function you can encode the function in a variable: foo() { echo $*; } - export fun=`typeset -f foo`; parallel 'eval "$fun";' foo ::: works + export PARALLEL_ENV=`typeset -f foo` + parallel foo ::: works -To export all functions and make them available when running remote: +To export all functions and also make them available when running remote: - export fun=`typeset -f`; parallel --env fun 'eval "$fun";' foo ::: works + env_parallel() { + export PARALLEL_ENV="$(alias | perl -pe 's/^/alias /';typeset -p;typeset -f)"; + `which parallel` "$@"; + unset PARALLEL_ENV; + } + # call as: + env_parallel [normal parallel options] -=cut -# ssh ksh@lo 'foo() { echo $* ; }; export fun="`typeset -f`"; parallel -S ksh@lo --env fun "eval \"\$fun\";"foo ::: works' -# ssh zsh@lo 'foo() { echo $* ; }; export fun="`typeset -f`"; parallel -S zsh@lo --env fun "eval \"\$fun\";"foo ::: works' +B: If it is a Zsh function you can encode the function in a variable: -# If it is a zsh function you will need to use this helper function -# B to export and to set $PARALLEL_SHELL to bash: -# -# function exportf (){ -# export $(echo $1)="`whence -f $1 | sed -e "s/$1 //" `" -# } -# -# function my_func(){ -# echo $1; -# echo "hello"; -# } -# -# exportf my_func -# parallel --env my_func "my_func {}" ::: 1 2 -# TODO accept --env without -S -=pod + foo() { + echo $*; + } + PARALLEL_ENV="$(typeset -f foo)"; + export PARALLEL_ENV + parallel foo ::: works + +To export all functions and also make them available when running remote: + + env_parallel() { + PARALLEL_ENV="$(alias | perl -pe 's/^/alias /';typeset -f)"; + export PARALLEL_ENV + `which parallel` "$@"; + unset PARALLEL_ENV; + } + # call as: + env_parallel [normal parallel options] + +B: If it is a Fish function you can encode the function in a variable: + + function foo + echo $argv; + end + + setenv PARALLEL_ENV (functions foo|perl -pe 's/\n/\001/') + parallel foo ::: works + +To export all functions and also make them available when running remote: + + function env_parallel + setenv PARALLEL_ENV (functions -n | perl -pe 's/,/\n/g' | while read d; functions $d; end|perl -pe 's/\n/\001/') + parallel $argv; + set -e PARALLEL_ENV + end + # call as: + env_parallel [normal parallel options] The command cannot contain the character \257 (macron: ¯). diff --git a/src/parallel_tutorial.html b/src/parallel_tutorial.html index d506baa2..e42c2a80 100644 --- a/src/parallel_tutorial.html +++ b/src/parallel_tutorial.html @@ -1434,9 +1434,9 @@ 
  force 4 cpus on server
-

Servers can be put into groups by prepending '@groupname' to the server and the group can then be selected by appending '@groupname' to the argument if using '--hostgrp'.

+

Servers can be put into groups by prepending '@groupname' to the server and the group can then be selected by appending '@groupname' to the argument if using '--hostgroup'.

-
  parallel --hostgrp -S @grp1/$SERVER1 -S @grp2/SERVER2 echo {} ::: run_on_grp1@grp1 run_on_grp2@grp2
+
  parallel --hostgroup -S @grp1/$SERVER1 -S @grp2/$SERVER2 echo {} ::: run_on_grp1@grp1 run_on_grp2@grp2

Output:

diff --git a/src/parallel_tutorial.pod b/src/parallel_tutorial.pod index 211709e9..50ddc58a 100644 --- a/src/parallel_tutorial.pod +++ b/src/parallel_tutorial.pod @@ -1387,9 +1387,9 @@ Output: Servers can be put into groups by prepending '@groupname' to the server and the group can then be selected by appending '@groupname' to -the argument if using '--hostgrp'. +the argument if using '--hostgroup'. - parallel --hostgrp -S @grp1/$SERVER1 -S @grp2/SERVER2 echo {} ::: run_on_grp1@grp1 run_on_grp2@grp2 + parallel --hostgroup -S @grp1/$SERVER1 -S @grp2/$SERVER2 echo {} ::: run_on_grp1@grp1 run_on_grp2@grp2 Output: diff --git a/testsuite/tests-to-run/parallel-local-ssh4.sh b/testsuite/tests-to-run/parallel-local-ssh4.sh index cf42faf2..4673d6a6 100644 --- a/testsuite/tests-to-run/parallel-local-ssh4.sh +++ b/testsuite/tests-to-run/parallel-local-ssh4.sh @@ -65,3 +65,59 @@ echo '### bug #44143: csh and nice' parallel --nice 1 -S csh@lo setenv B {}\; echo '$B' ::: OK EOF + +echo +echo "### Fish environment" +stdout ssh -q fish@lo <<'EOS' | grep -v 'packages can be updated.' +alias alias_echo=echo; +function func_echo + echo $argv; +end +function env_parallel + setenv PARALLEL_ENV (functions -n | perl -pe 's/,/\n/g' | while read d; functions $d; end|perl -pe 's/\n/\001/') + parallel $argv; + set -e PARALLEL_ENV +end +env_parallel alias_echo ::: alias_works +env_parallel func_echo ::: function_works +env_parallel -S fish@lo alias_echo ::: alias_works_over_ssh +env_parallel -S fish@lo func_echo ::: function_works_over_ssh +EOS + +echo +echo "### Zsh environment" +stdout ssh -q zsh@lo <<'EOS' | grep -v 'packages can be updated.' +alias alias_echo=echo; +func_echo() { + echo $*; +} +env_parallel() { + PARALLEL_ENV="$(typeset -f)"; + export PARALLEL_ENV + `which parallel` "$@"; + unset PARALLEL_ENV; +} +env_parallel alias_echo ::: alias_does_not_work +env_parallel func_echo ::: function_works +env_parallel -S zsh@lo alias_echo ::: alias_does_not_work_over_ssh +env_parallel -S zsh@lo func_echo ::: function_works_over_ssh +EOS + +echo +echo "### Ksh environment" +stdout ssh -q ksh@lo <<'EOS' | grep -v 'packages can be updated.' +alias alias_echo=echo; +func_echo() { + echo $*; +} +env_parallel() { + export PARALLEL_ENV="$(alias | perl -pe 's/^/alias /';typeset -p;typeset -f)"; + `which parallel` "$@"; + unset PARALLEL_ENV; +} +env_parallel alias_echo ::: alias_works +env_parallel func_echo ::: function_works +env_parallel -S ksh@lo alias_echo ::: alias_works_over_ssh +env_parallel -S ksh@lo func_echo ::: function_works_over_ssh +EOS + diff --git a/testsuite/tests-to-run/parallel-tutorial.sh b/testsuite/tests-to-run/parallel-tutorial.sh index 58fe91f9..99f74c15 100644 --- a/testsuite/tests-to-run/parallel-tutorial.sh +++ b/testsuite/tests-to-run/parallel-tutorial.sh @@ -24,7 +24,7 @@ perl -ne '$/="\n\n"; /^Output/../^[^O]\S/ and next; /^ / and print;' ../../src/ stdout parallel -j7 -vd'\n\n' | perl -pe '$|=1; # --tmux - s:(/tmp\S+)(/tms).....:$1XXXXX$2:; + s:(/tmp\S+)(tms).....:$1$2XXXXX:; # --files s:(/tmp\S+par).....(\....):$1XXXXX$2:; # --eta --progress diff --git a/testsuite/wanted-results/parallel-local-ssh4 b/testsuite/wanted-results/parallel-local-ssh4 index ac15cd63..7439baa0 100644 --- a/testsuite/wanted-results/parallel-local-ssh4 +++ b/testsuite/wanted-results/parallel-local-ssh4 @@ -69,3 +69,42 @@ echo '### bug #44143: csh and nice' ### bug #44143: csh and nice parallel --nice 1 -S csh@lo setenv B {}\; echo '$B' ::: OK OK + +### Fish environment +Welcome to Linux Mint 17 Qiana (GNU/Linux 3.16.0-31-lowlatency x86_64) + +Welcome to Linux Mint + * Documentation: http://www.linuxmint.com + +0 updates are security updates. + +alias_works +function_works +alias_works_over_ssh +function_works_over_ssh + +### Zsh environment +Welcome to Linux Mint 17 Qiana (GNU/Linux 3.16.0-31-lowlatency x86_64) + +Welcome to Linux Mint + * Documentation: http://www.linuxmint.com + +0 updates are security updates. + +zsh:10: command not found: alias_echo +function_works +zsh:10: command not found: alias_echo +function_works_over_ssh + +### Ksh environment +Welcome to Linux Mint 17 Qiana (GNU/Linux 3.16.0-31-lowlatency x86_64) + +Welcome to Linux Mint + * Documentation: http://www.linuxmint.com + +0 updates are security updates. + +alias_works +function_works +alias_works_over_ssh +function_works_over_ssh diff --git a/testsuite/wanted-results/parallel-local22 b/testsuite/wanted-results/parallel-local22 index 4771605b..806b30ed 100644 --- a/testsuite/wanted-results/parallel-local22 +++ b/testsuite/wanted-results/parallel-local22 @@ -90,7 +90,7 @@ With script in $PARALLEL /bin/bash=/home/tange/privat/parallel/testsuite echo '### bug #42892: parallel -a nonexiting --pipepart' ### bug #42892: parallel -a nonexiting --pipepart parallel --pipepart -a nonexisting wc -parallel: Error: Cannot open input file `nonexisting': No such file or directory. +parallel: Error: nonexisting is not a seekable file. echo '### bug #42913: Dont use $SHELL but the shell currently running' ### bug #42913: Dont use $SHELL but the shell currently running echo '## Unknown shell => $SHELL (bash)' diff --git a/testsuite/wanted-results/parallel-tutorial b/testsuite/wanted-results/parallel-tutorial index 2a287c2e..7772b39c 100644 --- a/testsuite/wanted-results/parallel-tutorial +++ b/testsuite/wanted-results/parallel-tutorial @@ -404,10 +404,38 @@ B D wait 3 parallel --use-cpus-instead-of-cores -N0 sleep 1 :::: num8 + parallel --shuf echo ::: 1 2 3 ::: a b c ::: A B C +123 abc ABC +123 abc ABC +123 abc ABC +123 abc ABC +123 abc ABC +123 abc ABC +123 abc ABC +123 abc ABC +123 abc ABC +123 abc ABC +123 abc ABC +123 abc ABC +123 abc ABC +123 abc ABC +123 abc ABC +123 abc ABC +123 abc ABC +123 abc ABC +123 abc ABC +123 abc ABC +123 abc ABC +123 abc ABC +123 abc ABC +123 abc ABC +123 abc ABC +123 abc ABC +123 abc ABC seq 10 20 | parallel --tmux 'echo start {}; sleep {}; echo done {}' -See output with: tmux -S /tmp/parallel-tutorialXXXXX/tms attach - tmux -S /tmp/parXXXXX.tms attach +See output with: tmux -S /tmp/parallel-tutorial/tmsXXXXX attach + tmux -S /tmp/tmsXXXXX attach no sessions parallel --delay 2.5 echo Starting {}\;date ::: 1 2 3 Starting 1 @@ -446,7 +474,7 @@ BASE64 parallel --joblog /tmp/log exit ::: 1 2 3 0 cat /tmp/log; parallel --resume-failed --joblog /tmp/log exit ::: 1 2 3 0 0 0 cat /tmp/log; - parallel -j2 --halt 1 echo {}\; exit {} ::: 0 0 1 2 3 + parallel -j2 --halt soon,fail=1 echo {}\; exit {} ::: 0 0 1 2 3 Seq Host Starttime JobRuntime Send Receive Exitval Signal Command 1 : TIMESTAMP 9.999 0 0 1 0 exit 1 2 : TIMESTAMP 9.999 0 0 2 0 exit 2 @@ -483,14 +511,13 @@ echo 1; exit 1 parallel: Starting no more jobs. Waiting for 1 jobs to finish. parallel: This job failed: echo 2; exit 2 - parallel -j2 --halt 2 echo {}\; exit {} ::: 0 0 1 2 3 + parallel -j2 --halt now,fail=1 echo {}\; exit {} ::: 0 0 1 2 3 0 0 1 parallel: This job failed: echo 1; exit 1 - parallel -j2 --halt 20% echo {}\; exit {} ::: 0 0 1 2 3 4 5 6 7 -0 + parallel -j2 --halt soon,fail=20% echo {}\; exit {} ::: 0 1 2 3 4 5 6 7 8 9 0 1 2 @@ -502,6 +529,15 @@ echo 2; exit 2 parallel: Starting no more jobs. Waiting for 1 jobs to finish. parallel: This job failed: echo 3; exit 3 + parallel -j2 --halt soon,success=1 echo {}\; exit {} ::: 1 2 3 0 4 5 6 +1 +2 +3 +0 +4 +parallel: This job succeeded: +echo 0; exit 0 +parallel: Starting no more jobs. Waiting for 1 jobs to finish. parallel -k --retries 3 'echo tried {} >>/tmp/runs; echo completed {}; exit {}' ::: 1 2 0 cat /tmp/runs completed 1 @@ -540,6 +576,9 @@ more hosts parallel -S 4/$SERVER1 echo force {} cpus on server ::: 4 force 4 cpus on server + parallel --hostgroup -S @grp1/$SERVER1 -S @grp2/$SERVER2 echo {} ::: run_on_grp1@grp1 run_on_grp2@grp2 +run_on_grp1 +run_on_grp2 echo This is input_file > input_file parallel -S $SERVER1 --transfer cat ::: input_file This is input_file @@ -895,6 +934,15 @@ The second finished Fourth started The third finished The fourth finished + sem --id foo -u 'echo Slow started; sleep 5; echo Slow ended' && + sem --id foo --semaphoretimeout 1 'echo Force this running after 1 sec' && + sem --id foo --semaphoretimeout -2 'echo Give up after 1 sec' + sem --id foo --wait +Slow started +Force this running after 1 sec +Slow ended +parallel: Warning: Semaphore timed out. Stealing the semaphore. +parallel: Warning: Semaphore timed out. Exiting. parallel --help Usage: