tange/parallel
1
0
Fork 0
mirror of https://git.savannah.gnu.org/git/parallel.git synced 2024-11-21 21:47:54 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

parallel: {hgrp} = intersection of the hgrp of job and sshlogin.

Browse source
This commit is contained in:
Ole Tange 2020-12-20 23:24:34 +01:00
parent 48e29256e2
commit 2a600a97be
5 changed files with 117 additions and 87 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
NEWS
View file

@ -8,6 +8,9 @@
* --delay 123auto will auto-adjust --delay. If jobs fail due to being
spawned too quickly, --delay will exponentially increase.
* --plus sets {hgrp} that gives the intersection of the hostgroups of
the job and the sshlogin that the job is run on.
* Bug fixes and man page updates.
News about GNU Parallel:

9
src/parallel
View file

@ -2212,6 +2212,8 @@ sub init_globals() {
'{host}' => '1 $_="\${PARALLEL_SSHHOST}";uq()',
# {sshlogin} = sshlogin
'{sshlogin}' => '1 $_="\${PARALLEL_SSHLOGIN}";uq()',
# {hgrp} = hostgroups
'{hgrp}' => '1 $_="\${PARALLEL_HOSTGROUPS}";uq()',
);
# Modifiable copy of %Global::replace
%Global::rpl = %Global::replace;
@ -9181,7 +9183,8 @@ sub sshlogin_wrap($) {
# So --env myfunc should look for BASH_FUNC_myfunc() and BASH_FUNC_myfunc%%
push(@vars, "PARALLEL_PID", "PARALLEL_SEQ",
"PARALLEL_SSHLOGIN", "PARALLEL_SSHHOST", "PARALLEL_JOBSLOT",
"PARALLEL_SSHLOGIN", "PARALLEL_SSHHOST", "PARALLEL_HOSTGROUPS",
"PARALLEL_JOBSLOT",
map { ("BASH_FUNC_$_()", "BASH_FUNC_$_%%") } @vars);
# Keep only defined variables
return grep { defined($ENV{$_}) } @vars;
@ -9243,6 +9246,10 @@ sub sshlogin_wrap($) {
$ENV{'PARALLEL_JOBSLOT'} = $self->slot();
$ENV{'PARALLEL_SSHLOGIN'} = $sshlogin->string();
$ENV{'PARALLEL_SSHHOST'} = $sshlogin->serverlogin();
if ($opt::hostgroups) {
$ENV{'PARALLEL_HOSTGROUPS'} =
join',',$sshlogin->in_hostgroups($self->hostgroups());
}
$ENV{'PARALLEL_PID'} = $$;
if($serverlogin eq ":") {
if($opt::workdir) {

22
src/parallel.pod
View file

@ -1073,9 +1073,9 @@ B<--header :> is an alias for B<--header '.*\n'>.
If I<regexp> is a number, it is a fixed number of lines.
=item B<--hostgroups>
=item B<--hostgroups> (alpha testing)
=item B<--hgrp>
=item B<--hgrp> (alpha testing)
Enable hostgroups on arguments. If an argument contains '@' the string
after '@' will be removed and treated as a list of hostgroups on which
@ -1093,7 +1093,7 @@ B<my_grp1_arg> may be run on either B<myserver1> or B<myserver2>,
B<third> may be run on either B<myserver1> or B<myserver3>,
but B<arg_for_grp2> will only be run on B<myserver2>.
See also: B<--sshlogin>.
See also: B<--sshlogin>, B<$PARALLEL_HOSTGROUPS>.
=item B<-I> I<replace-str>
@ -1494,9 +1494,9 @@ of each job is saved in a file and the filename is then printed.
See also: B<--results>
=item B<--pipe>
=item B<--pipe> (alpha testing)
=item B<--spreadstdin>
=item B<--spreadstdin> (alpha testing)
Spread input to jobs on stdin (standard input). Read a block of data
from stdin (standard input) and give one block of data as input to one
@ -1579,6 +1579,7 @@ Shorthands for variables:
{slot} $PARALLEL_JOBSLOT (see {%})
{sshlogin} $PARALLEL_SSHLOGIN
{host} $PARALLEL_SSHHOST
{hgrp} $PARALLEL_HOSTGROUPS
The following dynamic replacement strings are also activated. They are
inspired by bash's parameter expansion:
@ -4853,6 +4854,17 @@ the currently running jobs are finished before exiting.
Dir where GNU B<parallel> stores config files, semaphores, and caches
information between invocations. Default: $HOME/.parallel.
=item $PARALLEL_HOSTGROUPS
When using B<--hostgroups> GNU B<parallel> sets this to the
intersection of the hostgroups of the job and the sshlogin that the
job is run on.
Remember to quote the $, so it gets evaluated by the correct shell. Or
use B<--plus> and {hgrp}.
=item $PARALLEL_JOBSLOT
Set by GNU B<parallel> and can be used in jobs run by GNU B<parallel>.

111
testsuite/tests-to-run/parallel-local-ssh3.sh
View file

@ -1,58 +1,81 @@
#!/bin/bash
# SSH only allowed to localhost/lo
cat <<'EOF' | parallel -vj100% --retries 3 -k --joblog /tmp/jl-`basename $0` -L1 -r
echo '### trailing space in sshlogin'
echo 'sshlogin trailing space' | parallel --sshlogin "ssh -l parallel localhost " echo
echo '### Special char file and dir transfer return and cleanup'
cd /tmp;
mkdir -p d"`perl -e 'print pack("c*",1..9,11..46,48..255)'`";
echo local > d"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"/f"`perl -e 'print pack("c*",1..9,11..46,48..255)'`";
ssh parallel@lo rm -rf d'*'/;
mytouch() {
cat d"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"/f"`perl -e 'print pack("c*",1..9,11..46,48..255)'`" > d"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"/g"`perl -e 'print pack("c*",1..9,11..46,48..255)'`";
echo remote OK >> d"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"/g"`perl -e 'print pack("c*",1..9,11..46,48..255)'`";
};
export -f mytouch;
parallel --env mytouch -Sparallel@lo --transfer
--return {=s:/f:/g:=}
mytouch
::: d"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"/f"`perl -e 'print pack("c*",1..9,11..46,48..255)'`";
cat d"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"/g"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"
par_trailing_space_sshlogin() {
echo '### trailing space in sshlogin'
echo 'sshlogin trailing space' |
parallel --sshlogin "ssh -l parallel localhost " echo
}
echo '### Uniq {=perlexpr=} in return - not used in command'
cd /tmp;
rm -f /tmp/parallel_perlexpr.2Parallel_PerlexPr;
echo local > parallel_perlexpr;
parallel -Sparallel@lo --trc {=s/pr/pr.2/=}{=s/p/P/g=} echo remote OK '>' {}.2{=s/p/P/g=} ::: parallel_perlexpr;
cat /tmp/parallel_perlexpr.2Parallel_PerlexPr;
rm -f /tmp/parallel_perlexpr.2Parallel_PerlexPr /tmp/parallel_perlexpr
par_special_char_trc() {
echo '### Special char file and dir transfer return and cleanup'
cd /tmp
mkdir -p d"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"
echo local > d"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"/f"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"
ssh parallel@lo rm -rf d'*'/
mytouch() {
cat d"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"/f"`perl -e 'print pack("c*",1..9,11..46,48..255)'`" > d"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"/g"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"
echo remote OK >> d"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"/g"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"
}
export -f mytouch
parallel --env mytouch -Sparallel@lo --transfer --return {=s:/f:/g:=} mytouch \
::: d"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"/f"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"
cat d"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"/g"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"
# TODO Should be changed to --return '{=s:/f:/g:=}' and tested with csh - is error code kept?
}
# Should be changed to --return '{=s:/f:/g:=}' and tested with csh - is error code kept?
par_rpl_perlexpr_not_used_in_command() {
echo '### Uniq {=perlexpr=} in return - not used in command'
cd /tmp
rm -f /tmp/parallel_perlexpr.2Parallel_PerlexPr
echo local > parallel_perlexpr
parallel -Sparallel@lo --trc {=s/pr/pr.2/=}{=s/p/P/g=} echo remote OK '>' {}.2{=s/p/P/g=} ::: parallel_perlexpr
cat /tmp/parallel_perlexpr.2Parallel_PerlexPr
rm -f /tmp/parallel_perlexpr.2Parallel_PerlexPr /tmp/parallel_perlexpr
echo '### functions and --nice'
myfunc() { echo OK $*; }; export -f myfunc; parallel --nice 10 --env myfunc -S parallel@lo myfunc ::: func
}
echo '### bug #45906: {= in header =}'
rm -f returnfile45906;
parallel --rpl '{G} $_=lc($_)' -S parallel@lo --return {G} --cleanup echo {G} '>' {G} ::: RETURNFILE45906;
ls returnfile45906
par_remote_function_nice() {
echo '### functions and --nice'
myfunc() {
echo OK $*
}
export -f myfunc
parallel --nice 10 --env myfunc -S parallel@lo myfunc ::: func
}
echo "### bug #47608: parallel --nonall -S lo 'echo ::: ' blocks"
parallel --nonall -S lo 'echo ::: '
par_rplstr_return() {
echo '### bug #45906: {= in header =}'
rm -f returnfile45906
parallel --rpl '{G} $_=lc($_)' -S parallel@lo --return {G} --cleanup echo {G} '>' {G} ::: RETURNFILE45906
ls returnfile45906
}
echo '### exported function to csh but with PARALLEL_SHELL=bash'
doit() { echo "$1"; };
export -f doit;
stdout parallel --env doit -S csh@lo doit ::: not_OK;
PARALLEL_SHELL=bash parallel --env doit -S csh@lo doit ::: OK
par_nonall_should_not_block() {
echo "### bug #47608: parallel --nonall -S lo 'echo ::: ' blocks"
parallel --nonall -S lo 'echo ::: '
}
echo '### bug #49404: "Max jobs to run" does not equal the number of jobs specified when using GNU Parallel on remote server?'
echo should give 10 running jobs
stdout parallel -S 16/lo --progress true ::: {1..10} | grep /.10
EOF
par_export_functions_csh() {
echo '### exported function to csh but with PARALLEL_SHELL=bash'
doit() { echo "$1"; }
export -f doit
stdout parallel --env doit -S csh@lo doit ::: not_OK
PARALLEL_SHELL=bash parallel --env doit -S csh@lo doit ::: OK
}
par_progress_text_max_jobs_to_run() {
echo '### bug #49404: "Max jobs to run" does not equal the number of jobs specified when using GNU Parallel on remote server?'
echo should give 10 running jobs
stdout parallel -S 16/lo --progress true ::: {1..10} | grep /.10
}
par_hgrp_rpl() {
echo '### Implement {hgrp} replacement string'
parallel -k --plus --hgrp -S @b/bash@lo -S @c/csh@lo 'echo {sshlogin} {hgrp}' ::: b@b c@c
}
par_header_in_return() {
echo '### bug #45907: --header : + --return {header}'

59
testsuite/wanted-results/parallel-local-ssh3
View file

@ -1,49 +1,34 @@
echo '### trailing space in sshlogin'
### trailing space in sshlogin
echo 'sshlogin trailing space' | parallel --sshlogin "ssh -l parallel localhost " echo
sshlogin trailing space
echo '### Special char file and dir transfer return and cleanup'
### Special char file and dir transfer return and cleanup
cd /tmp; mkdir -p d"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"; echo local > d"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"/f"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"; ssh parallel@lo rm -rf d'*'/; mytouch() { cat d"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"/f"`perl -e 'print pack("c*",1..9,11..46,48..255)'`" > d"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"/g"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"; echo remote OK >> d"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"/g"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"; }; export -f mytouch; parallel --env mytouch -Sparallel@lo --transfer --return {=s:/f:/g:=} mytouch ::: d"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"/f"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"; cat d"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"/g"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"
local
remote OK
echo '### Uniq {=perlexpr=} in return - not used in command'
### Uniq {=perlexpr=} in return - not used in command
cd /tmp; rm -f /tmp/parallel_perlexpr.2Parallel_PerlexPr; echo local > parallel_perlexpr; parallel -Sparallel@lo --trc {=s/pr/pr.2/=}{=s/p/P/g=} echo remote OK '>' {}.2{=s/p/P/g=} ::: parallel_perlexpr; cat /tmp/parallel_perlexpr.2Parallel_PerlexPr; rm -f /tmp/parallel_perlexpr.2Parallel_PerlexPr /tmp/parallel_perlexpr
remote OK
# Should be changed to --return '{=s:/f:/g:=}' and tested with csh - is error code kept?
echo '### functions and --nice'
### functions and --nice
myfunc() { echo OK $*; }; export -f myfunc; parallel --nice 10 --env myfunc -S parallel@lo myfunc ::: func
OK func
echo '### bug #45906: {= in header =}'
### bug #45906: {= in header =}
rm -f returnfile45906; parallel --rpl '{G} $_=lc($_)' -S parallel@lo --return {G} --cleanup echo {G} '>' {G} ::: RETURNFILE45906; ls returnfile45906
returnfile45906
echo "### bug #47608: parallel --nonall -S lo 'echo ::: ' blocks"
### bug #47608: parallel --nonall -S lo 'echo ::: ' blocks
parallel --nonall -S lo 'echo ::: '
:::
echo '### exported function to csh but with PARALLEL_SHELL=bash'
### exported function to csh but with PARALLEL_SHELL=bash
doit() { echo "$1"; }; export -f doit; stdout parallel --env doit -S csh@lo doit ::: not_OK; PARALLEL_SHELL=bash parallel --env doit -S csh@lo doit ::: OK
CSH/TCSH DO NOT SUPPORT newlines IN VARIABLES/FUNCTIONS. Unset doit
OK
echo '### bug #49404: "Max jobs to run" does not equal the number of jobs specified when using GNU Parallel on remote server?'
### bug #49404: "Max jobs to run" does not equal the number of jobs specified when using GNU Parallel on remote server?
echo should give 10 running jobs
should give 10 running jobs
stdout parallel -S 16/lo --progress true ::: {1..10} | grep /.10
1:lo / 16 / 10
par_export_functions_csh ### exported function to csh but with PARALLEL_SHELL=bash
par_export_functions_csh CSH/TCSH DO NOT SUPPORT newlines IN VARIABLES/FUNCTIONS. Unset doit
par_export_functions_csh OK
par_header_in_return ### bug #45907: --header : + --return {header}
par_header_in_return returnfile45907
par_hgrp_rpl ### Implement {hgrp} replacement string
par_hgrp_rpl bash@lo b
par_hgrp_rpl csh@lo c
par_hostgroup_only_on_args ### Auto add hostgroup if given on on argument
par_hostgroup_only_on_args parallel: Warning: Adding hostgroups: sh@lo
par_hostgroup_only_on_args sh
par_nonall_should_not_block ### bug #47608: parallel --nonall -S lo 'echo ::: ' blocks
par_nonall_should_not_block :::
par_progress_text_max_jobs_to_run ### bug #49404: "Max jobs to run" does not equal the number of jobs specified when using GNU Parallel on remote server?
par_progress_text_max_jobs_to_run should give 10 running jobs
par_progress_text_max_jobs_to_run 1:lo / 16 / 10
par_quoting_for_onall ### bug #35427: quoting of {2} broken for --onall
par_quoting_for_onall /bin/ls
par_remote_function_nice ### functions and --nice
par_remote_function_nice OK func
par_return_with_fixedstring ### Test --return with fixed string (Gave undef warnings)
par_return_with_fixedstring OK
par_rpl_perlexpr_not_used_in_command ### Uniq {=perlexpr=} in return - not used in command
par_rpl_perlexpr_not_used_in_command remote OK
par_rplstr_return ### bug #45906: {= in header =}
par_rplstr_return returnfile45906
par_special_char_trc ### Special char file and dir transfer return and cleanup
par_special_char_trc local
par_special_char_trc remote OK
par_trailing_space_sshlogin ### trailing space in sshlogin
par_trailing_space_sshlogin sshlogin trailing space
par_trc_with_space ### Test --trc with space added in filename
par_trc_with_space original
par_trc_with_special_chars ### Test --trc with >|< added in filename