parallel: {hgrp} = intersection of the hgrp of job and sshlogin.

This commit is contained in:
Ole Tange 2020-12-20 23:24:34 +01:00
parent 48e29256e2
commit 2a600a97be
5 changed files with 117 additions and 87 deletions

3
NEWS
View file

@ -8,6 +8,9 @@
* --delay 123auto will auto-adjust --delay. If jobs fail due to being * --delay 123auto will auto-adjust --delay. If jobs fail due to being
spawned too quickly, --delay will exponentially increase. spawned too quickly, --delay will exponentially increase.
* --plus sets {hgrp} that gives the intersection of the hostgroups of
the job and the sshlogin that the job is run on.
* Bug fixes and man page updates. * Bug fixes and man page updates.
News about GNU Parallel: News about GNU Parallel:

View file

@ -2212,6 +2212,8 @@ sub init_globals() {
'{host}' => '1 $_="\${PARALLEL_SSHHOST}";uq()', '{host}' => '1 $_="\${PARALLEL_SSHHOST}";uq()',
# {sshlogin} = sshlogin # {sshlogin} = sshlogin
'{sshlogin}' => '1 $_="\${PARALLEL_SSHLOGIN}";uq()', '{sshlogin}' => '1 $_="\${PARALLEL_SSHLOGIN}";uq()',
# {hgrp} = hostgroups
'{hgrp}' => '1 $_="\${PARALLEL_HOSTGROUPS}";uq()',
); );
# Modifiable copy of %Global::replace # Modifiable copy of %Global::replace
%Global::rpl = %Global::replace; %Global::rpl = %Global::replace;
@ -9181,7 +9183,8 @@ sub sshlogin_wrap($) {
# So --env myfunc should look for BASH_FUNC_myfunc() and BASH_FUNC_myfunc%% # So --env myfunc should look for BASH_FUNC_myfunc() and BASH_FUNC_myfunc%%
push(@vars, "PARALLEL_PID", "PARALLEL_SEQ", push(@vars, "PARALLEL_PID", "PARALLEL_SEQ",
"PARALLEL_SSHLOGIN", "PARALLEL_SSHHOST", "PARALLEL_JOBSLOT", "PARALLEL_SSHLOGIN", "PARALLEL_SSHHOST", "PARALLEL_HOSTGROUPS",
"PARALLEL_JOBSLOT",
map { ("BASH_FUNC_$_()", "BASH_FUNC_$_%%") } @vars); map { ("BASH_FUNC_$_()", "BASH_FUNC_$_%%") } @vars);
# Keep only defined variables # Keep only defined variables
return grep { defined($ENV{$_}) } @vars; return grep { defined($ENV{$_}) } @vars;
@ -9243,6 +9246,10 @@ sub sshlogin_wrap($) {
$ENV{'PARALLEL_JOBSLOT'} = $self->slot(); $ENV{'PARALLEL_JOBSLOT'} = $self->slot();
$ENV{'PARALLEL_SSHLOGIN'} = $sshlogin->string(); $ENV{'PARALLEL_SSHLOGIN'} = $sshlogin->string();
$ENV{'PARALLEL_SSHHOST'} = $sshlogin->serverlogin(); $ENV{'PARALLEL_SSHHOST'} = $sshlogin->serverlogin();
if ($opt::hostgroups) {
$ENV{'PARALLEL_HOSTGROUPS'} =
join',',$sshlogin->in_hostgroups($self->hostgroups());
}
$ENV{'PARALLEL_PID'} = $$; $ENV{'PARALLEL_PID'} = $$;
if($serverlogin eq ":") { if($serverlogin eq ":") {
if($opt::workdir) { if($opt::workdir) {

View file

@ -1073,9 +1073,9 @@ B<--header :> is an alias for B<--header '.*\n'>.
If I<regexp> is a number, it is a fixed number of lines. If I<regexp> is a number, it is a fixed number of lines.
=item B<--hostgroups> =item B<--hostgroups> (alpha testing)
=item B<--hgrp> =item B<--hgrp> (alpha testing)
Enable hostgroups on arguments. If an argument contains '@' the string Enable hostgroups on arguments. If an argument contains '@' the string
after '@' will be removed and treated as a list of hostgroups on which after '@' will be removed and treated as a list of hostgroups on which
@ -1093,7 +1093,7 @@ B<my_grp1_arg> may be run on either B<myserver1> or B<myserver2>,
B<third> may be run on either B<myserver1> or B<myserver3>, B<third> may be run on either B<myserver1> or B<myserver3>,
but B<arg_for_grp2> will only be run on B<myserver2>. but B<arg_for_grp2> will only be run on B<myserver2>.
See also: B<--sshlogin>. See also: B<--sshlogin>, B<$PARALLEL_HOSTGROUPS>.
=item B<-I> I<replace-str> =item B<-I> I<replace-str>
@ -1494,9 +1494,9 @@ of each job is saved in a file and the filename is then printed.
See also: B<--results> See also: B<--results>
=item B<--pipe> =item B<--pipe> (alpha testing)
=item B<--spreadstdin> =item B<--spreadstdin> (alpha testing)
Spread input to jobs on stdin (standard input). Read a block of data Spread input to jobs on stdin (standard input). Read a block of data
from stdin (standard input) and give one block of data as input to one from stdin (standard input) and give one block of data as input to one
@ -1579,6 +1579,7 @@ Shorthands for variables:
{slot} $PARALLEL_JOBSLOT (see {%}) {slot} $PARALLEL_JOBSLOT (see {%})
{sshlogin} $PARALLEL_SSHLOGIN {sshlogin} $PARALLEL_SSHLOGIN
{host} $PARALLEL_SSHHOST {host} $PARALLEL_SSHHOST
{hgrp} $PARALLEL_HOSTGROUPS
The following dynamic replacement strings are also activated. They are The following dynamic replacement strings are also activated. They are
inspired by bash's parameter expansion: inspired by bash's parameter expansion:
@ -4853,6 +4854,17 @@ the currently running jobs are finished before exiting.
Dir where GNU B<parallel> stores config files, semaphores, and caches Dir where GNU B<parallel> stores config files, semaphores, and caches
information between invocations. Default: $HOME/.parallel. information between invocations. Default: $HOME/.parallel.
=item $PARALLEL_HOSTGROUPS
When using B<--hostgroups> GNU B<parallel> sets this to the
intersection of the hostgroups of the job and the sshlogin that the
job is run on.
Remember to quote the $, so it gets evaluated by the correct shell. Or
use B<--plus> and {hgrp}.
=item $PARALLEL_JOBSLOT =item $PARALLEL_JOBSLOT
Set by GNU B<parallel> and can be used in jobs run by GNU B<parallel>. Set by GNU B<parallel> and can be used in jobs run by GNU B<parallel>.

View file

@ -1,58 +1,81 @@
#!/bin/bash #!/bin/bash
# SSH only allowed to localhost/lo # SSH only allowed to localhost/lo
cat <<'EOF' | parallel -vj100% --retries 3 -k --joblog /tmp/jl-`basename $0` -L1 -r
par_trailing_space_sshlogin() {
echo '### trailing space in sshlogin' echo '### trailing space in sshlogin'
echo 'sshlogin trailing space' | parallel --sshlogin "ssh -l parallel localhost " echo echo 'sshlogin trailing space' |
parallel --sshlogin "ssh -l parallel localhost " echo
}
par_special_char_trc() {
echo '### Special char file and dir transfer return and cleanup' echo '### Special char file and dir transfer return and cleanup'
cd /tmp; cd /tmp
mkdir -p d"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"; mkdir -p d"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"
echo local > d"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"/f"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"; echo local > d"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"/f"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"
ssh parallel@lo rm -rf d'*'/; ssh parallel@lo rm -rf d'*'/
mytouch() { mytouch() {
cat d"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"/f"`perl -e 'print pack("c*",1..9,11..46,48..255)'`" > d"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"/g"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"; cat d"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"/f"`perl -e 'print pack("c*",1..9,11..46,48..255)'`" > d"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"/g"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"
echo remote OK >> d"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"/g"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"; echo remote OK >> d"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"/g"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"
}; }
export -f mytouch; export -f mytouch
parallel --env mytouch -Sparallel@lo --transfer parallel --env mytouch -Sparallel@lo --transfer --return {=s:/f:/g:=} mytouch \
--return {=s:/f:/g:=} ::: d"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"/f"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"
mytouch
::: d"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"/f"`perl -e 'print pack("c*",1..9,11..46,48..255)'`";
cat d"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"/g"`perl -e 'print pack("c*",1..9,11..46,48..255)'`" cat d"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"/g"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"
# TODO Should be changed to --return '{=s:/f:/g:=}' and tested with csh - is error code kept?
}
par_rpl_perlexpr_not_used_in_command() {
echo '### Uniq {=perlexpr=} in return - not used in command' echo '### Uniq {=perlexpr=} in return - not used in command'
cd /tmp; cd /tmp
rm -f /tmp/parallel_perlexpr.2Parallel_PerlexPr; rm -f /tmp/parallel_perlexpr.2Parallel_PerlexPr
echo local > parallel_perlexpr; echo local > parallel_perlexpr
parallel -Sparallel@lo --trc {=s/pr/pr.2/=}{=s/p/P/g=} echo remote OK '>' {}.2{=s/p/P/g=} ::: parallel_perlexpr; parallel -Sparallel@lo --trc {=s/pr/pr.2/=}{=s/p/P/g=} echo remote OK '>' {}.2{=s/p/P/g=} ::: parallel_perlexpr
cat /tmp/parallel_perlexpr.2Parallel_PerlexPr; cat /tmp/parallel_perlexpr.2Parallel_PerlexPr
rm -f /tmp/parallel_perlexpr.2Parallel_PerlexPr /tmp/parallel_perlexpr rm -f /tmp/parallel_perlexpr.2Parallel_PerlexPr /tmp/parallel_perlexpr
# Should be changed to --return '{=s:/f:/g:=}' and tested with csh - is error code kept? }
par_remote_function_nice() {
echo '### functions and --nice' echo '### functions and --nice'
myfunc() { echo OK $*; }; export -f myfunc; parallel --nice 10 --env myfunc -S parallel@lo myfunc ::: func myfunc() {
echo OK $*
}
export -f myfunc
parallel --nice 10 --env myfunc -S parallel@lo myfunc ::: func
}
par_rplstr_return() {
echo '### bug #45906: {= in header =}' echo '### bug #45906: {= in header =}'
rm -f returnfile45906; rm -f returnfile45906
parallel --rpl '{G} $_=lc($_)' -S parallel@lo --return {G} --cleanup echo {G} '>' {G} ::: RETURNFILE45906; parallel --rpl '{G} $_=lc($_)' -S parallel@lo --return {G} --cleanup echo {G} '>' {G} ::: RETURNFILE45906
ls returnfile45906 ls returnfile45906
}
par_nonall_should_not_block() {
echo "### bug #47608: parallel --nonall -S lo 'echo ::: ' blocks" echo "### bug #47608: parallel --nonall -S lo 'echo ::: ' blocks"
parallel --nonall -S lo 'echo ::: ' parallel --nonall -S lo 'echo ::: '
}
par_export_functions_csh() {
echo '### exported function to csh but with PARALLEL_SHELL=bash' echo '### exported function to csh but with PARALLEL_SHELL=bash'
doit() { echo "$1"; }; doit() { echo "$1"; }
export -f doit; export -f doit
stdout parallel --env doit -S csh@lo doit ::: not_OK; stdout parallel --env doit -S csh@lo doit ::: not_OK
PARALLEL_SHELL=bash parallel --env doit -S csh@lo doit ::: OK PARALLEL_SHELL=bash parallel --env doit -S csh@lo doit ::: OK
}
par_progress_text_max_jobs_to_run() {
echo '### bug #49404: "Max jobs to run" does not equal the number of jobs specified when using GNU Parallel on remote server?' echo '### bug #49404: "Max jobs to run" does not equal the number of jobs specified when using GNU Parallel on remote server?'
echo should give 10 running jobs echo should give 10 running jobs
stdout parallel -S 16/lo --progress true ::: {1..10} | grep /.10 stdout parallel -S 16/lo --progress true ::: {1..10} | grep /.10
EOF }
par_hgrp_rpl() {
echo '### Implement {hgrp} replacement string'
parallel -k --plus --hgrp -S @b/bash@lo -S @c/csh@lo 'echo {sshlogin} {hgrp}' ::: b@b c@c
}
par_header_in_return() { par_header_in_return() {
echo '### bug #45907: --header : + --return {header}' echo '### bug #45907: --header : + --return {header}'

View file

@ -1,49 +1,34 @@
echo '### trailing space in sshlogin' par_export_functions_csh ### exported function to csh but with PARALLEL_SHELL=bash
### trailing space in sshlogin par_export_functions_csh CSH/TCSH DO NOT SUPPORT newlines IN VARIABLES/FUNCTIONS. Unset doit
echo 'sshlogin trailing space' | parallel --sshlogin "ssh -l parallel localhost " echo par_export_functions_csh OK
sshlogin trailing space
echo '### Special char file and dir transfer return and cleanup'
### Special char file and dir transfer return and cleanup
cd /tmp; mkdir -p d"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"; echo local > d"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"/f"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"; ssh parallel@lo rm -rf d'*'/; mytouch() { cat d"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"/f"`perl -e 'print pack("c*",1..9,11..46,48..255)'`" > d"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"/g"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"; echo remote OK >> d"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"/g"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"; }; export -f mytouch; parallel --env mytouch -Sparallel@lo --transfer --return {=s:/f:/g:=} mytouch ::: d"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"/f"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"; cat d"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"/g"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"
local
remote OK
echo '### Uniq {=perlexpr=} in return - not used in command'
### Uniq {=perlexpr=} in return - not used in command
cd /tmp; rm -f /tmp/parallel_perlexpr.2Parallel_PerlexPr; echo local > parallel_perlexpr; parallel -Sparallel@lo --trc {=s/pr/pr.2/=}{=s/p/P/g=} echo remote OK '>' {}.2{=s/p/P/g=} ::: parallel_perlexpr; cat /tmp/parallel_perlexpr.2Parallel_PerlexPr; rm -f /tmp/parallel_perlexpr.2Parallel_PerlexPr /tmp/parallel_perlexpr
remote OK
# Should be changed to --return '{=s:/f:/g:=}' and tested with csh - is error code kept?
echo '### functions and --nice'
### functions and --nice
myfunc() { echo OK $*; }; export -f myfunc; parallel --nice 10 --env myfunc -S parallel@lo myfunc ::: func
OK func
echo '### bug #45906: {= in header =}'
### bug #45906: {= in header =}
rm -f returnfile45906; parallel --rpl '{G} $_=lc($_)' -S parallel@lo --return {G} --cleanup echo {G} '>' {G} ::: RETURNFILE45906; ls returnfile45906
returnfile45906
echo "### bug #47608: parallel --nonall -S lo 'echo ::: ' blocks"
### bug #47608: parallel --nonall -S lo 'echo ::: ' blocks
parallel --nonall -S lo 'echo ::: '
:::
echo '### exported function to csh but with PARALLEL_SHELL=bash'
### exported function to csh but with PARALLEL_SHELL=bash
doit() { echo "$1"; }; export -f doit; stdout parallel --env doit -S csh@lo doit ::: not_OK; PARALLEL_SHELL=bash parallel --env doit -S csh@lo doit ::: OK
CSH/TCSH DO NOT SUPPORT newlines IN VARIABLES/FUNCTIONS. Unset doit
OK
echo '### bug #49404: "Max jobs to run" does not equal the number of jobs specified when using GNU Parallel on remote server?'
### bug #49404: "Max jobs to run" does not equal the number of jobs specified when using GNU Parallel on remote server?
echo should give 10 running jobs
should give 10 running jobs
stdout parallel -S 16/lo --progress true ::: {1..10} | grep /.10
1:lo / 16 / 10
par_header_in_return ### bug #45907: --header : + --return {header} par_header_in_return ### bug #45907: --header : + --return {header}
par_header_in_return returnfile45907 par_header_in_return returnfile45907
par_hgrp_rpl ### Implement {hgrp} replacement string
par_hgrp_rpl bash@lo b
par_hgrp_rpl csh@lo c
par_hostgroup_only_on_args ### Auto add hostgroup if given on on argument par_hostgroup_only_on_args ### Auto add hostgroup if given on on argument
par_hostgroup_only_on_args parallel: Warning: Adding hostgroups: sh@lo par_hostgroup_only_on_args parallel: Warning: Adding hostgroups: sh@lo
par_hostgroup_only_on_args sh par_hostgroup_only_on_args sh
par_nonall_should_not_block ### bug #47608: parallel --nonall -S lo 'echo ::: ' blocks
par_nonall_should_not_block :::
par_progress_text_max_jobs_to_run ### bug #49404: "Max jobs to run" does not equal the number of jobs specified when using GNU Parallel on remote server?
par_progress_text_max_jobs_to_run should give 10 running jobs
par_progress_text_max_jobs_to_run 1:lo / 16 / 10
par_quoting_for_onall ### bug #35427: quoting of {2} broken for --onall par_quoting_for_onall ### bug #35427: quoting of {2} broken for --onall
par_quoting_for_onall /bin/ls par_quoting_for_onall /bin/ls
par_remote_function_nice ### functions and --nice
par_remote_function_nice OK func
par_return_with_fixedstring ### Test --return with fixed string (Gave undef warnings) par_return_with_fixedstring ### Test --return with fixed string (Gave undef warnings)
par_return_with_fixedstring OK par_return_with_fixedstring OK
par_rpl_perlexpr_not_used_in_command ### Uniq {=perlexpr=} in return - not used in command
par_rpl_perlexpr_not_used_in_command remote OK
par_rplstr_return ### bug #45906: {= in header =}
par_rplstr_return returnfile45906
par_special_char_trc ### Special char file and dir transfer return and cleanup
par_special_char_trc local
par_special_char_trc remote OK
par_trailing_space_sshlogin ### trailing space in sshlogin
par_trailing_space_sshlogin sshlogin trailing space
par_trc_with_space ### Test --trc with space added in filename par_trc_with_space ### Test --trc with space added in filename
par_trc_with_space original par_trc_with_space original
par_trc_with_special_chars ### Test --trc with >|< added in filename par_trc_with_special_chars ### Test --trc with >|< added in filename