parallel: {hgrp} = intersection of the hgrp of job and sshlogin.

NEWS

@@ -8,6 +8,9 @@
 * --delay 123auto will auto-adjust --delay. If jobs fail due to being
   spawned too quickly, --delay will exponentially increase.
 
+* --plus sets {hgrp} that gives the intersection of the hostgroups of
+  the job and the sshlogin that the job is run on.
+
 * Bug fixes and man page updates.
 
 News about GNU Parallel:

src/parallel

@@ -2212,6 +2212,8 @@ sub init_globals() {
 	 '{host}' => '1 $_="\${PARALLEL_SSHHOST}";uq()',
 	 # {sshlogin} = sshlogin
 	 '{sshlogin}' => '1 $_="\${PARALLEL_SSHLOGIN}";uq()',
+	 # {hgrp} = hostgroups
+	 '{hgrp}' => '1 $_="\${PARALLEL_HOSTGROUPS}";uq()',
 	);
     # Modifiable copy of %Global::replace
     %Global::rpl = %Global::replace;
@@ -9181,7 +9183,8 @@ sub sshlogin_wrap($) {
 	# So --env myfunc should look for BASH_FUNC_myfunc() and BASH_FUNC_myfunc%%
 
 	push(@vars, "PARALLEL_PID", "PARALLEL_SEQ",
-	     "PARALLEL_SSHLOGIN", "PARALLEL_SSHHOST", "PARALLEL_JOBSLOT",
+	     "PARALLEL_SSHLOGIN", "PARALLEL_SSHHOST", "PARALLEL_HOSTGROUPS",
+	     "PARALLEL_JOBSLOT",
 	     map { ("BASH_FUNC_$_()", "BASH_FUNC_$_%%") } @vars);
 	# Keep only defined variables
 	return grep { defined($ENV{$_}) } @vars;
@@ -9243,6 +9246,10 @@ sub sshlogin_wrap($) {
 	$ENV{'PARALLEL_JOBSLOT'} = $self->slot();
 	$ENV{'PARALLEL_SSHLOGIN'} = $sshlogin->string();
 	$ENV{'PARALLEL_SSHHOST'} = $sshlogin->serverlogin();
+	if ($opt::hostgroups) {
+	    $ENV{'PARALLEL_HOSTGROUPS'} =
+		join',',$sshlogin->in_hostgroups($self->hostgroups());
+	}
 	$ENV{'PARALLEL_PID'} = $$;
 	if($serverlogin eq ":") {
 	    if($opt::workdir) {

src/parallel.pod

@@ -1073,9 +1073,9 @@ B<--header :> is an alias for B<--header '.*\n'>.
 If I<regexp> is a number, it is a fixed number of lines.
 
 
-=item B<--hostgroups>
+=item B<--hostgroups> (alpha testing)
 
-=item B<--hgrp>
+=item B<--hgrp> (alpha testing)
 
 Enable hostgroups on arguments. If an argument contains '@' the string
 after '@' will be removed and treated as a list of hostgroups on which
@@ -1093,7 +1093,7 @@ B<my_grp1_arg> may be run on either B<myserver1> or B<myserver2>,
 B<third> may be run on either B<myserver1> or B<myserver3>,
 but B<arg_for_grp2> will only be run on B<myserver2>.
 
-See also: B<--sshlogin>.
+See also: B<--sshlogin>, B<$PARALLEL_HOSTGROUPS>.
 
 
 =item B<-I> I<replace-str>
@@ -1494,9 +1494,9 @@ of each job is saved in a file and the filename is then printed.
 See also: B<--results>
 
 
-=item B<--pipe>
+=item B<--pipe> (alpha testing)
 
-=item B<--spreadstdin>
+=item B<--spreadstdin> (alpha testing)
 
 Spread input to jobs on stdin (standard input). Read a block of data
 from stdin (standard input) and give one block of data as input to one
@@ -1579,6 +1579,7 @@ Shorthands for variables:
   {slot}        $PARALLEL_JOBSLOT (see {%})
   {sshlogin}    $PARALLEL_SSHLOGIN
   {host}        $PARALLEL_SSHHOST
+  {hgrp}        $PARALLEL_HOSTGROUPS
 
 The following dynamic replacement strings are also activated. They are
 inspired by bash's parameter expansion:
@@ -4853,6 +4854,17 @@ the currently running jobs are finished before exiting.
 Dir where GNU B<parallel> stores config files, semaphores, and caches
 information between invocations. Default: $HOME/.parallel.
 
+
+=item $PARALLEL_HOSTGROUPS
+
+When using B<--hostgroups> GNU B<parallel> sets this to the
+intersection of the hostgroups of the job and the sshlogin that the
+job is run on.
+
+Remember to quote the $, so it gets evaluated by the correct shell. Or
+use B<--plus> and {hgrp}.
+
+
 =item $PARALLEL_JOBSLOT
 
 Set by GNU B<parallel> and can be used in jobs run by GNU B<parallel>.

testsuite/tests-to-run/parallel-local-ssh3.sh

@@ -1,58 +1,81 @@
 #!/bin/bash
 
-
 # SSH only allowed to localhost/lo
-cat <<'EOF' | parallel -vj100% --retries 3 -k --joblog /tmp/jl-`basename $0` -L1 -r
+
+par_trailing_space_sshlogin() {
     echo '### trailing space in sshlogin'
-  echo 'sshlogin trailing space' | parallel  --sshlogin "ssh -l parallel localhost " echo
+    echo 'sshlogin trailing space' |
+	parallel  --sshlogin "ssh -l parallel localhost " echo
+}
 
+par_special_char_trc() {
     echo '### Special char file and dir transfer return and cleanup'
-  cd /tmp; 
+    cd /tmp
-  mkdir -p d"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"; 
+    mkdir -p d"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"
-  echo local > d"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"/f"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"; 
+    echo local > d"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"/f"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"
-  ssh parallel@lo rm -rf d'*'/; 
+    ssh parallel@lo rm -rf d'*'/
     mytouch() {
-    cat d"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"/f"`perl -e 'print pack("c*",1..9,11..46,48..255)'`" > d"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"/g"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"; 
+	cat d"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"/f"`perl -e 'print pack("c*",1..9,11..46,48..255)'`" > d"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"/g"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"
-    echo remote OK >> d"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"/g"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"; 
+	echo remote OK >> d"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"/g"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"
-  }; 
+    }
-  export -f mytouch; 
+    export -f mytouch
-  parallel --env mytouch -Sparallel@lo --transfer 
+    parallel --env mytouch -Sparallel@lo --transfer --return {=s:/f:/g:=} mytouch \
-    --return {=s:/f:/g:=} 
+	     ::: d"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"/f"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"
-    mytouch 
-    ::: d"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"/f"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"; 
     cat d"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"/g"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"
+    # TODO Should be changed to --return '{=s:/f:/g:=}' and tested with csh - is error code kept?
+}
 
+par_rpl_perlexpr_not_used_in_command() {
     echo '### Uniq {=perlexpr=} in return - not used in command'
-  cd /tmp; 
+    cd /tmp
-  rm -f /tmp/parallel_perlexpr.2Parallel_PerlexPr; 
+    rm -f /tmp/parallel_perlexpr.2Parallel_PerlexPr
-  echo local > parallel_perlexpr; 
+    echo local > parallel_perlexpr
-  parallel -Sparallel@lo --trc {=s/pr/pr.2/=}{=s/p/P/g=} echo remote OK '>' {}.2{=s/p/P/g=} ::: parallel_perlexpr; 
+    parallel -Sparallel@lo --trc {=s/pr/pr.2/=}{=s/p/P/g=} echo remote OK '>' {}.2{=s/p/P/g=} ::: parallel_perlexpr
-  cat /tmp/parallel_perlexpr.2Parallel_PerlexPr; 
+    cat /tmp/parallel_perlexpr.2Parallel_PerlexPr
     rm -f /tmp/parallel_perlexpr.2Parallel_PerlexPr /tmp/parallel_perlexpr
 
-#  Should be changed to --return '{=s:/f:/g:=}' and tested with csh - is error code kept?
+}
 
+par_remote_function_nice() {
     echo '### functions and --nice'
-  myfunc() { echo OK $*; }; export -f myfunc; parallel --nice 10 --env myfunc -S parallel@lo myfunc ::: func
+    myfunc() {
+	echo OK $*
+    }
+    export -f myfunc
+    parallel --nice 10 --env myfunc -S parallel@lo myfunc ::: func
+}
 
+par_rplstr_return() {
     echo '### bug #45906: {= in header =}'
-  rm -f returnfile45906; 
+    rm -f returnfile45906
-  parallel --rpl '{G} $_=lc($_)' -S parallel@lo --return {G} --cleanup echo {G} '>' {G} ::: RETURNFILE45906; 
+    parallel --rpl '{G} $_=lc($_)' -S parallel@lo --return {G} --cleanup echo {G} '>' {G} ::: RETURNFILE45906
     ls returnfile45906
+}
 
+par_nonall_should_not_block() {
     echo "### bug #47608: parallel --nonall -S lo 'echo ::: ' blocks"
     parallel --nonall -S lo 'echo ::: '
+}
 
+par_export_functions_csh() {
     echo '### exported function to csh but with PARALLEL_SHELL=bash'
-  doit() { echo "$1"; }; 
+    doit() { echo "$1"; }
-  export -f doit; 
+    export -f doit
-  stdout parallel --env doit -S csh@lo doit ::: not_OK; 
+    stdout parallel --env doit -S csh@lo doit ::: not_OK
     PARALLEL_SHELL=bash parallel --env doit -S csh@lo doit ::: OK
+}
 
+
+par_progress_text_max_jobs_to_run() {
     echo '### bug #49404: "Max jobs to run" does not equal the number of jobs specified when using GNU Parallel on remote server?'
     echo should give 10 running jobs
     stdout parallel -S 16/lo --progress true ::: {1..10} | grep /.10
-EOF
+}
+
+par_hgrp_rpl() {
+    echo '### Implement {hgrp} replacement string'
+    parallel -k --plus --hgrp -S @b/bash@lo -S @c/csh@lo 'echo {sshlogin} {hgrp}' ::: b@b c@c
+}
 
 par_header_in_return() {
     echo '### bug #45907: --header : + --return {header}'

testsuite/wanted-results/parallel-local-ssh3

@@ -1,49 +1,34 @@
-echo '### trailing space in sshlogin'
+par_export_functions_csh	### exported function to csh but with PARALLEL_SHELL=bash
-### trailing space in sshlogin
+par_export_functions_csh	CSH/TCSH DO NOT SUPPORT newlines IN VARIABLES/FUNCTIONS. Unset doit
-  echo 'sshlogin trailing space' | parallel  --sshlogin "ssh -l parallel localhost " echo
+par_export_functions_csh	OK
-sshlogin trailing space
-echo '### Special char file and dir transfer return and cleanup'
-### Special char file and dir transfer return and cleanup
-  cd /tmp;   mkdir -p d"`perl -e 'print pack("c*",1..9,11..46,48..255)'`";   echo local > d"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"/f"`perl -e 'print pack("c*",1..9,11..46,48..255)'`";   ssh parallel@lo rm -rf d'*'/;   mytouch() {     cat d"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"/f"`perl -e 'print pack("c*",1..9,11..46,48..255)'`" > d"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"/g"`perl -e 'print pack("c*",1..9,11..46,48..255)'`";     echo remote OK >> d"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"/g"`perl -e 'print pack("c*",1..9,11..46,48..255)'`";   };   export -f mytouch;   parallel --env mytouch -Sparallel@lo --transfer     --return {=s:/f:/g:=}     mytouch     ::: d"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"/f"`perl -e 'print pack("c*",1..9,11..46,48..255)'`";   cat d"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"/g"`perl -e 'print pack("c*",1..9,11..46,48..255)'`"
-local
-remote OK
-echo '### Uniq {=perlexpr=} in return - not used in command'
-### Uniq {=perlexpr=} in return - not used in command
-  cd /tmp;   rm -f /tmp/parallel_perlexpr.2Parallel_PerlexPr;   echo local > parallel_perlexpr;   parallel -Sparallel@lo --trc {=s/pr/pr.2/=}{=s/p/P/g=} echo remote OK '>' {}.2{=s/p/P/g=} ::: parallel_perlexpr;   cat /tmp/parallel_perlexpr.2Parallel_PerlexPr;   rm -f /tmp/parallel_perlexpr.2Parallel_PerlexPr /tmp/parallel_perlexpr
-remote OK
-#  Should be changed to --return '{=s:/f:/g:=}' and tested with csh - is error code kept?
-echo '### functions and --nice'
-### functions and --nice
-  myfunc() { echo OK $*; }; export -f myfunc; parallel --nice 10 --env myfunc -S parallel@lo myfunc ::: func
-OK func
-echo '### bug #45906: {= in header =}'
-### bug #45906: {= in header =}
-  rm -f returnfile45906;   parallel --rpl '{G} $_=lc($_)' -S parallel@lo --return {G} --cleanup echo {G} '>' {G} ::: RETURNFILE45906;   ls returnfile45906
-returnfile45906
-echo "### bug #47608: parallel --nonall -S lo 'echo ::: ' blocks"
-### bug #47608: parallel --nonall -S lo 'echo ::: ' blocks
-  parallel --nonall -S lo 'echo ::: '
-:::
-echo '### exported function to csh but with PARALLEL_SHELL=bash'
-### exported function to csh but with PARALLEL_SHELL=bash
-  doit() { echo "$1"; };   export -f doit;   stdout parallel --env doit -S csh@lo doit ::: not_OK;   PARALLEL_SHELL=bash parallel --env doit -S csh@lo doit ::: OK
-CSH/TCSH DO NOT SUPPORT newlines IN VARIABLES/FUNCTIONS. Unset doit
-OK
-echo '### bug #49404: "Max jobs to run" does not equal the number of jobs specified when using GNU Parallel on remote server?'
-### bug #49404: "Max jobs to run" does not equal the number of jobs specified when using GNU Parallel on remote server?
-  echo should give 10 running jobs
-should give 10 running jobs
-  stdout parallel -S 16/lo --progress true ::: {1..10} | grep /.10
-1:lo / 16 / 10
 par_header_in_return	### bug #45907: --header : + --return {header}
 par_header_in_return	returnfile45907
+par_hgrp_rpl	### Implement {hgrp} replacement string
+par_hgrp_rpl	bash@lo b
+par_hgrp_rpl	csh@lo c
 par_hostgroup_only_on_args	### Auto add hostgroup if given on on argument
 par_hostgroup_only_on_args	parallel: Warning: Adding hostgroups: sh@lo
 par_hostgroup_only_on_args	sh
+par_nonall_should_not_block	### bug #47608: parallel --nonall -S lo 'echo ::: ' blocks
+par_nonall_should_not_block	:::
+par_progress_text_max_jobs_to_run	### bug #49404: "Max jobs to run" does not equal the number of jobs specified when using GNU Parallel on remote server?
+par_progress_text_max_jobs_to_run	should give 10 running jobs
+par_progress_text_max_jobs_to_run	1:lo / 16 / 10
 par_quoting_for_onall	### bug #35427: quoting of {2} broken for --onall
 par_quoting_for_onall	/bin/ls
+par_remote_function_nice	### functions and --nice
+par_remote_function_nice	OK func
 par_return_with_fixedstring	### Test --return with fixed string (Gave undef warnings)
 par_return_with_fixedstring	OK
+par_rpl_perlexpr_not_used_in_command	### Uniq {=perlexpr=} in return - not used in command
+par_rpl_perlexpr_not_used_in_command	remote OK
+par_rplstr_return	### bug #45906: {= in header =}
+par_rplstr_return	returnfile45906
+par_special_char_trc	### Special char file and dir transfer return and cleanup
+par_special_char_trc	local
+par_special_char_trc	remote OK
+par_trailing_space_sshlogin	### trailing space in sshlogin
+par_trailing_space_sshlogin	sshlogin trailing space
 par_trc_with_space	### Test --trc with space added in filename
 par_trc_with_space	original
 par_trc_with_special_chars	### Test --trc with >|< added in filename