src/parallel: -S @hostgroup implemented.

src/parallel

@@ -394,11 +394,12 @@ sub spreadstdin {
 		    "Increasing to --blocksize $blocksize\n");
       }
     }
+    ::debug("init", "Done reading input\n");
+
     # If there is anything left in the buffer write it
     substr($buf,0,0) = "";
     write_record_to_pipe($chunk_number++,\$header,\$buf,$recstart,$recend,length $buf);
 
-    ::debug("init", "Done reading input\n");
     $Global::start_no_new_jobs ||= 1;
     if($opt::roundrobin) {
 	for my $job (values %Global::running) {
@@ -2303,9 +2304,15 @@ sub parse_sshlogin {
         }
     }
     $Global::minimal_command_line_length = 8_000_000;
+    my @allowed_hostgroups;
     for my $ncpu_sshlogin_string (::uniq(@login)) {
 	my $sshlogin = SSHLogin->new($ncpu_sshlogin_string);
 	my $sshlogin_string = $sshlogin->string();
+	if($sshlogin_string eq "") {
+	    # This is an ssh group: -S @webservers
+	    push @allowed_hostgroups, $sshlogin->hostgroups();
+	    next;
+	}
 	if($Global::host{$sshlogin_string}) {
 	    # This sshlogin has already been added:
 	    # It is probably a host that has come back
@@ -2328,6 +2335,15 @@ sub parse_sshlogin {
 	    ::min($Global::minimal_command_line_length, $sshlogin->maxlength());
         $Global::host{$sshlogin_string} = $sshlogin;
     }
+    if(@allowed_hostgroups) {
+	# Remove hosts that are not in these groups
+	while (my ($string, $sshlogin) = each %Global::host) {
+	    if(not $sshlogin->in_hostgroups(@allowed_hostgroups)) {
+		delete $Global::host{$string};
+	    }
+	}
+    }
+
     # debug("start", "sshlogin: ", my_dump(%Global::host),"\n");
     if($opt::transfer or @opt::return or $opt::cleanup or @opt::basefile) {
         if(not remote_hosts()) {
@@ -3366,7 +3382,7 @@ sub new {
     #   user@server
     #   ncpu/user@server
     #   @grp+grp/user@server
-    if($sshlogin_string =~ s:^\@([^/]+)/::) {
+    if($sshlogin_string =~ s:^\@([^/]+)/?::) {
         # Look for SSHLogin hostgroups
         %hostgroups = map { $_ => 1 } split(/\+/, $1);
     }
@@ -3459,6 +3475,11 @@ sub in_hostgroups {
     return grep { defined $self->{'hostgroups'}{$_} } @_;
 }
 
+sub hostgroups {
+    my $self = shift;
+    return keys %{$self->{'hostgroups'}};
+}
+
 sub inc_jobs_completed {
     my $self = shift;
     $self->{'jobs_completed'}++;
@@ -3478,7 +3499,6 @@ sub set_max_jobs_running {
     $self->{'orig_max_jobs_running'} ||= $self->{'max_jobs_running'};
 }
 
-
 sub swapping {
     my $self = shift;
     my $swapping = $self->swap_activity();

src/parallel.pod

@@ -1590,6 +1590,9 @@ If I<hostgroups> is given, the I<sshlogin> will be added to that
 hostgroup. Multiple hostgroups are separated by '+'. The I<sshlogin>
 will always be added to a hostgroup named the same as I<sshlogin>.
 
+If only the I<hostgroups> is given, only the sshlogins in those
+hostgroups will be used.
+
 GNU B<parallel> will determine the number of CPU cores on the remote
 computers and run the number of jobs as specified by B<-j>.  If the
 number I<ncpu> is given GNU B<parallel> will use this number for

testsuite/tests-to-run/niceload04.sh

@@ -1,5 +1,8 @@
 #!/bin/bash
 
+cp /bin/sleep /tmp/mysleep
+killall -9 mysleep 2>/dev/null
+
 # force load > 10
 while uptime | grep -v age:.[1-9][0-9].[0-9][0-9] >/dev/null ; do (timeout 5 nice burnP6 2>/dev/null &) done
 
@@ -10,10 +13,10 @@ export PID2=$!
 sleep 2 &
 export PID3=$!
 echo '### multiple -p'
-  stdout /usr/bin/time -f %e niceload -l 9 -H -p $PID1 -p $PID2 -p $PID3 | perl -ne '$_ => 5 and print "Multiple -p OK\n"'
+  stdout /usr/bin/time -f %e niceload -l 9 -H -p $PID1 -p $PID2 -p $PID3 | perl -ne '$_ > 5 and print "Multiple -p OK\n"' &
 
-sleep 2 &
-sleep 2 &
-sleep 2 &
+/tmp/mysleep 2 &
+/tmp/mysleep 2 &
+/tmp/mysleep 2 &
 echo '### --prg'
-  stdout /usr/bin/time -f %e niceload -l 7 -H --prg sleep | perl -ne '$_ => 5 and print "--prg OK\n"'
+  stdout /usr/bin/time -f %e niceload -l 8 -H --prg mysleep | perl -ne '$_ > 5 and print "--prg OK\n"'

testsuite/tests-to-run/parallel-local-ssh2.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | stdout parallel -vj7 -k -L1
+cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | stdout parallel -vj6 -k -L1
 echo "### bug #43518: GNU Parallel doesn't obey servers' jobs capacity when an ssh login file is reloaded"
   # Pre-20141106 Would reset the number of jobs run on all sshlogin if --slf changed
   # Thus must take at least 25 sec to run

testsuite/tests-to-run/parallel-local-ssh3.sh

@@ -24,4 +24,13 @@ echo '### --hostgroup'
 echo '### --hostgroup --sshlogin with @'
   parallel -k --hostgroups -S parallel@lo echo ::: no_group implicit_group@parallel@lo
 
+echo '### --hostgroup -S @group'
+  parallel -S @g1/ -S @g1/1/tcsh@lo -S @g1/1/localhost -S @g2/1/parallel@lo whoami\;true ::: {1..6} | sort
+
+echo '### --hostgroup -S @group1 -Sgrp2'
+  parallel -S @g1/ -S @g2 -S @g1/1/tcsh@lo -S @g1/1/localhost -S @g2/1/parallel@lo whoami\;true ::: {1..6} | sort
+
+echo '### --hostgroup -S @group1+grp2'
+  parallel -S @g1+g2 -S @g1/1/tcsh@lo -S @g1/1/localhost -S @g2/1/parallel@lo whoami\;true ::: {1..6} | sort
+
 EOF

testsuite/tests-to-run/parallel-remote1.sh

@@ -14,11 +14,6 @@ chmod 755 /tmp/myssh1 /tmp/myssh2
 seq 1 100 | parallel --sshdelay 0.05 --sshlogin "/tmp/myssh1 $SSHLOGIN1,/tmp/myssh2 $SSHLOGIN2" -k echo
 
 cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/\;s/\$SSHLOGIN1/$SSHLOGIN1/\;s/\$SSHLOGIN2/$SSHLOGIN2/\;s/\$SSHLOGIN3/$SSHLOGIN3/ | parallel -vj2 -k -L1
-echo '### bug #41964: --controlmaster not seems to reuse OpenSSH connections to the same host'
-  (parallel -S redhat9.tange.dk true ::: {1..20}; echo No --controlmaster - finish last) & 
-  (parallel -M -S redhat9.tange.dk true ::: {1..20}; echo With --controlmaster - finish first) & 
-  wait
-
 echo '### --filter-hosts - OK, non-such-user, connection refused, wrong host'
   parallel --nonall --filter-hosts -S localhost,NoUser@localhost,154.54.72.206,"ssh 5.5.5.5" hostname
 
@@ -34,4 +29,9 @@ echo '### test --filter-hosts with server w/o ssh, non-existing server'
 
 echo '### Missing: test --filter-hosts proxied through the one host'
 
+echo '### bug #41964: --controlmaster not seems to reuse OpenSSH connections to the same host'
+  (parallel -S redhat9.tange.dk true ::: {1..20}; echo No --controlmaster - finish last) & 
+  (parallel -M -S redhat9.tange.dk true ::: {1..20}; echo With --controlmaster - finish first) & 
+  wait
+
 EOF

testsuite/wanted-results/niceload04

@@ -1,4 +1,4 @@
 ### multiple -p
-Multiple -p OK
 ### --prg
+Multiple -p OK
 --prg OK

testsuite/wanted-results/parallel-local-ssh3

@@ -60,3 +60,30 @@ echo '### --hostgroup --sshlogin with @'
   parallel -k --hostgroups -S parallel@lo echo ::: no_group implicit_group@parallel@lo
 no_group
 implicit_group
+echo '### --hostgroup -S @group'
+### --hostgroup -S @group
+  parallel -S @g1/ -S @g1/1/tcsh@lo -S @g1/1/localhost -S @g2/1/parallel@lo whoami\;true ::: {1..6} | sort
+tange
+tange
+tange
+tcsh
+tcsh
+tcsh
+echo '### --hostgroup -S @group1 -Sgrp2'
+### --hostgroup -S @group1 -Sgrp2
+  parallel -S @g1/ -S @g2 -S @g1/1/tcsh@lo -S @g1/1/localhost -S @g2/1/parallel@lo whoami\;true ::: {1..6} | sort
+parallel
+parallel
+tange
+tange
+tcsh
+tcsh
+echo '### --hostgroup -S @group1+grp2'
+### --hostgroup -S @group1+grp2
+  parallel -S @g1+g2 -S @g1/1/tcsh@lo -S @g1/1/localhost -S @g2/1/parallel@lo whoami\;true ::: {1..6} | sort
+parallel
+parallel
+tange
+tange
+tcsh
+tcsh