From 2fb56b5755940e926fc50ac76dc1a2e9cc703d16 Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Thu, 13 Nov 2014 16:57:41 +0100 Subject: [PATCH] src/parallel: -S @hostgroup implemented. --- src/parallel | 26 +++++++++++++++--- src/parallel.pod | 3 +++ testsuite/tests-to-run/niceload04.sh | 13 +++++---- testsuite/tests-to-run/parallel-local-ssh2.sh | 2 +- testsuite/tests-to-run/parallel-local-ssh3.sh | 9 +++++++ testsuite/tests-to-run/parallel-remote1.sh | 10 +++---- testsuite/wanted-results/niceload04 | 2 +- testsuite/wanted-results/parallel-local-ssh3 | 27 +++++++++++++++++++ 8 files changed, 77 insertions(+), 15 deletions(-) diff --git a/src/parallel b/src/parallel index 965c23e4..cc8feb7b 100755 --- a/src/parallel +++ b/src/parallel @@ -394,11 +394,12 @@ sub spreadstdin { "Increasing to --blocksize $blocksize\n"); } } + ::debug("init", "Done reading input\n"); + # If there is anything left in the buffer write it substr($buf,0,0) = ""; write_record_to_pipe($chunk_number++,\$header,\$buf,$recstart,$recend,length $buf); - ::debug("init", "Done reading input\n"); $Global::start_no_new_jobs ||= 1; if($opt::roundrobin) { for my $job (values %Global::running) { @@ -2303,9 +2304,15 @@ sub parse_sshlogin { } } $Global::minimal_command_line_length = 8_000_000; + my @allowed_hostgroups; for my $ncpu_sshlogin_string (::uniq(@login)) { my $sshlogin = SSHLogin->new($ncpu_sshlogin_string); my $sshlogin_string = $sshlogin->string(); + if($sshlogin_string eq "") { + # This is an ssh group: -S @webservers + push @allowed_hostgroups, $sshlogin->hostgroups(); + next; + } if($Global::host{$sshlogin_string}) { # This sshlogin has already been added: # It is probably a host that has come back @@ -2328,6 +2335,15 @@ sub parse_sshlogin { ::min($Global::minimal_command_line_length, $sshlogin->maxlength()); $Global::host{$sshlogin_string} = $sshlogin; } + if(@allowed_hostgroups) { + # Remove hosts that are not in these groups + while (my ($string, $sshlogin) = each %Global::host) { + if(not $sshlogin->in_hostgroups(@allowed_hostgroups)) { + delete $Global::host{$string}; + } + } + } + # debug("start", "sshlogin: ", my_dump(%Global::host),"\n"); if($opt::transfer or @opt::return or $opt::cleanup or @opt::basefile) { if(not remote_hosts()) { @@ -3366,7 +3382,7 @@ sub new { # user@server # ncpu/user@server # @grp+grp/user@server - if($sshlogin_string =~ s:^\@([^/]+)/::) { + if($sshlogin_string =~ s:^\@([^/]+)/?::) { # Look for SSHLogin hostgroups %hostgroups = map { $_ => 1 } split(/\+/, $1); } @@ -3459,6 +3475,11 @@ sub in_hostgroups { return grep { defined $self->{'hostgroups'}{$_} } @_; } +sub hostgroups { + my $self = shift; + return keys %{$self->{'hostgroups'}}; +} + sub inc_jobs_completed { my $self = shift; $self->{'jobs_completed'}++; @@ -3478,7 +3499,6 @@ sub set_max_jobs_running { $self->{'orig_max_jobs_running'} ||= $self->{'max_jobs_running'}; } - sub swapping { my $self = shift; my $swapping = $self->swap_activity(); diff --git a/src/parallel.pod b/src/parallel.pod index d6e8c785..4452a01c 100644 --- a/src/parallel.pod +++ b/src/parallel.pod @@ -1590,6 +1590,9 @@ If I is given, the I will be added to that hostgroup. Multiple hostgroups are separated by '+'. The I will always be added to a hostgroup named the same as I. +If only the I is given, only the sshlogins in those +hostgroups will be used. + GNU B will determine the number of CPU cores on the remote computers and run the number of jobs as specified by B<-j>. If the number I is given GNU B will use this number for diff --git a/testsuite/tests-to-run/niceload04.sh b/testsuite/tests-to-run/niceload04.sh index 9092125b..f7f766d6 100755 --- a/testsuite/tests-to-run/niceload04.sh +++ b/testsuite/tests-to-run/niceload04.sh @@ -1,5 +1,8 @@ #!/bin/bash +cp /bin/sleep /tmp/mysleep +killall -9 mysleep 2>/dev/null + # force load > 10 while uptime | grep -v age:.[1-9][0-9].[0-9][0-9] >/dev/null ; do (timeout 5 nice burnP6 2>/dev/null &) done @@ -10,10 +13,10 @@ export PID2=$! sleep 2 & export PID3=$! echo '### multiple -p' - stdout /usr/bin/time -f %e niceload -l 9 -H -p $PID1 -p $PID2 -p $PID3 | perl -ne '$_ => 5 and print "Multiple -p OK\n"' + stdout /usr/bin/time -f %e niceload -l 9 -H -p $PID1 -p $PID2 -p $PID3 | perl -ne '$_ > 5 and print "Multiple -p OK\n"' & -sleep 2 & -sleep 2 & -sleep 2 & +/tmp/mysleep 2 & +/tmp/mysleep 2 & +/tmp/mysleep 2 & echo '### --prg' - stdout /usr/bin/time -f %e niceload -l 7 -H --prg sleep | perl -ne '$_ => 5 and print "--prg OK\n"' + stdout /usr/bin/time -f %e niceload -l 8 -H --prg mysleep | perl -ne '$_ > 5 and print "--prg OK\n"' diff --git a/testsuite/tests-to-run/parallel-local-ssh2.sh b/testsuite/tests-to-run/parallel-local-ssh2.sh index a602c9f3..cccc9985 100644 --- a/testsuite/tests-to-run/parallel-local-ssh2.sh +++ b/testsuite/tests-to-run/parallel-local-ssh2.sh @@ -1,6 +1,6 @@ #!/bin/bash -cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | stdout parallel -vj7 -k -L1 +cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | stdout parallel -vj6 -k -L1 echo "### bug #43518: GNU Parallel doesn't obey servers' jobs capacity when an ssh login file is reloaded" # Pre-20141106 Would reset the number of jobs run on all sshlogin if --slf changed # Thus must take at least 25 sec to run diff --git a/testsuite/tests-to-run/parallel-local-ssh3.sh b/testsuite/tests-to-run/parallel-local-ssh3.sh index 958c9bc4..d90e5edc 100644 --- a/testsuite/tests-to-run/parallel-local-ssh3.sh +++ b/testsuite/tests-to-run/parallel-local-ssh3.sh @@ -24,4 +24,13 @@ echo '### --hostgroup' echo '### --hostgroup --sshlogin with @' parallel -k --hostgroups -S parallel@lo echo ::: no_group implicit_group@parallel@lo +echo '### --hostgroup -S @group' + parallel -S @g1/ -S @g1/1/tcsh@lo -S @g1/1/localhost -S @g2/1/parallel@lo whoami\;true ::: {1..6} | sort + +echo '### --hostgroup -S @group1 -Sgrp2' + parallel -S @g1/ -S @g2 -S @g1/1/tcsh@lo -S @g1/1/localhost -S @g2/1/parallel@lo whoami\;true ::: {1..6} | sort + +echo '### --hostgroup -S @group1+grp2' + parallel -S @g1+g2 -S @g1/1/tcsh@lo -S @g1/1/localhost -S @g2/1/parallel@lo whoami\;true ::: {1..6} | sort + EOF diff --git a/testsuite/tests-to-run/parallel-remote1.sh b/testsuite/tests-to-run/parallel-remote1.sh index 83c1e14d..d8d4c3f9 100644 --- a/testsuite/tests-to-run/parallel-remote1.sh +++ b/testsuite/tests-to-run/parallel-remote1.sh @@ -14,11 +14,6 @@ chmod 755 /tmp/myssh1 /tmp/myssh2 seq 1 100 | parallel --sshdelay 0.05 --sshlogin "/tmp/myssh1 $SSHLOGIN1,/tmp/myssh2 $SSHLOGIN2" -k echo cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/\;s/\$SSHLOGIN1/$SSHLOGIN1/\;s/\$SSHLOGIN2/$SSHLOGIN2/\;s/\$SSHLOGIN3/$SSHLOGIN3/ | parallel -vj2 -k -L1 -echo '### bug #41964: --controlmaster not seems to reuse OpenSSH connections to the same host' - (parallel -S redhat9.tange.dk true ::: {1..20}; echo No --controlmaster - finish last) & - (parallel -M -S redhat9.tange.dk true ::: {1..20}; echo With --controlmaster - finish first) & - wait - echo '### --filter-hosts - OK, non-such-user, connection refused, wrong host' parallel --nonall --filter-hosts -S localhost,NoUser@localhost,154.54.72.206,"ssh 5.5.5.5" hostname @@ -34,4 +29,9 @@ echo '### test --filter-hosts with server w/o ssh, non-existing server' echo '### Missing: test --filter-hosts proxied through the one host' +echo '### bug #41964: --controlmaster not seems to reuse OpenSSH connections to the same host' + (parallel -S redhat9.tange.dk true ::: {1..20}; echo No --controlmaster - finish last) & + (parallel -M -S redhat9.tange.dk true ::: {1..20}; echo With --controlmaster - finish first) & + wait + EOF diff --git a/testsuite/wanted-results/niceload04 b/testsuite/wanted-results/niceload04 index 1f46a994..59eb35d0 100644 --- a/testsuite/wanted-results/niceload04 +++ b/testsuite/wanted-results/niceload04 @@ -1,4 +1,4 @@ ### multiple -p -Multiple -p OK ### --prg +Multiple -p OK --prg OK diff --git a/testsuite/wanted-results/parallel-local-ssh3 b/testsuite/wanted-results/parallel-local-ssh3 index 303f4a9a..3773b912 100644 --- a/testsuite/wanted-results/parallel-local-ssh3 +++ b/testsuite/wanted-results/parallel-local-ssh3 @@ -60,3 +60,30 @@ echo '### --hostgroup --sshlogin with @' parallel -k --hostgroups -S parallel@lo echo ::: no_group implicit_group@parallel@lo no_group implicit_group +echo '### --hostgroup -S @group' +### --hostgroup -S @group + parallel -S @g1/ -S @g1/1/tcsh@lo -S @g1/1/localhost -S @g2/1/parallel@lo whoami\;true ::: {1..6} | sort +tange +tange +tange +tcsh +tcsh +tcsh +echo '### --hostgroup -S @group1 -Sgrp2' +### --hostgroup -S @group1 -Sgrp2 + parallel -S @g1/ -S @g2 -S @g1/1/tcsh@lo -S @g1/1/localhost -S @g2/1/parallel@lo whoami\;true ::: {1..6} | sort +parallel +parallel +tange +tange +tcsh +tcsh +echo '### --hostgroup -S @group1+grp2' +### --hostgroup -S @group1+grp2 + parallel -S @g1+g2 -S @g1/1/tcsh@lo -S @g1/1/localhost -S @g2/1/parallel@lo whoami\;true ::: {1..6} | sort +parallel +parallel +tange +tange +tcsh +tcsh