tange/parallel
1
0
Fork 0
mirror of https://git.savannah.gnu.org/git/parallel.git synced 2024-11-22 22:17:54 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

src/parallel: If --filter-hosts (and thus time_to_login) do not try to find the ssh spawning limit.

Browse source
This commit is contained in:
Ole Tange 2013-06-30 20:09:02 +02:00
parent 822f0d4d73
commit b6e210af96
1 changed files with 17 additions and 8 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

25
src/parallel
View file

@ -112,7 +112,7 @@ if($opt::filter_hosts and (@opt::sshlogin or @opt::sshloginfile)) {
my ($fh, $tmpfile) = ::tempfile(SUFFIX => ".ssh"); my ($fh, $tmpfile) = ::tempfile(SUFFIX => ".ssh");
print $fh @cores, @cpus, @maxline, @echo; print $fh @cores, @cpus, @maxline, @echo;
close $fh; close $fh;
my $cmd = "cat $tmpfile | $0 -j0 -s 1000 --joblog - --plain --tag --tagstring {1} --colsep '\t' -k eval {2}"; my $cmd = "cat $tmpfile | $0 -j0 -s 1000 --joblog - --plain --tag --tagstring {1} --colsep '\t' -k eval {2} 2>/dev/null";
::debug($cmd."\n"); ::debug($cmd."\n");
open(my $host_fh, "-|", $cmd) || ::die_bug("parallel host check: $cmd"); open(my $host_fh, "-|", $cmd) || ::die_bug("parallel host check: $cmd");
my (%ncores, %ncpus, %time_to_login, %maxlen, %echo); my (%ncores, %ncpus, %time_to_login, %maxlen, %echo);
@ -142,9 +142,8 @@ if($opt::filter_hosts and (@opt::sshlogin or @opt::sshloginfile)) {
$ncpus{$col[1]} = 1; $ncpus{$col[1]} = 1;
$maxlen{$col[1]} = Limits::Command::max_length(); $maxlen{$col[1]} = Limits::Command::max_length();
} elsif($col[0] =~ /^\d+$/ and $Global::host{$host}) { } elsif($col[0] =~ /^\d+$/ and $Global::host{$host}) {
# 1 server 1338156112.05 0.303 0 0 0 0
# parallel --number-of-cores ; parallel --number-of-cpus
# Remember how log it took to log in # Remember how log it took to log in
# 2 : 1372607672.654 0.675 0 0 0 0 eval true\ m\;ssh\ m\ echo
$time_to_login{$host} = ::min($time_to_login{$host},$col[3]); $time_to_login{$host} = ::min($time_to_login{$host},$col[3]);
} else { } else {
::die_bug("host check unmatched long jobline: $_"); ::die_bug("host check unmatched long jobline: $_");
@ -1341,6 +1340,7 @@ sub start_more_jobs {
next; next;
} }
if($sshlogin->too_fast_remote_login()) { if($sshlogin->too_fast_remote_login()) {
# It has been too short since
next; next;
} }
debug($sshlogin->string()." has ".$sshlogin->jobs_running() debug($sshlogin->string()." has ".$sshlogin->jobs_running()
@ -1458,7 +1458,9 @@ sub drain_job_queue {
and not $Global::JobQueue->empty()) { and not $Global::JobQueue->empty()) {
# These jobs may not be started because of loadavg # These jobs may not be started because of loadavg
# or too little time between each ssh login. # or too little time between each ssh login.
start_more_jobs(); if(start_more_jobs() > 0) {
$sleep = $sleep/2;
}
} }
# Sometimes SIGCHLD is not registered, so force reaper # Sometimes SIGCHLD is not registered, so force reaper
$sleep = ::reap_usleep($sleep); $sleep = ::reap_usleep($sleep);
@ -2450,10 +2452,13 @@ sub swap_activity {
sub too_fast_remote_login { sub too_fast_remote_login {
my $self = shift; my $self = shift;
if($self->{'last_login_at'} and $self->{'time_to_login'}) { if($self->{'last_login_at'} and $self->{'time_to_login'}) {
# If now <= last_login + wait time: Then it is too soon. # sshd normally allows 10 simultaneous logins
# A login takes time_to_login
# So time_to_login/5 should be safe
# If now <= last_login + time_to_login/5: Then it is too soon.
my $too_fast = (::now() <= $self->{'last_login_at'} my $too_fast = (::now() <= $self->{'last_login_at'}
+ $self->{'time_to_login'}); + $self->{'time_to_login'}/5);
::debug("Too fast? $too_fast\n"); ::debug("Too fast? $too_fast ");
return $too_fast; return $too_fast;
} else { } else {
# No logins so far (or time_to_login not computed): it is not too fast # No logins so far (or time_to_login not computed): it is not too fast
@ -2805,6 +2810,10 @@ sub simultaneous_sshlogin_limit {
# min($wanted_processes,$working_simultaneous_ssh_logins-1) # min($wanted_processes,$working_simultaneous_ssh_logins-1)
my $self = shift; my $self = shift;
my $wanted_processes = shift; my $wanted_processes = shift;
if($self->{'time_to_login'}) {
return $wanted_processes;
}
# Try twice because it guesses wrong sometimes # Try twice because it guesses wrong sometimes
# Choose the minimal # Choose the minimal
my $ssh_limit = my $ssh_limit =
@ -3446,7 +3455,7 @@ sub empty {
my $self = shift; my $self = shift;
my $empty = (not @{$self->{'unget'}}) my $empty = (not @{$self->{'unget'}})
&& $self->{'commandlinequeue'}->empty(); && $self->{'commandlinequeue'}->empty();
::debug("JobQueue->empty $empty\n"); ::debug("JobQueue->empty $empty ");
return $empty; return $empty;
} }