mirror of
https://git.savannah.gnu.org/git/parallel.git
synced 2024-11-25 23:47:53 +00:00
parallel: Exponential backoff and backon sleep between jobs.
This commit is contained in:
parent
0cb0cdde3e
commit
85ab16e26a
|
@ -1,3 +1,4 @@
|
||||||
|
replace: true
|
||||||
version: 1.1
|
version: 1.1
|
||||||
directory: parallel
|
directory: parallel
|
||||||
filename: parallel-20100424.tar.bz2
|
filename: parallel-20100424.tar.bz2
|
||||||
|
|
26
src/parallel
26
src/parallel
|
@ -2118,16 +2118,19 @@ sub hostname {
|
||||||
|
|
||||||
sub reap_usleep {
|
sub reap_usleep {
|
||||||
# Reap dead children.
|
# Reap dead children.
|
||||||
# If no children: Sleep specified amount with exponential backoff
|
# If no dead children: Sleep specified amount with exponential backoff
|
||||||
# Returns:
|
# Returns:
|
||||||
# 0.00001 if children reaped (0.00001 ms works best on highend)
|
# $ms/2+0.001 if children reaped
|
||||||
# $ms*1.1 if no children reaped
|
# $ms*1.1 if no children reaped
|
||||||
my $ms = shift;
|
my $ms = shift;
|
||||||
if(reaper()) {
|
if(reaper()) {
|
||||||
return 0.00001;
|
# Sleep exponentially shorter (1/2^n) if a job finished
|
||||||
|
return $ms/2+0.001;
|
||||||
} else {
|
} else {
|
||||||
usleep($ms);
|
usleep($ms);
|
||||||
return (($ms < 1000) ? ($ms * 1.1) : ($ms)); # exponential back off
|
# Sleep exponentially longer (1.1^n) if a job did not finish
|
||||||
|
# though at most 1000 ms.
|
||||||
|
return (($ms < 1000) ? ($ms * 1.1) : ($ms));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -2813,7 +2816,7 @@ sub simultaneous_sshlogin {
|
||||||
my $serverlogin = $self->serverlogin();
|
my $serverlogin = $self->serverlogin();
|
||||||
my $sshdelay = $opt::sshdelay ? "sleep $opt::sshdelay;" : "";
|
my $sshdelay = $opt::sshdelay ? "sleep $opt::sshdelay;" : "";
|
||||||
my $cmd = "$sshdelay$sshcmd $serverlogin echo simultaneouslogin </dev/null 2>&1 &"x$wanted_processes;
|
my $cmd = "$sshdelay$sshcmd $serverlogin echo simultaneouslogin </dev/null 2>&1 &"x$wanted_processes;
|
||||||
::debug("Trying $wanted_processes logins at $serverlogin");
|
::debug("Trying $wanted_processes logins at $serverlogin\n");
|
||||||
open (my $simul_fh, "-|", "($cmd)|grep simultaneouslogin | wc -l") or
|
open (my $simul_fh, "-|", "($cmd)|grep simultaneouslogin | wc -l") or
|
||||||
::die_bug("simultaneouslogin");
|
::die_bug("simultaneouslogin");
|
||||||
my $ssh_limit = <$simul_fh>;
|
my $ssh_limit = <$simul_fh>;
|
||||||
|
@ -3606,7 +3609,7 @@ sub runtime {
|
||||||
# Returns:
|
# Returns:
|
||||||
# Run time in seconds
|
# Run time in seconds
|
||||||
my $self = shift;
|
my $self = shift;
|
||||||
return $self->endtime() - $self->starttime();
|
return int(($self->endtime() - $self->starttime())*1000)/1000;
|
||||||
}
|
}
|
||||||
|
|
||||||
sub endtime {
|
sub endtime {
|
||||||
|
@ -4176,7 +4179,10 @@ sub start {
|
||||||
$Global::timeoutq->insert($job);
|
$Global::timeoutq->insert($job);
|
||||||
}
|
}
|
||||||
if($opt::delay) {
|
if($opt::delay) {
|
||||||
::usleep($opt::delay*1000);
|
my $now = ::now();
|
||||||
|
$Global::last_job_started_at ||= $now;
|
||||||
|
::usleep(($opt::delay - ($now - $Global::last_job_started_at))*1000);
|
||||||
|
$Global::last_job_started_at = $now;
|
||||||
}
|
}
|
||||||
return $job;
|
return $job;
|
||||||
} else {
|
} else {
|
||||||
|
@ -5017,7 +5023,7 @@ sub unget {
|
||||||
sub empty {
|
sub empty {
|
||||||
my $self = shift;
|
my $self = shift;
|
||||||
my $empty = (not @{$self->{'unget'}}) && $self->{'arg_queue'}->empty();
|
my $empty = (not @{$self->{'unget'}}) && $self->{'arg_queue'}->empty();
|
||||||
::debug("CommandLineQueue->empty $empty\n");
|
::debug("CommandLineQueue->empty $empty");
|
||||||
return $empty;
|
return $empty;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -5179,7 +5185,7 @@ sub empty {
|
||||||
my $self = shift;
|
my $self = shift;
|
||||||
my $empty = not @{$self->{'unget'}};
|
my $empty = not @{$self->{'unget'}};
|
||||||
$empty &&= $self->{'arg_sub_queue'}->empty();
|
$empty &&= $self->{'arg_sub_queue'}->empty();
|
||||||
::debug("RecordQueue->empty $empty\n");
|
::debug("RecordQueue->empty $empty");
|
||||||
return $empty;
|
return $empty;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -5291,7 +5297,7 @@ sub empty {
|
||||||
for my $fh (@{$self->{'fhs'}}) {
|
for my $fh (@{$self->{'fhs'}}) {
|
||||||
$empty &&= eof($fh);
|
$empty &&= eof($fh);
|
||||||
}
|
}
|
||||||
::debug("MultifileQueue->empty $empty\n");
|
::debug("MultifileQueue->empty $empty");
|
||||||
return $empty;
|
return $empty;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -220,4 +220,4 @@ EOF
|
||||||
echo '### Test of -j filename with file content changing';
|
echo '### Test of -j filename with file content changing';
|
||||||
echo 1 >/tmp/jobs_to_run2;
|
echo 1 >/tmp/jobs_to_run2;
|
||||||
(sleep 3; echo 10 >/tmp/jobs_to_run2) &
|
(sleep 3; echo 10 >/tmp/jobs_to_run2) &
|
||||||
parallel -j /tmp/jobs_to_run2 -v sleep {} ::: 3.3 1.21 1.43 1.54 1.32 1 1 1 1 1 1 1 1 1 1 1
|
parallel -j /tmp/jobs_to_run2 -v sleep {} ::: 3.3 1.5 1.5 1.5 1.5 1 1 1 1 1 1 1 1 1 1 1
|
||||||
|
|
|
@ -1,7 +1,7 @@
|
||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
|
|
||||||
P="scosysv centos dragonfly netbsd freebsd solaris openbsd debian aix hpux qnx irix suse minix openindiana mandriva unixware miros raspberrypi hurd ultrix ubuntu redhat"
|
P="scosysv centos dragonfly netbsd freebsd solaris openbsd debian aix hpux qnx irix suse minix openindiana mandriva unixware miros raspberrypi hurd ultrix ubuntu redhat"
|
||||||
P="scosysv centos dragonfly netbsd freebsd solaris openbsd debian aix hpux qnx irix suse minix openindiana mandriva unixware miros raspberrypi hurd ultrix ubuntu"
|
P="scosysv centos dragonfly netbsd freebsd solaris openbsd debian aix hpux qnx irix suse minix openindiana mandriva unixware raspberrypi hurd ultrix ubuntu"
|
||||||
#P="scosysv hpux qnx irix openindiana ultrix"
|
#P="scosysv hpux qnx irix openindiana ultrix"
|
||||||
POLAR=`parallel echo {}.polarhome.com ::: $P`
|
POLAR=`parallel echo {}.polarhome.com ::: $P`
|
||||||
|
|
||||||
|
|
|
@ -323,10 +323,10 @@ sleep 1
|
||||||
sleep 1
|
sleep 1
|
||||||
sleep 1
|
sleep 1
|
||||||
sleep 1
|
sleep 1
|
||||||
sleep 1.21
|
sleep 1.5
|
||||||
sleep 1.32
|
sleep 1.5
|
||||||
sleep 1.43
|
sleep 1.5
|
||||||
sleep 1.54
|
sleep 1.5
|
||||||
sleep 1
|
sleep 1
|
||||||
sleep 1
|
sleep 1
|
||||||
sleep 1
|
sleep 1
|
||||||
|
|
Loading…
Reference in a new issue