parallel: Exponential backoff and backon sleep between jobs.

doc/parallel.directive

@@ -1,3 +1,4 @@
+replace: true
 version: 1.1
 directory: parallel
 filename: parallel-20100424.tar.bz2

src/parallel

@@ -2118,16 +2118,19 @@ sub hostname {
 
 sub reap_usleep {
     # Reap dead children.
-    # If no children: Sleep specified amount with exponential backoff
+    # If no dead children: Sleep specified amount with exponential backoff
     # Returns:
-    #   0.00001 if children reaped (0.00001 ms works best on highend)
+    #   $ms/2+0.001 if children reaped
     #   $ms*1.1 if no children reaped
     my $ms = shift;
     if(reaper()) {
-	return 0.00001;
+	# Sleep exponentially shorter (1/2^n) if a job finished
+	return $ms/2+0.001;
     } else {
 	usleep($ms);
-	return (($ms < 1000) ? ($ms * 1.1) : ($ms)); # exponential back off
+	# Sleep exponentially longer (1.1^n) if a job did not finish
+	# though at most 1000 ms.
+	return (($ms < 1000) ? ($ms * 1.1) : ($ms));
     }
 }
 
@@ -2813,7 +2816,7 @@ sub simultaneous_sshlogin {
     my $serverlogin = $self->serverlogin();
     my $sshdelay = $opt::sshdelay ? "sleep $opt::sshdelay;" : "";
     my $cmd = "$sshdelay$sshcmd $serverlogin echo simultaneouslogin </dev/null 2>&1 &"x$wanted_processes;
-    ::debug("Trying $wanted_processes logins at $serverlogin");
+    ::debug("Trying $wanted_processes logins at $serverlogin\n");
     open (my $simul_fh, "-|", "($cmd)|grep simultaneouslogin | wc -l") or
 	::die_bug("simultaneouslogin");
     my $ssh_limit = <$simul_fh>;
@@ -3606,7 +3609,7 @@ sub runtime {
     # Returns:
     #   Run time in seconds
     my $self = shift;
-    return $self->endtime() - $self->starttime();
+    return int(($self->endtime() - $self->starttime())*1000)/1000;
 }
 
 sub endtime {
@@ -4176,7 +4179,10 @@ sub start {
 	    $Global::timeoutq->insert($job);
 	}
 	if($opt::delay) {
-	    ::usleep($opt::delay*1000);
+	    my $now = ::now();
+	    $Global::last_job_started_at ||= $now;
+	    ::usleep(($opt::delay - ($now - $Global::last_job_started_at))*1000);
+	    $Global::last_job_started_at = $now;
 	}
 	return $job;
     } else {
@@ -5017,7 +5023,7 @@ sub unget {
 sub empty {
     my $self = shift;
     my $empty = (not @{$self->{'unget'}}) && $self->{'arg_queue'}->empty();
-    ::debug("CommandLineQueue->empty $empty\n");
+    ::debug("CommandLineQueue->empty $empty");
     return $empty;
 }
 
@@ -5179,7 +5185,7 @@ sub empty {
     my $self = shift;
     my $empty = not @{$self->{'unget'}};
     $empty &&= $self->{'arg_sub_queue'}->empty();
-    ::debug("RecordQueue->empty $empty\n");
+    ::debug("RecordQueue->empty $empty");
     return $empty;
 }
 
@@ -5291,7 +5297,7 @@ sub empty {
     for my $fh (@{$self->{'fhs'}}) {
 	$empty &&= eof($fh);
     }
-    ::debug("MultifileQueue->empty $empty\n");
+    ::debug("MultifileQueue->empty $empty");
     return $empty;
 }
 

testsuite/tests-to-run/parallel-local9.sh

@@ -220,4 +220,4 @@ EOF
 echo '### Test of -j filename with file content changing'; 
   echo 1 >/tmp/jobs_to_run2; 
   (sleep 3; echo 10 >/tmp/jobs_to_run2) &
-  parallel -j /tmp/jobs_to_run2 -v sleep {} ::: 3.3 1.21 1.43 1.54 1.32 1 1 1 1 1 1 1 1 1 1 1
+  parallel -j /tmp/jobs_to_run2 -v sleep {} ::: 3.3 1.5 1.5 1.5 1.5 1 1 1 1 1 1 1 1 1 1 1

testsuite/tests-to-run/parallel-polarhome.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 P="scosysv centos dragonfly netbsd freebsd solaris openbsd debian aix hpux qnx irix suse minix openindiana mandriva unixware miros raspberrypi hurd ultrix ubuntu redhat"
-P="scosysv centos dragonfly netbsd freebsd solaris openbsd debian aix hpux qnx irix suse minix openindiana mandriva unixware miros raspberrypi hurd ultrix ubuntu"
+P="scosysv centos dragonfly netbsd freebsd solaris openbsd debian aix hpux qnx irix suse minix openindiana mandriva unixware raspberrypi hurd ultrix ubuntu"
 #P="scosysv hpux qnx irix openindiana ultrix"
 POLAR=`parallel echo {}.polarhome.com ::: $P`
 

testsuite/wanted-results/parallel-local9

@@ -323,10 +323,10 @@ sleep 1
 sleep 1
 sleep 1
 sleep 1
-sleep 1.21
-sleep 1.32
-sleep 1.43
-sleep 1.54
+sleep 1.5
+sleep 1.5
+sleep 1.5
+sleep 1.5
 sleep 1
 sleep 1
 sleep 1