parallel: --(ssh)delay auto implemented (bug #58911).

src/parallel

@@ -1867,6 +1867,9 @@ sub parse_options(@) {
 	# Set --delay to --sshdelay if not set
 	$opt::delay = $opt::sshdelay;
     }
+    $Global::sshdelayauto = $opt::sshdelay =~ s/auto$//;
+    $opt::sshdelay = multiply_time_units($opt::sshdelay);
+    $Global::delayauto = $opt::delay =~ s/auto$//;
     $opt::delay = multiply_time_units($opt::delay);
     if($opt::compress_program) {
 	$opt::compress = 1;
@@ -4763,6 +4766,17 @@ sub reaper() {
 	$job->set_exitstatus($? >> 8);
 	$job->set_exitsignal($? & 127);
     }
+    if($Global::delayauto or $Global::sshdelayauto) {
+	if($job->exitstatus()) {
+	    # Job failed: Increase delay (if $opt::(ssh)delay set)
+	    $opt::delay &&= $opt::delay * 2;
+	    $opt::sshdelay &&= $opt::sshdelay * 2;
+	} else {
+	    # Job succeeded: Decrease delay (if $opt::(ssh)delay set)
+	    $opt::delay &&= $opt::delay * 0.9;
+	    $opt::sshdelay &&= $opt::sshdelay * 0.9;
+	}
+    }
 
     debug("run", "seq ",$job->seq()," died (", $job->exitstatus(), ")");
     $job->set_endtime(::now());

src/parallel.pod

@@ -711,7 +711,7 @@ Even quoted newlines are parsed correctly:
 When used with B<--pipe> only pass full CSV-records.
 
 
-=item B<--delay> I<mytime>
+=item B<--delay> I<mytime> (alpha testing)
 
 Delay starting next job by I<mytime>. GNU B<parallel> will pause
 I<mytime> after starting each job. I<mytime> is normally in seconds,
@@ -719,6 +719,10 @@ but can be floats postfixed with B<s>, B<m>, B<h>, or B<d> which would
 multiply the float by 1, 60, 3600, or 86400. Thus these are
 equivalent: B<--delay 100000> and B<--delay 1d3.5h16.6m4s>.
 
+If you append 'auto' to I<mytime> (e.g. 13m3sauto) GNU B<parallel> will
+automatically try to find the optimal value: If a job fails, I<mytime>
+is doubled. If a job succeeds, I<mytime> is decreased by 10%.
+
 
 =item B<--delimiter> I<delim>
 
@@ -2416,11 +2420,12 @@ be overridden with B<--ssh>. It can also be set on a per server
 basis (see B<--sshlogin>).
 
 
-=item B<--sshdelay> I<secs>
+=item B<--sshdelay> I<mytime> (alpha testing)
 
-Delay starting next ssh by I<secs> seconds. GNU B<parallel> will pause
+Delay starting next ssh by I<mytime>. GNU B<parallel> will not start
-I<secs> seconds after starting each ssh. I<secs> can be less than 1
+another ssh for the next I<mytime>.
-seconds.
+
+For details on I<mytime> see B<--delay>.
 
 
 =item B<-S> I<[@hostgroups/][ncpus/]sshlogin[,[@hostgroups/][ncpus/]sshlogin[,...]]>

testsuite/Makefile

@@ -7,7 +7,7 @@ testsuite: 3
 	touch ~/.parallel/will-cite
 	make stopvm
 
-### Limited testsets - run once
+### Limited test sets - run once
 # No 100s, 300s, mem, polarhome, tutorial
 1: ../src/parallel tests-to-run/* wanted-results/* prereqlocal startdb prereqremote
 	TRIES=1 time bash Start.sh '' '00s|mem|polarhome|tutorial' || true
@@ -59,7 +59,7 @@ testlocalshort: ../src/parallel tests-to-run/*local* wanted-results/*local* prer
 testdb: ../src/parallel tests-to-run/*sql* wanted-results/*sql* prereqdb
 	time bash Start.sh sql NONE
 
-# Portable tests that runs everywhere
+# Portable tests that run everywhere
 portable:
 	# Cache sudo password
 	sudo echo

testsuite/tests-to-run/parallel-local-race01.sh

@@ -99,6 +99,34 @@ par_bug56403() {
 
 }
 
+par_delay_Xauto() {
+    echo 'TODO: --retries for those that fail and --sshdelay'
+    echo '### bug #58911: --delay Xauto'
+    tmp=$(tempfile)
+    doit() {
+	perl -e '$a=shift;
+	     $m = -M $a < 0.0000001;
+	     `touch $a`;
+	     print "$m\n";
+	     exit $m;' $1;
+    }
+    export -f doit
+    #seq 1000 | ppar --jl - -v --delay 0.1auto -q doit "$tmp"
+    before=`date +%s`
+    out=$(seq 30 | parallel --delay 0.03 -q doit "$tmp")
+    after=`date +%s`
+    normaldiff=$((after-before))
+    echo $normaldiff
+    
+    before=`date +%s`
+    out=$(seq 30 | parallel --delay 0.03auto -q doit "$tmp")
+    after=`date +%s`
+    autodiff=$((after-before))
+    echo $autodiff
+    
+    rm "$tmp"
+}
+
 export -f $(compgen -A function | grep par_)
 compgen -A function | grep par_ | sort |
     #    parallel --joblog /tmp/jl-`basename $0` -j10 --tag -k '{} 2>&1'

testsuite/wanted-results/parallel-local-race01

@@ -5,6 +5,10 @@ par_bug56403	3	job3cjob3d
 par_bug56403	1	job1a
 par_bug56403	1	job1b
 par_bug56403	2	job2cjob2d
+par_delay_Xauto	TODO: --retries for those that fail and --sshdelay
+par_delay_Xauto	### bug #58911: --delay Xauto
+par_delay_Xauto	1
+par_delay_Xauto	25
 par_distribute_input_by_ability	### bug #48290: round-robin does not distribute data based on business
 par_distribute_input_by_ability	### Distribute input to jobs that are ready
 par_distribute_input_by_ability	Job-slot n is 50% slower than n+1, so the order should be 1..7

testsuite/wanted-results/parallel-local19

@@ -1,4 +0,0 @@
-#echo '### Test compress'
-#  seq 5 | parallel -j2 --tag --compress 'seq {} | pv -q -L 10'
-#echo '### Test compress - stderr'
-#  seq 5 | parallel -j2 --tag --compress 'seq {} | pv -q -L 10 >&2' 2>&1 >/dev/null