src/parallel: If --load is a file: Recheck it every time max_loadavg is called.

src/parallel

@@ -1277,7 +1277,9 @@ sub start_more_jobs {
     # Returns:
     #   number of jobs started
     my $jobs_started = 0;
-    if(not $Global::start_no_new_jobs) {
+    if($Global::start_no_new_jobs) {
+	return $jobs_started;
+    }
     if($Global::max_procs_file) {
 	my $mtime = (stat($Global::max_procs_file))[9];
 	if($mtime > $Global::max_procs_file_last_mod) {
@@ -1287,16 +1289,6 @@ sub start_more_jobs {
 	    }
 	}
     }
-        if($Global::max_load_file) {
-            my $mtime = (stat($Global::max_load_file))[9];
-            if($mtime > $Global::max_load_file_last_mod) {
-                $Global::max_load_file_last_mod = $mtime;
-                for my $sshlogin (values %Global::host) {
-                    $sshlogin->set_max_loadavg(undef);
-                }
-            }
-        }
-
     my $sleep = 0.0001; # 0.01 ms - better performance on highend
     for my $sshlogin (values %Global::host) {
 	debug("Running jobs before on ".$sshlogin->string().": ".$sshlogin->jobs_running()."\n");
@@ -1333,7 +1325,6 @@ sub start_more_jobs {
 	debug("Running jobs after on ".$sshlogin->string().": ".$sshlogin->jobs_running()
 	      ." of ".$sshlogin->max_jobs_running() ."\n");
     }
-    }
     return $jobs_started;
 }
 
@@ -2523,6 +2514,16 @@ sub loadavg {
 
 sub max_loadavg {
     my $self = shift;
+    # If --load is a file it might be changed
+    if($Global::max_load_file) {
+	my $mtime = (stat($Global::max_load_file))[9];
+	if($mtime > $Global::max_load_file_last_mod) {
+	    $Global::max_load_file_last_mod = $mtime;
+	    for my $sshlogin (values %Global::host) {
+		$sshlogin->set_max_loadavg(undef);
+	    }
+	}
+    }
     if(not defined $self->{'max_loadavg'}) {
         $self->{'max_loadavg'} =
             $self->compute_max_loadavg($opt::load);
@@ -3766,7 +3767,7 @@ sub sshreturn {
 	my @cmd = ();
 	my $rsync_destdir = ($relpath ? "./" : "/");
 	my $ret_file = $file;
-	my $cd;
+	my $cd = "";
 	if($relpath) {
 	    #   rsync -avR /foo/./bar/baz.c remote:/tmp/
 	    # == (on old systems)

testsuite/tests-to-run/test45.sh

@@ -1,125 +1,40 @@
 #!/bin/bash
 
-rsync -Ha --delete input-files/segfault/ tmp/
-cd tmp
-
 SERVER1=parallel-server3
-SERVER2=parallel-server2
+SERVER2=parallel-server1
 
 # -L1 will join lines ending in ' '
 cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | parallel -j0 -k -L1
-echo '### Test --load locally'; 
+echo "### BUG: The length for -X is not close to max (131072)"; 
-  echo '# This will force the loadavg > 10'; 
+  seq 1 60000 | nice parallel -X echo {.} aa {}{.} {}{}d{} {}dd{}d{.} |head -n 1 |wc
-  while uptime | grep -v age:.[1-9][0-9].[0-9][0-9] >/dev/null ; do (timeout 5 nice burnP6 2>/dev/null &) done; 
+  seq 1 60000 | nice parallel -X echo a{}b{}c |head -n 1 |wc
+  seq 1 60000 | nice parallel -X echo |head -n 1 |wc
+  seq 1 60000 | nice parallel -X echo a{}b{}c {} |head -n 1 |wc
+  seq 1 60000 | nice parallel -X echo {}aa{} |head -n 1 |wc
+  seq 1 60000 | nice parallel -X echo {} aa {} |head -n 1 |wc
+
+echo '### bug #32191: Deep recursion on subroutine main::get_job_with_sshlogin'
+  seq 1 150 | stdout parallel -j9 --retries 2 -S localhost,: "/bin/non-existant 2>/dev/null"
+
+echo '### Test --load locally - should take >10s'
+  echo '# This will run 10 processes in parallel for 10s'; 
+  seq 10 | parallel --nice 19 --timeout 10 -j0 -N0 "gzip < /dev/zero > /dev/null" &
   stdout /usr/bin/time -f %e parallel --load 10 sleep ::: 1 | perl -ne '$_ > 10 and print "OK\n"'
 
-echo '### Test --load remote'; 
+echo '### Test --load remote'
-  ssh parallel@$SERVER2 "while uptime | grep -v age:.[1-9][0-9].[0-9][0-9] >/dev/null ; do (timeout 5 nice burnP6 2>/dev/null &) done"; 
+  ssh parallel@$SERVER2 'seq 10 | parallel --timeout 10 -j0 -N0 "gzip < /dev/zero > /dev/null"' &
   stdout /usr/bin/time -f %e parallel -S parallel@$SERVER2 --load 10 sleep ::: 1 | perl -ne '$_ > 10 and print "OK\n"'
 
-echo '### Test --load read from a file - more than 3s'; 
+echo '### Test --load read from a file - more than 3s'
-  echo '# This will force the loadavg > 10'; 
+  echo '# This will run 10 processes in parallel for 10s'; 
-  while uptime | grep -v age:.[1-9][0-9].[0-9][0-9] >/dev/null ; do (timeout 5 nice burnP6 2>/dev/null &) done; 
+  seq 10 | parallel --nice 19 --timeout 10 -j0 -N0 "gzip < /dev/zero > /dev/null" &
-  ( echo 8 > /tmp/parallel_load_file; sleep 3; echo 100 > /tmp/parallel_load_file ) & 
+  ( echo 8 > /tmp/parallel_load_file; sleep 4; echo 1000 > /tmp/parallel_load_file ) & 
-  stdout /usr/bin/time -f %e parallel --load /tmp/parallel_load_file sleep ::: 1 | perl -ne '$_ > 3 and print "OK\n"'
+  sleep 1;stdout /usr/bin/time -f %e parallel --load /tmp/parallel_load_file sleep ::: 1 | perl -ne '$_ > 3 and print "OK\n"'
 
 echo '### Test --load read from a file - less than 10s'; 
-  echo '# This will force the loadavg > 10'; 
+  echo '# This will run 10 processes in parallel for 10s'; 
-  while uptime | grep -v age:.[1-9][0-9].[0-9][0-9] >/dev/null ; do (timeout 5 nice burnP6 2>/dev/null &) done; 
+  seq 10 | parallel --nice 19 --timeout 10 -j0 -N0 "gzip < /dev/zero > /dev/null" &
-  ( echo 8 > /tmp/parallel_load_file2; sleep 3; echo 100 > /tmp/parallel_load_file2 ) & 
+  ( echo 8 > /tmp/parallel_load_file2; sleep 4; echo 1000 > /tmp/parallel_load_file2 ) & 
   stdout /usr/bin/time -f %e parallel --load /tmp/parallel_load_file2 sleep ::: 1 | perl -ne '$_ < 10 and print "OK\n"'
 
-echo '### Bug in --load'; 
-  parallel -k --load 30 sleep 0.1\;echo ::: 1 2 3
-
-echo '### Test --timeout'; 
-  parallel -j0 -k --timeout 1 echo {}\; sleep {}\; echo {} ::: 1.1 6.6 7.7 8.8
-
-echo '### Test retired'; 
-  stdout parallel -B; 
-  stdout parallel -g; 
-  stdout parallel -H; 
-  stdout parallel -T; 
-  stdout parallel -U; 
-  stdout parallel -W; 
-  stdout parallel -Y;
-
-echo '### Test --joblog followed by --resume --joblog'; 
-  rm -f /tmp/joblog; 
-  timeout -k 1 1 parallel -j2 --joblog /tmp/joblog sleep {} ::: 1.1 2.2 3.3 4.4 2>/dev/null; 
-  parallel -j2 --resume --joblog /tmp/joblog sleep {} ::: 1.1 2.2 3.3 4.4; 
-  cat /tmp/joblog | wc -lw; 
-  rm -f /tmp/joblog;
-
-echo '### Test --resume --joblog followed by --resume --joblog'; 
-  rm -f /tmp/joblog2; 
-  timeout -k 1 1 parallel -j2 --resume --joblog /tmp/joblog2 sleep {} ::: 1.1 2.2 3.3 4.4 2>/dev/null; 
-  parallel -j2 --resume --joblog /tmp/joblog2 sleep {} ::: 1.1 2.2 3.3 4.4; 
-  cat /tmp/joblog2 | wc -lw; 
-  rm -f /tmp/joblog2;
-
-echo '### Test --header'; 
-  printf "a\tb\n1.2\t3/4.5" | parallel --colsep "\t" --header "\n" echo {b} {a} {b.} {b/} {b//} {b/.}; 
-
-echo '### 64-bit wierdness - this did not complete on a 64-bit machine'; 
-  seq 1 2 | parallel -j1 'seq 1 1 | parallel true'
-
-echo "### BUG-fix: bash -c 'parallel -a <(seq 1 3) echo'"; 
-  stdout bash -c 'parallel -k -a <(seq 1 3)  echo'
-
-echo "### BUG: The length for -X is not close to max (131072)"; 
-  seq 1 60000 | parallel -X echo {.} aa {}{.} {}{}d{} {}dd{}d{.} |head -n 1 |wc; 
-  seq 1 60000 | parallel -X echo a{}b{}c |head -n 1 |wc; 
-  seq 1 60000 | parallel -X echo |head -n 1 |wc; 
-  seq 1 60000 | parallel -X echo a{}b{}c {} |head -n 1 |wc; 
-  seq 1 60000 | parallel -X echo {}aa{} |head -n 1 |wc; 
-  seq 1 60000 | parallel -X echo {} aa {} |head -n 1 |wc;
-
-echo "### bug #35268: shell_quote doesn't treats [] brackets correctly"; 
-  touch /tmp/foo1; 
-  stdout parallel echo ::: '/tmp/foo[123]'
-
-echo '### Test make .deb package'; 
-  cd ~/privat/parallel/packager/debian; 
-  stdout make | grep 'To install the GNU Parallel Debian package, run:'
-
-echo '### Test of segfaulting issue'; 
-  echo 'This gave /home/tange/bin/stdout: line 3: 20374 Segmentation fault      "$@" 2>&1'; 
-  echo 'before adding wait() before exit'; 
-  seq 1 300 | stdout parallel ./trysegfault
-
-echo '### Test basic --arg-sep'; 
-  parallel -k echo ::: a b
-
-echo '### Run commands using --arg-sep'; 
-  parallel -kv ::: 'echo a' 'echo b'
-
-echo '### Change --arg-sep'; 
-  parallel --arg-sep ::: -kv ::: 'echo a' 'echo b'; 
-  parallel --arg-sep .--- -kv .--- 'echo a' 'echo b'; 
-  parallel --argsep ::: -kv ::: 'echo a' 'echo b'; 
-  parallel --argsep .--- -kv .--- 'echo a' 'echo b'
-
-echo '### Test stdin goes to first command only'
-echo via cat |parallel --arg-sep .--- -kv .--- 'cat' 'echo b'
-echo via cat |parallel -kv ::: 'cat' 'echo b'
-
-echo '### Bug made 4 5 go before 1 2 3'; 
-  parallel -k ::: "sleep 1; echo 1" "echo 2" "echo 3" "echo 4" "echo 5"
-
-echo '### Bug made 3 go before 1 2'; 
-  parallel -kj 1 ::: "sleep 1; echo 1" "echo 2" "echo 3"
-
-echo '### Bug did not quote'; 
-  echo '>' | parallel -v echo; 
-  parallel -v echo ::: '>'; 
-  (echo '>'; echo  2) | parallel -j1 -vX echo; 
-  parallel -X -j1 echo ::: '>' 2
-
-echo '### Must not quote'; 
-  echo 'echo | wc -l' | parallel -v; 
-  parallel -v ::: 'echo | wc -l'; 
-  echo 'echo a b c | wc -w' | parallel -v; 
-  parallel -kv ::: 'echo a b c | wc -w' 'echo a b | wc -w'
-
 EOF

testsuite/wanted-results/test45

@@ -1,57 +1,3 @@
-### Test --load locally
-# This will force the loadavg > 10
-OK
-### Test --load remote
-OK
-### Test --load read from a file - more than 3s
-# This will force the loadavg > 10
-OK
-### Test --load read from a file - less than 10s
-# This will force the loadavg > 10
-OK
-### Bug in --load
-1
-2
-3
-### Test --timeout
-1.1
-1.1
-6.6
-7.7
-8.8
-### Test retired
-parallel: Error: -g has been retired. Use --group.
-parallel: Error: -B has been retired. Use --bf.
-parallel: Error: -T has been retired. Use --tty.
-parallel: Error: -U has been retired. Use --er.
-parallel: Error: -W has been retired. Use --wd.
-parallel: Error: -Y has been retired. Use --shebang.
-parallel: Error: -H has been retired. Use --halt.
-parallel: Error: -g has been retired. Use --group.
-parallel: Error: -B has been retired. Use --bf.
-parallel: Error: -T has been retired. Use --tty.
-parallel: Error: -U has been retired. Use --er.
-parallel: Error: -W has been retired. Use --wd.
-parallel: Error: -Y has been retired. Use --shebang.
-parallel: Error: -H has been retired. Use --halt.
-parallel: Error: -g has been retired. Use --group.
-parallel: Error: -B has been retired. Use --bf.
-parallel: Error: -T has been retired. Use --tty.
-parallel: Error: -U has been retired. Use --er.
-parallel: Error: -W has been retired. Use --wd.
-parallel: Error: -Y has been retired. Use --shebang.
-parallel: Error: -H has been retired. Use --halt.
-### Test --joblog followed by --resume --joblog
-      5      49
-### Test --resume --joblog followed by --resume --joblog
-      5      49
-### Test --header
-3/4.5 1.2 3/4 4.5 3 4
-### 64-bit wierdness - this did not complete on a 64-bit machine
-### BUG-fix: bash -c 'parallel -a <(seq 1 3) echo'
-1
-2
-3
 ### BUG: The length for -X is not close to max (131072)
       1   12821  131060
       1   10948  131060
@@ -59,73 +5,15 @@ parallel: Error: -H has been retired. Use --halt.
       1   15810  131064
       1   11790  131058
       1   25545  131055
-### bug #35268: shell_quote doesn't treats [] brackets correctly
+### bug #32191: Deep recursion on subroutine main::get_job_with_sshlogin
-/tmp/foo[123]
+### Test --load locally - should take >10s
-### Test make .deb package
+# This will run 10 processes in parallel for 10s
-To install the GNU Parallel Debian package, run:
+OK
-### Test of segfaulting issue
+### Test --load remote
-This gave /home/tange/bin/stdout: line 3: 20374 Segmentation fault      "$@" 2>&1
+OK
-before adding wait() before exit
+### Test --load read from a file - more than 3s
-### Test basic --arg-sep
+# This will run 10 processes in parallel for 10s
-a
+OK
-b
+### Test --load read from a file - less than 10s
-### Run commands using --arg-sep
+# This will run 10 processes in parallel for 10s
-echo a
+OK
-a
-echo b
-b
-### Change --arg-sep
-echo a
-a
-echo b
-b
-echo a
-a
-echo b
-b
-echo a
-a
-echo b
-b
-echo a
-a
-echo b
-b
-### Test stdin goes to first command only
-cat
-via cat
-echo b
-b
-cat
-via cat
-echo b
-b
-### Bug made 4 5 go before 1 2 3
-1
-2
-3
-4
-5
-### Bug made 3 go before 1 2
-1
-2
-3
-### Bug did not quote
-echo \>
->
-echo \>
->
-echo \> 2
-> 2
-> 2
-### Must not quote
-echo | wc -l
-1
-echo | wc -l
-1
-echo a b c | wc -w
-3
-echo a b c | wc -w
-3
-echo a b | wc -w
-2