parallel: Allow for empty loadavg file. This can happen on NFS.

Passes testsuite.

README

@@ -30,7 +30,7 @@ For security reasons it is recommended you use your package manager to
 install. But if you cannot do that then you can use this 10 seconds
 installation.
 
-The 10 seconds installation will try do to a full installation; if
+The 10 seconds installation will try to do a full installation; if
 that fails, a personal installation; if that fails, a minimal
 installation.
 

doc/release_new_version

@@ -220,8 +220,14 @@ New in this release:
 
 * GNU Parallel was cited in:
 
+* GNU Parallel was cited in: Parallel Execution of Structural Mechanic Tasks with Use of Small Supercomputers http://www.sciencedirect.com/science/article/pii/S1877705817324815
+
 https://support.ehelp.edu.au/support/solutions/articles/6000089713-tips-for-running-jobs-on-your-vm#parallel
 
+https://pzelasko.github.io/posts/speed-up-ml-model-development-with-gnu-parallel
+
+http://tdhopper.com/blog/2017/Jun/07/parallelizing-a-python-function-for-the-extremely-lazy/
+
 <<Citation not OK: BAMClipper: removing primers from alignments to minimize false-negative mutations in amplicon next-generation sequencing https://www.nature.com/articles/s41598-017-01703-6>>
 
 <<Wrong citation https://iris.sissa.it/retrieve/handle/20.500.11767/36149/10823/And%C3%B2_tesi.pdf>>

src/parallel

@@ -5279,6 +5279,7 @@ sub loadavg_too_high {
                     awk '{print $2,$1}' };
 		$dummy="echo S COMMAND;echo R dummy";
 		%ps=(
+		    # TODO Find better code for AIX
 		    'aix' => "uptime",
 		    'cygwin' => $cygwin,
 		    'darwin' => $bsd,
@@ -5327,19 +5328,24 @@ sub loadavg {
 	local $/; # $/ = undef => slurp whole file
         my $load_out = <$load_fh>;
         close $load_fh;
-	# Count lines starting with D,O,R but command does not start with [
+	if($load_out =~ /\S/) {
-	my $load =()= ($load_out=~/(^\s?[DOR]\S* +(?=[^\[])\S)/gm);
+	    # Content can be empty if ~/ is on NFS
-        if($load > 0) {
+	    # due to reading being non-atomic.
-	    # load is overestimated by 1
+	    #
-            $self->{'loadavg'} = $load - 1;
+	    # Count lines starting with D,O,R but command does not start with [
-            ::debug("load", "New loadavg: ", $self->{'loadavg'},"\n");
+	    my $load =()= ($load_out=~/(^\s?[DOR]\S* +(?=[^\[])\S)/gm);
-        } elsif ($load_out=~/average: (\d+.\d+)/) {
+	    if($load > 0) {
-	    # AIX does not support instant load average
+		# load is overestimated by 1
-	    # 04:11AM   up 21 days,  12:55,  1 user,  load average: 1.85, 1.57, 1.55
+		$self->{'loadavg'} = $load - 1;
-	    $self->{'loadavg'} = $1;
+		::debug("load", "New loadavg: ", $self->{'loadavg'},"\n");
-	} else {
+	    } elsif ($load_out=~/average: (\d+.\d+)/) {
-	    ::die_bug("loadavg_invalid_content: " .
+		# AIX does not support instant load average
-		      $self->{'loadavg_file'} . "\n$load_out");
+		# 04:11AM   up 21 days,  12:55,  1 user,  load average: 1.85, 1.57, 1.55
+		$self->{'loadavg'} = $1;
+	    } else {
+		::die_bug("loadavg_invalid_content: " .
+			  $self->{'loadavg_file'} . "\n$load_out");
+	    }
 	}
 	$update_loadavg_file = 1;
     } else {

src/parallel.pod

@@ -1933,21 +1933,21 @@ inputsource.
 
 Like this:
 
-  #!/usr/bin/parallel --shebang -r traceroute
+  #!/usr/bin/parallel --shebang -r wget
 
-  qubes-os.org
+  https://ftpmirror.gnu.org/parallel/parallel-20120822.tar.bz2
-  debian.org
+  https://ftpmirror.gnu.org/parallel/parallel-20130822.tar.bz2
-  freenetproject.org
+  https://ftpmirror.gnu.org/parallel/parallel-20140822.tar.bz2
 
 B<--shebang> must be set as the first option.
 
 On FreeBSD B<env> is needed:
 
-  #!/usr/bin/env -S parallel --shebang -r traceroute
+  #!/usr/bin/env -S parallel --shebang -r wget
 
-  qubes-os.org
+  https://ftpmirror.gnu.org/parallel/parallel-20120822.tar.bz2
-  debian.org
+  https://ftpmirror.gnu.org/parallel/parallel-20130822.tar.bz2
-  freenetproject.org
+  https://ftpmirror.gnu.org/parallel/parallel-20140822.tar.bz2
 
 There are many limitations of shebang (#!) depending on your operating
 system. See details on http://www.in-ulm.de/~mascheck/various/shebang/
@@ -2406,7 +2406,7 @@ care about the output. Compare these:
     'sleep {};echo -n start{};sleep {};echo {}end'
 
 It also disables B<--tag>. GNU B<parallel> outputs faster with
-B<-u>. Compare the speed of these:
+B<-u>. Compare the speeds of these:
 
   parallel seq ::: 300000000 >/dev/null
   parallel -u seq ::: 300000000 >/dev/null
@@ -3106,11 +3106,12 @@ printed as soon as possible you can use B<-u>.
 
 Compare the output of:
 
-  parallel traceroute ::: qubes-os.org debian.org freenetproject.org
+  parallel wget --limit-rate=100k \
-  parallel --line-buffer traceroute ::: \
+    https://ftpmirror.gnu.org/parallel/parallel-20{}0822.tar.bz2 ::: {12..16}
-    qubes-os.org debian.org freenetproject.org
+  parallel --line-buffer wget --limit-rate=100k \
-  parallel -u traceroute ::: qubes-os.org debian.org freenetproject.org
+    https://ftpmirror.gnu.org/parallel/parallel-20{}0822.tar.bz2 ::: {12..16}
-
+  parallel -u wget --limit-rate=100k \
+    https://ftpmirror.gnu.org/parallel/parallel-20{}0822.tar.bz2 ::: {12..16}
 
 =head1 EXAMPLE: Tag output lines
 
@@ -3118,13 +3119,15 @@ GNU B<parallel> groups the output lines, but it can be hard to see
 where the different jobs begin. B<--tag> prepends the argument to make
 that more visible:
 
-  parallel --tag traceroute ::: \
+  parallel --tag wget --limit-rate=100k \
-    qubes-os.org debian.org freenetproject.org
+    https://ftpmirror.gnu.org/parallel/parallel-20{}0822.tar.bz2 \
+    ::: {12..16}
 
 B<--tag> works with B<--line-buffer> but not with B<-u>:
 
-  parallel --tag --line-buffer traceroute \
+  parallel --tag --line-buffer wget --limit-rate=100k \
-    ::: qubes-os.org debian.org freenetproject.org
+    https://ftpmirror.gnu.org/parallel/parallel-20{}0822.tar.bz2 \
+    ::: {12..16}
 
 Check the uptime of the servers in I<~/.parallel/sshloginfile>:
 

testsuite/tests-to-run/parallel-local-10s.sh

@@ -14,7 +14,8 @@ seq 1 3 | parallel -k --interactive "sleep 0.1; echo opt--interactive"
 _EOF
     chmod 755 /tmp/parallel-script-for-expect
 
-    expect -b - <<_EOF
+    (
+	expect -b - <<_EOF
 spawn /tmp/parallel-script-for-expect
 expect "echo opt-p 1"
 send "y\n"
@@ -32,8 +33,10 @@ expect "opt--interactive 1"
 expect "echo opt--interactive 3"
 send "y\n"
 expect "opt--interactive 3"
+send "\n"
 _EOF
-    echo
+	echo
+    ) | perl -ne '/\S/ and print'
 }
 
 par_k() {
@@ -44,16 +47,6 @@ par_k() {
 	echo "echo end") | stdout parallel -k -j0
 }
 
-par_sigterm() {
-    echo '### Test SIGTERM'
-    parallel -k -j5 sleep 10';' echo ::: {1..99} >/tmp/parallel$$ 2>&1 &
-    A=$!
-    sleep 19; kill -TERM $A
-    wait
-    sort /tmp/parallel$$
-    rm /tmp/parallel$$
-}
-
 par_pipepart_spawn() {
     echo '### bug #46214: Using --pipepart doesnt spawn multiple jobs in version 20150922'
     seq 1000000 > /tmp/num1000000;
@@ -316,7 +309,7 @@ par_plus_dyn_repl() {
 
 par_slow_total_jobs() {
     echo 'bug #51006: Slow total_jobs() eats job'
-    (echo a; sleep 6; echo b; sleep 6; seq 2) |
+    (echo a; sleep 7; echo b; sleep 7; seq 2) |
 	parallel -k echo '{=total_jobs()=}'
 }
 

testsuite/tests-to-run/parallel-local-30s.sh

@@ -4,6 +4,16 @@
 # Each should be taking 30-100s and be possible to run in parallel
 # I.e.: No race conditions, no logins
 
+par_sigterm() {
+    echo '### Test SIGTERM'
+    parallel -k -j5 sleep 15';' echo ::: {1..99} >/tmp/parallel$$ 2>&1 &
+    A=$!
+    sleep 29; kill -TERM $A
+    wait
+    sort /tmp/parallel$$
+    rm /tmp/parallel$$
+}
+
 par_race_condition1() {
     echo '### Test race condition on 8 CPU (my laptop)'
     seq 1 5000000 > /tmp/parallel_race_cond

testsuite/wanted-results/parallel-local-10s

@@ -268,8 +268,7 @@ par_interactive	sleep 0.1; echo opt--interactive 1 ?...y
 par_interactive	sleep 0.1; echo opt--interactive 2 ?...n
 par_interactive	sleep 0.1; echo opt--interactive 3 ?...y
 par_interactive	opt--interactive 1
-par_interactive	opt--interactive 3
+par_interactive	opt--interactive 3
-par_interactive	
 par_k	### Test -k
 par_k	parallel: Warning: Only enough file handles to run 9 jobs in parallel.
 par_k	parallel: Warning: Running 'parallel -j0 -N 9 --pipe parallel -j0' or
@@ -914,24 +913,6 @@ par_results_csv	    	9,:,999.999,999.999,0,6,0,0,"echo 23 12",23,12,"23 12
 par_results_csv	    	",
 par_round_robin_blocks	bug #49664: --round-robin does not complete
 par_round_robin_blocks	8
-par_sigterm	### Test SIGTERM
-par_sigterm	1
-par_sigterm	10
-par_sigterm	2
-par_sigterm	3
-par_sigterm	4
-par_sigterm	5
-par_sigterm	6
-par_sigterm	7
-par_sigterm	8
-par_sigterm	9
-par_sigterm	parallel: SIGTERM received. No new jobs will be started.
-par_sigterm	parallel: Waiting for these 5 jobs to finish. Send SIGTERM again to stop now.
-par_sigterm	parallel: sleep 10; echo 10
-par_sigterm	parallel: sleep 10; echo 6
-par_sigterm	parallel: sleep 10; echo 7
-par_sigterm	parallel: sleep 10; echo 8
-par_sigterm	parallel: sleep 10; echo 9
 par_slow_total_jobs	bug #51006: Slow total_jobs() eats job
 par_slow_total_jobs	parallel: Warning: Reading 2 arguments took longer than 10 seconds.
 par_slow_total_jobs	a

testsuite/wanted-results/parallel-local-30s

@@ -20,6 +20,24 @@ par_race_condition1	7
 par_race_condition1	8
 par_race_condition1	9
 par_race_condition1	10
+par_sigterm	### Test SIGTERM
+par_sigterm	1
+par_sigterm	10
+par_sigterm	2
+par_sigterm	3
+par_sigterm	4
+par_sigterm	5
+par_sigterm	6
+par_sigterm	7
+par_sigterm	8
+par_sigterm	9
+par_sigterm	parallel: SIGTERM received. No new jobs will be started.
+par_sigterm	parallel: Waiting for these 5 jobs to finish. Send SIGTERM again to stop now.
+par_sigterm	parallel: sleep 15; echo 10
+par_sigterm	parallel: sleep 15; echo 6
+par_sigterm	parallel: sleep 15; echo 7
+par_sigterm	parallel: sleep 15; echo 8
+par_sigterm	parallel: sleep 15; echo 9
 par_tmp_full	### Test --tmpdir running full. bug #40733 was caused by this
 par_tmp_full	parallel: Error: Output is incomplete. Cannot append to buffer file in /tmp/shm/parallel. Is the disk full?
 par_tmp_full	parallel: Error: Change $TMPDIR with --tmpdir or use --compress.

testsuite/wanted-results/parallel-polarhome

@@ -42,6 +42,7 @@ hpux.polarhome.com	Error in tempfile() using /XXXXXXXX.arg: Could not create tem
 hpux.polarhome.com	OK
 copy_and_test qnx.polarhome.com
 qnx.polarhome.com	### Run the test on qnx.polarhome.com
+qnx.polarhome.com	parallel: Warning: Cannot figure out number of CPU cores. Using 1.
 copy_and_test debian-ppc.polarhome.com
 debian-ppc.polarhome.com	### Run the test on debian-ppc.polarhome.com
 debian-ppc.polarhome.com	Works on debian-ppc.polarhome.com
@@ -166,6 +167,7 @@ netbsd.polarhome.com
 openbsd.polarhome.com
 openindiana
 pidora
+qnx
 raspbian
 redhat.polarhome.com
 scosysv.polarhome.com
@@ -195,6 +197,7 @@ netbsd.polarhome.com	Missing name for redirect.
 openbsd.polarhome.com	bash only A
 openindiana.polarhome.com	bash only A
 pidora.polarhome.com	bash only A
+qnx.polarhome.com	/bin/sh: syntax error: `(' unexpected
 raspbian.polarhome.com	bash only A
 redhat.polarhome.com	bash only A
 scosysv.polarhome.com	bash only A