parallel: Allow for empty loadavg file. This can happen on NFS.

Passes testsuite.
This commit is contained in:
Ole Tange 2017-06-18 21:15:36 +02:00
parent fc6ad919d3
commit 0b8e949cbb
9 changed files with 85 additions and 65 deletions

2
README
View file

@ -30,7 +30,7 @@ For security reasons it is recommended you use your package manager to
install. But if you cannot do that then you can use this 10 seconds
installation.
The 10 seconds installation will try do to a full installation; if
The 10 seconds installation will try to do a full installation; if
that fails, a personal installation; if that fails, a minimal
installation.

View file

@ -220,8 +220,14 @@ New in this release:
* GNU Parallel was cited in:
* GNU Parallel was cited in: Parallel Execution of Structural Mechanic Tasks with Use of Small Supercomputers http://www.sciencedirect.com/science/article/pii/S1877705817324815
https://support.ehelp.edu.au/support/solutions/articles/6000089713-tips-for-running-jobs-on-your-vm#parallel
https://pzelasko.github.io/posts/speed-up-ml-model-development-with-gnu-parallel
http://tdhopper.com/blog/2017/Jun/07/parallelizing-a-python-function-for-the-extremely-lazy/
<<Citation not OK: BAMClipper: removing primers from alignments to minimize false-negative mutations in amplicon next-generation sequencing https://www.nature.com/articles/s41598-017-01703-6>>
<<Wrong citation https://iris.sissa.it/retrieve/handle/20.500.11767/36149/10823/And%C3%B2_tesi.pdf>>

View file

@ -5279,6 +5279,7 @@ sub loadavg_too_high {
awk '{print $2,$1}' };
$dummy="echo S COMMAND;echo R dummy";
%ps=(
# TODO Find better code for AIX
'aix' => "uptime",
'cygwin' => $cygwin,
'darwin' => $bsd,
@ -5327,6 +5328,10 @@ sub loadavg {
local $/; # $/ = undef => slurp whole file
my $load_out = <$load_fh>;
close $load_fh;
if($load_out =~ /\S/) {
# Content can be empty if ~/ is on NFS
# due to reading being non-atomic.
#
# Count lines starting with D,O,R but command does not start with [
my $load =()= ($load_out=~/(^\s?[DOR]\S* +(?=[^\[])\S)/gm);
if($load > 0) {
@ -5341,6 +5346,7 @@ sub loadavg {
::die_bug("loadavg_invalid_content: " .
$self->{'loadavg_file'} . "\n$load_out");
}
}
$update_loadavg_file = 1;
} else {
::debug("load", "No loadavg file: ", $self->{'loadavg_file'});

View file

@ -1933,21 +1933,21 @@ inputsource.
Like this:
#!/usr/bin/parallel --shebang -r traceroute
#!/usr/bin/parallel --shebang -r wget
qubes-os.org
debian.org
freenetproject.org
https://ftpmirror.gnu.org/parallel/parallel-20120822.tar.bz2
https://ftpmirror.gnu.org/parallel/parallel-20130822.tar.bz2
https://ftpmirror.gnu.org/parallel/parallel-20140822.tar.bz2
B<--shebang> must be set as the first option.
On FreeBSD B<env> is needed:
#!/usr/bin/env -S parallel --shebang -r traceroute
#!/usr/bin/env -S parallel --shebang -r wget
qubes-os.org
debian.org
freenetproject.org
https://ftpmirror.gnu.org/parallel/parallel-20120822.tar.bz2
https://ftpmirror.gnu.org/parallel/parallel-20130822.tar.bz2
https://ftpmirror.gnu.org/parallel/parallel-20140822.tar.bz2
There are many limitations of shebang (#!) depending on your operating
system. See details on http://www.in-ulm.de/~mascheck/various/shebang/
@ -2406,7 +2406,7 @@ care about the output. Compare these:
'sleep {};echo -n start{};sleep {};echo {}end'
It also disables B<--tag>. GNU B<parallel> outputs faster with
B<-u>. Compare the speed of these:
B<-u>. Compare the speeds of these:
parallel seq ::: 300000000 >/dev/null
parallel -u seq ::: 300000000 >/dev/null
@ -3106,11 +3106,12 @@ printed as soon as possible you can use B<-u>.
Compare the output of:
parallel traceroute ::: qubes-os.org debian.org freenetproject.org
parallel --line-buffer traceroute ::: \
qubes-os.org debian.org freenetproject.org
parallel -u traceroute ::: qubes-os.org debian.org freenetproject.org
parallel wget --limit-rate=100k \
https://ftpmirror.gnu.org/parallel/parallel-20{}0822.tar.bz2 ::: {12..16}
parallel --line-buffer wget --limit-rate=100k \
https://ftpmirror.gnu.org/parallel/parallel-20{}0822.tar.bz2 ::: {12..16}
parallel -u wget --limit-rate=100k \
https://ftpmirror.gnu.org/parallel/parallel-20{}0822.tar.bz2 ::: {12..16}
=head1 EXAMPLE: Tag output lines
@ -3118,13 +3119,15 @@ GNU B<parallel> groups the output lines, but it can be hard to see
where the different jobs begin. B<--tag> prepends the argument to make
that more visible:
parallel --tag traceroute ::: \
qubes-os.org debian.org freenetproject.org
parallel --tag wget --limit-rate=100k \
https://ftpmirror.gnu.org/parallel/parallel-20{}0822.tar.bz2 \
::: {12..16}
B<--tag> works with B<--line-buffer> but not with B<-u>:
parallel --tag --line-buffer traceroute \
::: qubes-os.org debian.org freenetproject.org
parallel --tag --line-buffer wget --limit-rate=100k \
https://ftpmirror.gnu.org/parallel/parallel-20{}0822.tar.bz2 \
::: {12..16}
Check the uptime of the servers in I<~/.parallel/sshloginfile>:

View file

@ -14,6 +14,7 @@ seq 1 3 | parallel -k --interactive "sleep 0.1; echo opt--interactive"
_EOF
chmod 755 /tmp/parallel-script-for-expect
(
expect -b - <<_EOF
spawn /tmp/parallel-script-for-expect
expect "echo opt-p 1"
@ -32,8 +33,10 @@ expect "opt--interactive 1"
expect "echo opt--interactive 3"
send "y\n"
expect "opt--interactive 3"
send "\n"
_EOF
echo
) | perl -ne '/\S/ and print'
}
par_k() {
@ -44,16 +47,6 @@ par_k() {
echo "echo end") | stdout parallel -k -j0
}
par_sigterm() {
echo '### Test SIGTERM'
parallel -k -j5 sleep 10';' echo ::: {1..99} >/tmp/parallel$$ 2>&1 &
A=$!
sleep 19; kill -TERM $A
wait
sort /tmp/parallel$$
rm /tmp/parallel$$
}
par_pipepart_spawn() {
echo '### bug #46214: Using --pipepart doesnt spawn multiple jobs in version 20150922'
seq 1000000 > /tmp/num1000000;
@ -316,7 +309,7 @@ par_plus_dyn_repl() {
par_slow_total_jobs() {
echo 'bug #51006: Slow total_jobs() eats job'
(echo a; sleep 6; echo b; sleep 6; seq 2) |
(echo a; sleep 7; echo b; sleep 7; seq 2) |
parallel -k echo '{=total_jobs()=}'
}

View file

@ -4,6 +4,16 @@
# Each should be taking 30-100s and be possible to run in parallel
# I.e.: No race conditions, no logins
par_sigterm() {
echo '### Test SIGTERM'
parallel -k -j5 sleep 15';' echo ::: {1..99} >/tmp/parallel$$ 2>&1 &
A=$!
sleep 29; kill -TERM $A
wait
sort /tmp/parallel$$
rm /tmp/parallel$$
}
par_race_condition1() {
echo '### Test race condition on 8 CPU (my laptop)'
seq 1 5000000 > /tmp/parallel_race_cond

View file

@ -269,7 +269,6 @@ par_interactive sleep 0.1; echo opt--interactive 2 ?...n
par_interactive sleep 0.1; echo opt--interactive 3 ?...y
par_interactive opt--interactive 1
par_interactive opt--interactive 3
par_interactive
par_k ### Test -k
par_k parallel: Warning: Only enough file handles to run 9 jobs in parallel.
par_k parallel: Warning: Running 'parallel -j0 -N 9 --pipe parallel -j0' or
@ -914,24 +913,6 @@ par_results_csv 9,:,999.999,999.999,0,6,0,0,"echo 23 12",23,12,"23 12
par_results_csv ",
par_round_robin_blocks bug #49664: --round-robin does not complete
par_round_robin_blocks 8
par_sigterm ### Test SIGTERM
par_sigterm 1
par_sigterm 10
par_sigterm 2
par_sigterm 3
par_sigterm 4
par_sigterm 5
par_sigterm 6
par_sigterm 7
par_sigterm 8
par_sigterm 9
par_sigterm parallel: SIGTERM received. No new jobs will be started.
par_sigterm parallel: Waiting for these 5 jobs to finish. Send SIGTERM again to stop now.
par_sigterm parallel: sleep 10; echo 10
par_sigterm parallel: sleep 10; echo 6
par_sigterm parallel: sleep 10; echo 7
par_sigterm parallel: sleep 10; echo 8
par_sigterm parallel: sleep 10; echo 9
par_slow_total_jobs bug #51006: Slow total_jobs() eats job
par_slow_total_jobs parallel: Warning: Reading 2 arguments took longer than 10 seconds.
par_slow_total_jobs a

View file

@ -20,6 +20,24 @@ par_race_condition1 7
par_race_condition1 8
par_race_condition1 9
par_race_condition1 10
par_sigterm ### Test SIGTERM
par_sigterm 1
par_sigterm 10
par_sigterm 2
par_sigterm 3
par_sigterm 4
par_sigterm 5
par_sigterm 6
par_sigterm 7
par_sigterm 8
par_sigterm 9
par_sigterm parallel: SIGTERM received. No new jobs will be started.
par_sigterm parallel: Waiting for these 5 jobs to finish. Send SIGTERM again to stop now.
par_sigterm parallel: sleep 15; echo 10
par_sigterm parallel: sleep 15; echo 6
par_sigterm parallel: sleep 15; echo 7
par_sigterm parallel: sleep 15; echo 8
par_sigterm parallel: sleep 15; echo 9
par_tmp_full ### Test --tmpdir running full. bug #40733 was caused by this
par_tmp_full parallel: Error: Output is incomplete. Cannot append to buffer file in /tmp/shm/parallel. Is the disk full?
par_tmp_full parallel: Error: Change $TMPDIR with --tmpdir or use --compress.

View file

@ -42,6 +42,7 @@ hpux.polarhome.com Error in tempfile() using /XXXXXXXX.arg: Could not create tem
hpux.polarhome.com OK
copy_and_test qnx.polarhome.com
qnx.polarhome.com ### Run the test on qnx.polarhome.com
qnx.polarhome.com parallel: Warning: Cannot figure out number of CPU cores. Using 1.
copy_and_test debian-ppc.polarhome.com
debian-ppc.polarhome.com ### Run the test on debian-ppc.polarhome.com
debian-ppc.polarhome.com Works on debian-ppc.polarhome.com
@ -166,6 +167,7 @@ netbsd.polarhome.com
openbsd.polarhome.com
openindiana
pidora
qnx
raspbian
redhat.polarhome.com
scosysv.polarhome.com
@ -195,6 +197,7 @@ netbsd.polarhome.com Missing name for redirect.
openbsd.polarhome.com bash only A
openindiana.polarhome.com bash only A
pidora.polarhome.com bash only A
qnx.polarhome.com /bin/sh: syntax error: `(' unexpected
raspbian.polarhome.com bash only A
redhat.polarhome.com bash only A
scosysv.polarhome.com bash only A