mirror of
https://git.savannah.gnu.org/git/parallel.git
synced 2024-11-21 21:47:54 +00:00
parallel: Allow for empty loadavg file. This can happen on NFS.
Passes testsuite.
This commit is contained in:
Ole Tange
parent
fc6ad919d3
commit
0b8e949cbb
2
README
2
README
|
|
@ -30,7 +30,7 @@ For security reasons it is recommended you use your package manager to
|
|||
install. But if you cannot do that then you can use this 10 seconds
|
||||
installation.
|
||||
|
||||
The 10 seconds installation will try do to a full installation; if
|
||||
The 10 seconds installation will try to do a full installation; if
|
||||
that fails, a personal installation; if that fails, a minimal
|
||||
installation.
|
||||
|
||||
|
|
|
|||
|
|
@ -220,8 +220,14 @@ New in this release:
|
|||
|
||||
* GNU Parallel was cited in:
|
||||
|
||||
* GNU Parallel was cited in: Parallel Execution of Structural Mechanic Tasks with Use of Small Supercomputers http://www.sciencedirect.com/science/article/pii/S1877705817324815
|
||||
|
||||
https://support.ehelp.edu.au/support/solutions/articles/6000089713-tips-for-running-jobs-on-your-vm#parallel
|
||||
|
||||
https://pzelasko.github.io/posts/speed-up-ml-model-development-with-gnu-parallel
|
||||
|
||||
http://tdhopper.com/blog/2017/Jun/07/parallelizing-a-python-function-for-the-extremely-lazy/
|
||||
|
||||
<<Citation not OK: BAMClipper: removing primers from alignments to minimize false-negative mutations in amplicon next-generation sequencing https://www.nature.com/articles/s41598-017-01703-6>>
|
||||
|
||||
<<Wrong citation https://iris.sissa.it/retrieve/handle/20.500.11767/36149/10823/And%C3%B2_tesi.pdf>>
|
||||
|
|
|
|||
32
src/parallel
32
src/parallel
|
|
@ -5279,6 +5279,7 @@ sub loadavg_too_high {
|
|||
awk '{print $2,$1}' };
|
||||
$dummy="echo S COMMAND;echo R dummy";
|
||||
%ps=(
|
||||
# TODO Find better code for AIX
|
||||
'aix' => "uptime",
|
||||
'cygwin' => $cygwin,
|
||||
'darwin' => $bsd,
|
||||
|
|
@ -5327,19 +5328,24 @@ sub loadavg {
|
|||
local $/; # $/ = undef => slurp whole file
|
||||
my $load_out = <$load_fh>;
|
||||
close $load_fh;
|
||||
# Count lines starting with D,O,R but command does not start with [
|
||||
my $load =()= ($load_out=~/(^\s?[DOR]\S* +(?=[^\[])\S)/gm);
|
||||
if($load > 0) {
|
||||
# load is overestimated by 1
|
||||
$self->{'loadavg'} = $load - 1;
|
||||
::debug("load", "New loadavg: ", $self->{'loadavg'},"\n");
|
||||
} elsif ($load_out=~/average: (\d+.\d+)/) {
|
||||
# AIX does not support instant load average
|
||||
# 04:11AM up 21 days, 12:55, 1 user, load average: 1.85, 1.57, 1.55
|
||||
$self->{'loadavg'} = $1;
|
||||
} else {
|
||||
::die_bug("loadavg_invalid_content: " .
|
||||
$self->{'loadavg_file'} . "\n$load_out");
|
||||
if($load_out =~ /\S/) {
|
||||
# Content can be empty if ~/ is on NFS
|
||||
# due to reading being non-atomic.
|
||||
#
|
||||
# Count lines starting with D,O,R but command does not start with [
|
||||
my $load =()= ($load_out=~/(^\s?[DOR]\S* +(?=[^\[])\S)/gm);
|
||||
if($load > 0) {
|
||||
# load is overestimated by 1
|
||||
$self->{'loadavg'} = $load - 1;
|
||||
::debug("load", "New loadavg: ", $self->{'loadavg'},"\n");
|
||||
} elsif ($load_out=~/average: (\d+.\d+)/) {
|
||||
# AIX does not support instant load average
|
||||
# 04:11AM up 21 days, 12:55, 1 user, load average: 1.85, 1.57, 1.55
|
||||
$self->{'loadavg'} = $1;
|
||||
} else {
|
||||
::die_bug("loadavg_invalid_content: " .
|
||||
$self->{'loadavg_file'} . "\n$load_out");
|
||||
}
|
||||
}
|
||||
$update_loadavg_file = 1;
|
||||
} else {
|
||||
|
|
|
|||
|
|
@ -1933,21 +1933,21 @@ inputsource.
|
|||
|
||||
Like this:
|
||||
|
||||
#!/usr/bin/parallel --shebang -r traceroute
|
||||
#!/usr/bin/parallel --shebang -r wget
|
||||
|
||||
qubes-os.org
|
||||
debian.org
|
||||
freenetproject.org
|
||||
https://ftpmirror.gnu.org/parallel/parallel-20120822.tar.bz2
|
||||
https://ftpmirror.gnu.org/parallel/parallel-20130822.tar.bz2
|
||||
https://ftpmirror.gnu.org/parallel/parallel-20140822.tar.bz2
|
||||
|
||||
B<--shebang> must be set as the first option.
|
||||
|
||||
On FreeBSD B<env> is needed:
|
||||
|
||||
#!/usr/bin/env -S parallel --shebang -r traceroute
|
||||
#!/usr/bin/env -S parallel --shebang -r wget
|
||||
|
||||
qubes-os.org
|
||||
debian.org
|
||||
freenetproject.org
|
||||
https://ftpmirror.gnu.org/parallel/parallel-20120822.tar.bz2
|
||||
https://ftpmirror.gnu.org/parallel/parallel-20130822.tar.bz2
|
||||
https://ftpmirror.gnu.org/parallel/parallel-20140822.tar.bz2
|
||||
|
||||
There are many limitations of shebang (#!) depending on your operating
|
||||
system. See details on http://www.in-ulm.de/~mascheck/various/shebang/
|
||||
|
|
@ -2406,7 +2406,7 @@ care about the output. Compare these:
|
|||
'sleep {};echo -n start{};sleep {};echo {}end'
|
||||
|
||||
It also disables B<--tag>. GNU B<parallel> outputs faster with
|
||||
B<-u>. Compare the speed of these:
|
||||
B<-u>. Compare the speeds of these:
|
||||
|
||||
parallel seq ::: 300000000 >/dev/null
|
||||
parallel -u seq ::: 300000000 >/dev/null
|
||||
|
|
@ -3106,11 +3106,12 @@ printed as soon as possible you can use B<-u>.
|
|||
|
||||
Compare the output of:
|
||||
|
||||
parallel traceroute ::: qubes-os.org debian.org freenetproject.org
|
||||
parallel --line-buffer traceroute ::: \
|
||||
qubes-os.org debian.org freenetproject.org
|
||||
parallel -u traceroute ::: qubes-os.org debian.org freenetproject.org
|
||||
|
||||
parallel wget --limit-rate=100k \
|
||||
https://ftpmirror.gnu.org/parallel/parallel-20{}0822.tar.bz2 ::: {12..16}
|
||||
parallel --line-buffer wget --limit-rate=100k \
|
||||
https://ftpmirror.gnu.org/parallel/parallel-20{}0822.tar.bz2 ::: {12..16}
|
||||
parallel -u wget --limit-rate=100k \
|
||||
https://ftpmirror.gnu.org/parallel/parallel-20{}0822.tar.bz2 ::: {12..16}
|
||||
|
||||
=head1 EXAMPLE: Tag output lines
|
||||
|
||||
|
|
@ -3118,13 +3119,15 @@ GNU B<parallel> groups the output lines, but it can be hard to see
|
|||
where the different jobs begin. B<--tag> prepends the argument to make
|
||||
that more visible:
|
||||
|
||||
parallel --tag traceroute ::: \
|
||||
qubes-os.org debian.org freenetproject.org
|
||||
parallel --tag wget --limit-rate=100k \
|
||||
https://ftpmirror.gnu.org/parallel/parallel-20{}0822.tar.bz2 \
|
||||
::: {12..16}
|
||||
|
||||
B<--tag> works with B<--line-buffer> but not with B<-u>:
|
||||
|
||||
parallel --tag --line-buffer traceroute \
|
||||
::: qubes-os.org debian.org freenetproject.org
|
||||
parallel --tag --line-buffer wget --limit-rate=100k \
|
||||
https://ftpmirror.gnu.org/parallel/parallel-20{}0822.tar.bz2 \
|
||||
::: {12..16}
|
||||
|
||||
Check the uptime of the servers in I<~/.parallel/sshloginfile>:
|
||||
|
||||
|
|
|
|||
|
|
@ -14,7 +14,8 @@ seq 1 3 | parallel -k --interactive "sleep 0.1; echo opt--interactive"
|
|||
_EOF
|
||||
chmod 755 /tmp/parallel-script-for-expect
|
||||
|
||||
expect -b - <<_EOF
|
||||
(
|
||||
expect -b - <<_EOF
|
||||
spawn /tmp/parallel-script-for-expect
|
||||
expect "echo opt-p 1"
|
||||
send "y\n"
|
||||
|
|
@ -32,8 +33,10 @@ expect "opt--interactive 1"
|
|||
expect "echo opt--interactive 3"
|
||||
send "y\n"
|
||||
expect "opt--interactive 3"
|
||||
send "\n"
|
||||
_EOF
|
||||
echo
|
||||
echo
|
||||
) | perl -ne '/\S/ and print'
|
||||
}
|
||||
|
||||
par_k() {
|
||||
|
|
@ -44,16 +47,6 @@ par_k() {
|
|||
echo "echo end") | stdout parallel -k -j0
|
||||
}
|
||||
|
||||
par_sigterm() {
|
||||
echo '### Test SIGTERM'
|
||||
parallel -k -j5 sleep 10';' echo ::: {1..99} >/tmp/parallel$$ 2>&1 &
|
||||
A=$!
|
||||
sleep 19; kill -TERM $A
|
||||
wait
|
||||
sort /tmp/parallel$$
|
||||
rm /tmp/parallel$$
|
||||
}
|
||||
|
||||
par_pipepart_spawn() {
|
||||
echo '### bug #46214: Using --pipepart doesnt spawn multiple jobs in version 20150922'
|
||||
seq 1000000 > /tmp/num1000000;
|
||||
|
|
@ -316,7 +309,7 @@ par_plus_dyn_repl() {
|
|||
|
||||
par_slow_total_jobs() {
|
||||
echo 'bug #51006: Slow total_jobs() eats job'
|
||||
(echo a; sleep 6; echo b; sleep 6; seq 2) |
|
||||
(echo a; sleep 7; echo b; sleep 7; seq 2) |
|
||||
parallel -k echo '{=total_jobs()=}'
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -4,6 +4,16 @@
|
|||
# Each should be taking 30-100s and be possible to run in parallel
|
||||
# I.e.: No race conditions, no logins
|
||||
|
||||
par_sigterm() {
|
||||
echo '### Test SIGTERM'
|
||||
parallel -k -j5 sleep 15';' echo ::: {1..99} >/tmp/parallel$$ 2>&1 &
|
||||
A=$!
|
||||
sleep 29; kill -TERM $A
|
||||
wait
|
||||
sort /tmp/parallel$$
|
||||
rm /tmp/parallel$$
|
||||
}
|
||||
|
||||
par_race_condition1() {
|
||||
echo '### Test race condition on 8 CPU (my laptop)'
|
||||
seq 1 5000000 > /tmp/parallel_race_cond
|
||||
|
|
|
|||
|
|
@ -268,8 +268,7 @@ par_interactive sleep 0.1; echo opt--interactive 1 ?...y
|
|||
par_interactive sleep 0.1; echo opt--interactive 2 ?...n
|
||||
par_interactive sleep 0.1; echo opt--interactive 3 ?...y
|
||||
par_interactive opt--interactive 1
|
||||
par_interactive opt--interactive 3
|
||||
par_interactive
|
||||
par_interactive opt--interactive 3
|
||||
par_k ### Test -k
|
||||
par_k parallel: Warning: Only enough file handles to run 9 jobs in parallel.
|
||||
par_k parallel: Warning: Running 'parallel -j0 -N 9 --pipe parallel -j0' or
|
||||
|
|
@ -914,24 +913,6 @@ par_results_csv 9,:,999.999,999.999,0,6,0,0,"echo 23 12",23,12,"23 12
|
|||
par_results_csv ",
|
||||
par_round_robin_blocks bug #49664: --round-robin does not complete
|
||||
par_round_robin_blocks 8
|
||||
par_sigterm ### Test SIGTERM
|
||||
par_sigterm 1
|
||||
par_sigterm 10
|
||||
par_sigterm 2
|
||||
par_sigterm 3
|
||||
par_sigterm 4
|
||||
par_sigterm 5
|
||||
par_sigterm 6
|
||||
par_sigterm 7
|
||||
par_sigterm 8
|
||||
par_sigterm 9
|
||||
par_sigterm parallel: SIGTERM received. No new jobs will be started.
|
||||
par_sigterm parallel: Waiting for these 5 jobs to finish. Send SIGTERM again to stop now.
|
||||
par_sigterm parallel: sleep 10; echo 10
|
||||
par_sigterm parallel: sleep 10; echo 6
|
||||
par_sigterm parallel: sleep 10; echo 7
|
||||
par_sigterm parallel: sleep 10; echo 8
|
||||
par_sigterm parallel: sleep 10; echo 9
|
||||
par_slow_total_jobs bug #51006: Slow total_jobs() eats job
|
||||
par_slow_total_jobs parallel: Warning: Reading 2 arguments took longer than 10 seconds.
|
||||
par_slow_total_jobs a
|
||||
|
|
|
|||
|
|
@ -20,6 +20,24 @@ par_race_condition1 7
|
|||
par_race_condition1 8
|
||||
par_race_condition1 9
|
||||
par_race_condition1 10
|
||||
par_sigterm ### Test SIGTERM
|
||||
par_sigterm 1
|
||||
par_sigterm 10
|
||||
par_sigterm 2
|
||||
par_sigterm 3
|
||||
par_sigterm 4
|
||||
par_sigterm 5
|
||||
par_sigterm 6
|
||||
par_sigterm 7
|
||||
par_sigterm 8
|
||||
par_sigterm 9
|
||||
par_sigterm parallel: SIGTERM received. No new jobs will be started.
|
||||
par_sigterm parallel: Waiting for these 5 jobs to finish. Send SIGTERM again to stop now.
|
||||
par_sigterm parallel: sleep 15; echo 10
|
||||
par_sigterm parallel: sleep 15; echo 6
|
||||
par_sigterm parallel: sleep 15; echo 7
|
||||
par_sigterm parallel: sleep 15; echo 8
|
||||
par_sigterm parallel: sleep 15; echo 9
|
||||
par_tmp_full ### Test --tmpdir running full. bug #40733 was caused by this
|
||||
par_tmp_full parallel: Error: Output is incomplete. Cannot append to buffer file in /tmp/shm/parallel. Is the disk full?
|
||||
par_tmp_full parallel: Error: Change $TMPDIR with --tmpdir or use --compress.
|
||||
|
|
|
|||
|
|
@ -42,6 +42,7 @@ hpux.polarhome.com Error in tempfile() using /XXXXXXXX.arg: Could not create tem
|
|||
hpux.polarhome.com OK
|
||||
copy_and_test qnx.polarhome.com
|
||||
qnx.polarhome.com ### Run the test on qnx.polarhome.com
|
||||
qnx.polarhome.com parallel: Warning: Cannot figure out number of CPU cores. Using 1.
|
||||
copy_and_test debian-ppc.polarhome.com
|
||||
debian-ppc.polarhome.com ### Run the test on debian-ppc.polarhome.com
|
||||
debian-ppc.polarhome.com Works on debian-ppc.polarhome.com
|
||||
|
|
@ -166,6 +167,7 @@ netbsd.polarhome.com
|
|||
openbsd.polarhome.com
|
||||
openindiana
|
||||
pidora
|
||||
qnx
|
||||
raspbian
|
||||
redhat.polarhome.com
|
||||
scosysv.polarhome.com
|
||||
|
|
@ -195,6 +197,7 @@ netbsd.polarhome.com Missing name for redirect.
|
|||
openbsd.polarhome.com bash only A
|
||||
openindiana.polarhome.com bash only A
|
||||
pidora.polarhome.com bash only A
|
||||
qnx.polarhome.com /bin/sh: syntax error: `(' unexpected
|
||||
raspbian.polarhome.com bash only A
|
||||
redhat.polarhome.com bash only A
|
||||
scosysv.polarhome.com bash only A
|
||||
|
|
|
|||
Loading…
Reference in a new issue