From 0b8e949cbb3ccaac3c83b75c0e12087a665d7910 Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Sun, 18 Jun 2017 21:15:36 +0200 Subject: [PATCH] parallel: Allow for empty loadavg file. This can happen on NFS. Passes testsuite. --- README | 2 +- doc/release_new_version | 6 +++ src/parallel | 32 +++++++++------- src/parallel.pod | 39 +++++++++++--------- testsuite/tests-to-run/parallel-local-10s.sh | 19 +++------- testsuite/tests-to-run/parallel-local-30s.sh | 10 +++++ testsuite/wanted-results/parallel-local-10s | 21 +---------- testsuite/wanted-results/parallel-local-30s | 18 +++++++++ testsuite/wanted-results/parallel-polarhome | 3 ++ 9 files changed, 85 insertions(+), 65 deletions(-) diff --git a/README b/README index 1dd9b512..2e1bd1a6 100644 --- a/README +++ b/README @@ -30,7 +30,7 @@ For security reasons it is recommended you use your package manager to install. But if you cannot do that then you can use this 10 seconds installation. -The 10 seconds installation will try do to a full installation; if +The 10 seconds installation will try to do a full installation; if that fails, a personal installation; if that fails, a minimal installation. diff --git a/doc/release_new_version b/doc/release_new_version index 81e81467..52e67c89 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -220,8 +220,14 @@ New in this release: * GNU Parallel was cited in: +* GNU Parallel was cited in: Parallel Execution of Structural Mechanic Tasks with Use of Small Supercomputers http://www.sciencedirect.com/science/article/pii/S1877705817324815 + https://support.ehelp.edu.au/support/solutions/articles/6000089713-tips-for-running-jobs-on-your-vm#parallel +https://pzelasko.github.io/posts/speed-up-ml-model-development-with-gnu-parallel + +http://tdhopper.com/blog/2017/Jun/07/parallelizing-a-python-function-for-the-extremely-lazy/ + <> <> diff --git a/src/parallel b/src/parallel index f1d3ff76..1b856777 100755 --- a/src/parallel +++ b/src/parallel @@ -5279,6 +5279,7 @@ sub loadavg_too_high { awk '{print $2,$1}' }; $dummy="echo S COMMAND;echo R dummy"; %ps=( + # TODO Find better code for AIX 'aix' => "uptime", 'cygwin' => $cygwin, 'darwin' => $bsd, @@ -5327,19 +5328,24 @@ sub loadavg { local $/; # $/ = undef => slurp whole file my $load_out = <$load_fh>; close $load_fh; - # Count lines starting with D,O,R but command does not start with [ - my $load =()= ($load_out=~/(^\s?[DOR]\S* +(?=[^\[])\S)/gm); - if($load > 0) { - # load is overestimated by 1 - $self->{'loadavg'} = $load - 1; - ::debug("load", "New loadavg: ", $self->{'loadavg'},"\n"); - } elsif ($load_out=~/average: (\d+.\d+)/) { - # AIX does not support instant load average - # 04:11AM up 21 days, 12:55, 1 user, load average: 1.85, 1.57, 1.55 - $self->{'loadavg'} = $1; - } else { - ::die_bug("loadavg_invalid_content: " . - $self->{'loadavg_file'} . "\n$load_out"); + if($load_out =~ /\S/) { + # Content can be empty if ~/ is on NFS + # due to reading being non-atomic. + # + # Count lines starting with D,O,R but command does not start with [ + my $load =()= ($load_out=~/(^\s?[DOR]\S* +(?=[^\[])\S)/gm); + if($load > 0) { + # load is overestimated by 1 + $self->{'loadavg'} = $load - 1; + ::debug("load", "New loadavg: ", $self->{'loadavg'},"\n"); + } elsif ($load_out=~/average: (\d+.\d+)/) { + # AIX does not support instant load average + # 04:11AM up 21 days, 12:55, 1 user, load average: 1.85, 1.57, 1.55 + $self->{'loadavg'} = $1; + } else { + ::die_bug("loadavg_invalid_content: " . + $self->{'loadavg_file'} . "\n$load_out"); + } } $update_loadavg_file = 1; } else { diff --git a/src/parallel.pod b/src/parallel.pod index 8ce48b8c..bdb10a86 100644 --- a/src/parallel.pod +++ b/src/parallel.pod @@ -1933,21 +1933,21 @@ inputsource. Like this: - #!/usr/bin/parallel --shebang -r traceroute + #!/usr/bin/parallel --shebang -r wget - qubes-os.org - debian.org - freenetproject.org + https://ftpmirror.gnu.org/parallel/parallel-20120822.tar.bz2 + https://ftpmirror.gnu.org/parallel/parallel-20130822.tar.bz2 + https://ftpmirror.gnu.org/parallel/parallel-20140822.tar.bz2 B<--shebang> must be set as the first option. On FreeBSD B is needed: - #!/usr/bin/env -S parallel --shebang -r traceroute + #!/usr/bin/env -S parallel --shebang -r wget - qubes-os.org - debian.org - freenetproject.org + https://ftpmirror.gnu.org/parallel/parallel-20120822.tar.bz2 + https://ftpmirror.gnu.org/parallel/parallel-20130822.tar.bz2 + https://ftpmirror.gnu.org/parallel/parallel-20140822.tar.bz2 There are many limitations of shebang (#!) depending on your operating system. See details on http://www.in-ulm.de/~mascheck/various/shebang/ @@ -2406,7 +2406,7 @@ care about the output. Compare these: 'sleep {};echo -n start{};sleep {};echo {}end' It also disables B<--tag>. GNU B outputs faster with -B<-u>. Compare the speed of these: +B<-u>. Compare the speeds of these: parallel seq ::: 300000000 >/dev/null parallel -u seq ::: 300000000 >/dev/null @@ -3106,11 +3106,12 @@ printed as soon as possible you can use B<-u>. Compare the output of: - parallel traceroute ::: qubes-os.org debian.org freenetproject.org - parallel --line-buffer traceroute ::: \ - qubes-os.org debian.org freenetproject.org - parallel -u traceroute ::: qubes-os.org debian.org freenetproject.org - + parallel wget --limit-rate=100k \ + https://ftpmirror.gnu.org/parallel/parallel-20{}0822.tar.bz2 ::: {12..16} + parallel --line-buffer wget --limit-rate=100k \ + https://ftpmirror.gnu.org/parallel/parallel-20{}0822.tar.bz2 ::: {12..16} + parallel -u wget --limit-rate=100k \ + https://ftpmirror.gnu.org/parallel/parallel-20{}0822.tar.bz2 ::: {12..16} =head1 EXAMPLE: Tag output lines @@ -3118,13 +3119,15 @@ GNU B groups the output lines, but it can be hard to see where the different jobs begin. B<--tag> prepends the argument to make that more visible: - parallel --tag traceroute ::: \ - qubes-os.org debian.org freenetproject.org + parallel --tag wget --limit-rate=100k \ + https://ftpmirror.gnu.org/parallel/parallel-20{}0822.tar.bz2 \ + ::: {12..16} B<--tag> works with B<--line-buffer> but not with B<-u>: - parallel --tag --line-buffer traceroute \ - ::: qubes-os.org debian.org freenetproject.org + parallel --tag --line-buffer wget --limit-rate=100k \ + https://ftpmirror.gnu.org/parallel/parallel-20{}0822.tar.bz2 \ + ::: {12..16} Check the uptime of the servers in I<~/.parallel/sshloginfile>: diff --git a/testsuite/tests-to-run/parallel-local-10s.sh b/testsuite/tests-to-run/parallel-local-10s.sh index ee9dce26..e2ed3f52 100644 --- a/testsuite/tests-to-run/parallel-local-10s.sh +++ b/testsuite/tests-to-run/parallel-local-10s.sh @@ -14,7 +14,8 @@ seq 1 3 | parallel -k --interactive "sleep 0.1; echo opt--interactive" _EOF chmod 755 /tmp/parallel-script-for-expect - expect -b - <<_EOF + ( + expect -b - <<_EOF spawn /tmp/parallel-script-for-expect expect "echo opt-p 1" send "y\n" @@ -32,8 +33,10 @@ expect "opt--interactive 1" expect "echo opt--interactive 3" send "y\n" expect "opt--interactive 3" +send "\n" _EOF - echo + echo + ) | perl -ne '/\S/ and print' } par_k() { @@ -44,16 +47,6 @@ par_k() { echo "echo end") | stdout parallel -k -j0 } -par_sigterm() { - echo '### Test SIGTERM' - parallel -k -j5 sleep 10';' echo ::: {1..99} >/tmp/parallel$$ 2>&1 & - A=$! - sleep 19; kill -TERM $A - wait - sort /tmp/parallel$$ - rm /tmp/parallel$$ -} - par_pipepart_spawn() { echo '### bug #46214: Using --pipepart doesnt spawn multiple jobs in version 20150922' seq 1000000 > /tmp/num1000000; @@ -316,7 +309,7 @@ par_plus_dyn_repl() { par_slow_total_jobs() { echo 'bug #51006: Slow total_jobs() eats job' - (echo a; sleep 6; echo b; sleep 6; seq 2) | + (echo a; sleep 7; echo b; sleep 7; seq 2) | parallel -k echo '{=total_jobs()=}' } diff --git a/testsuite/tests-to-run/parallel-local-30s.sh b/testsuite/tests-to-run/parallel-local-30s.sh index 40b5084e..4be4fcbf 100755 --- a/testsuite/tests-to-run/parallel-local-30s.sh +++ b/testsuite/tests-to-run/parallel-local-30s.sh @@ -4,6 +4,16 @@ # Each should be taking 30-100s and be possible to run in parallel # I.e.: No race conditions, no logins +par_sigterm() { + echo '### Test SIGTERM' + parallel -k -j5 sleep 15';' echo ::: {1..99} >/tmp/parallel$$ 2>&1 & + A=$! + sleep 29; kill -TERM $A + wait + sort /tmp/parallel$$ + rm /tmp/parallel$$ +} + par_race_condition1() { echo '### Test race condition on 8 CPU (my laptop)' seq 1 5000000 > /tmp/parallel_race_cond diff --git a/testsuite/wanted-results/parallel-local-10s b/testsuite/wanted-results/parallel-local-10s index 4e535322..61992268 100644 --- a/testsuite/wanted-results/parallel-local-10s +++ b/testsuite/wanted-results/parallel-local-10s @@ -268,8 +268,7 @@ par_interactive sleep 0.1; echo opt--interactive 1 ?...y par_interactive sleep 0.1; echo opt--interactive 2 ?...n par_interactive sleep 0.1; echo opt--interactive 3 ?...y par_interactive opt--interactive 1 -par_interactive opt--interactive 3 -par_interactive +par_interactive opt--interactive 3 par_k ### Test -k par_k parallel: Warning: Only enough file handles to run 9 jobs in parallel. par_k parallel: Warning: Running 'parallel -j0 -N 9 --pipe parallel -j0' or @@ -914,24 +913,6 @@ par_results_csv 9,:,999.999,999.999,0,6,0,0,"echo 23 12",23,12,"23 12 par_results_csv ", par_round_robin_blocks bug #49664: --round-robin does not complete par_round_robin_blocks 8 -par_sigterm ### Test SIGTERM -par_sigterm 1 -par_sigterm 10 -par_sigterm 2 -par_sigterm 3 -par_sigterm 4 -par_sigterm 5 -par_sigterm 6 -par_sigterm 7 -par_sigterm 8 -par_sigterm 9 -par_sigterm parallel: SIGTERM received. No new jobs will be started. -par_sigterm parallel: Waiting for these 5 jobs to finish. Send SIGTERM again to stop now. -par_sigterm parallel: sleep 10; echo 10 -par_sigterm parallel: sleep 10; echo 6 -par_sigterm parallel: sleep 10; echo 7 -par_sigterm parallel: sleep 10; echo 8 -par_sigterm parallel: sleep 10; echo 9 par_slow_total_jobs bug #51006: Slow total_jobs() eats job par_slow_total_jobs parallel: Warning: Reading 2 arguments took longer than 10 seconds. par_slow_total_jobs a diff --git a/testsuite/wanted-results/parallel-local-30s b/testsuite/wanted-results/parallel-local-30s index 906d2e49..20db9332 100644 --- a/testsuite/wanted-results/parallel-local-30s +++ b/testsuite/wanted-results/parallel-local-30s @@ -20,6 +20,24 @@ par_race_condition1 7 par_race_condition1 8 par_race_condition1 9 par_race_condition1 10 +par_sigterm ### Test SIGTERM +par_sigterm 1 +par_sigterm 10 +par_sigterm 2 +par_sigterm 3 +par_sigterm 4 +par_sigterm 5 +par_sigterm 6 +par_sigterm 7 +par_sigterm 8 +par_sigterm 9 +par_sigterm parallel: SIGTERM received. No new jobs will be started. +par_sigterm parallel: Waiting for these 5 jobs to finish. Send SIGTERM again to stop now. +par_sigterm parallel: sleep 15; echo 10 +par_sigterm parallel: sleep 15; echo 6 +par_sigterm parallel: sleep 15; echo 7 +par_sigterm parallel: sleep 15; echo 8 +par_sigterm parallel: sleep 15; echo 9 par_tmp_full ### Test --tmpdir running full. bug #40733 was caused by this par_tmp_full parallel: Error: Output is incomplete. Cannot append to buffer file in /tmp/shm/parallel. Is the disk full? par_tmp_full parallel: Error: Change $TMPDIR with --tmpdir or use --compress. diff --git a/testsuite/wanted-results/parallel-polarhome b/testsuite/wanted-results/parallel-polarhome index aa144a6c..49d9056b 100644 --- a/testsuite/wanted-results/parallel-polarhome +++ b/testsuite/wanted-results/parallel-polarhome @@ -42,6 +42,7 @@ hpux.polarhome.com Error in tempfile() using /XXXXXXXX.arg: Could not create tem hpux.polarhome.com OK copy_and_test qnx.polarhome.com qnx.polarhome.com ### Run the test on qnx.polarhome.com +qnx.polarhome.com parallel: Warning: Cannot figure out number of CPU cores. Using 1. copy_and_test debian-ppc.polarhome.com debian-ppc.polarhome.com ### Run the test on debian-ppc.polarhome.com debian-ppc.polarhome.com Works on debian-ppc.polarhome.com @@ -166,6 +167,7 @@ netbsd.polarhome.com openbsd.polarhome.com openindiana pidora +qnx raspbian redhat.polarhome.com scosysv.polarhome.com @@ -195,6 +197,7 @@ netbsd.polarhome.com Missing name for redirect. openbsd.polarhome.com bash only A openindiana.polarhome.com bash only A pidora.polarhome.com bash only A +qnx.polarhome.com /bin/sh: syntax error: `(' unexpected raspbian.polarhome.com bash only A redhat.polarhome.com bash only A scosysv.polarhome.com bash only A