From dd25255ca8c44a6edfd58b469b8b49eae98e517b Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Tue, 16 May 2017 21:22:07 +0200 Subject: [PATCH] parallel: Fixed bug #51039: --dry-run --timeout 3600 -u breaks. --- src/Makefile.am | 2 +- src/Makefile.in | 2 +- src/parallel | 7 +- src/parallel.pod | 6 +- testsuite/tests-to-run/parallel-local-10s.sh | 2 +- testsuite/tests-to-run/parallel-local-3s.sh | 4 + testsuite/tests-to-run/parallel-tutorial.sh | 3 +- testsuite/wanted-results/parallel-local-3s | 2 + testsuite/wanted-results/parallel-tutorial | 149 ------------------- 9 files changed, 20 insertions(+), 157 deletions(-) diff --git a/src/Makefile.am b/src/Makefile.am index bfb76a23..210384a4 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -148,7 +148,7 @@ parcat.html: parcat niceload.html || echo "Warning: pod2html not found. Using old parcat.html" rm -f $(srcdir)/pod2htm* -# Depending on niceload.html to avoid stupid pod2html race condition +# Depending on parcat.html to avoid stupid pod2html race condition parset.html: parset.pod parcat.html pod2html --title "GNU parset" $(srcdir)/parset.pod > $(srcdir)/parset.htmln \ && mv $(srcdir)/parset.htmln $(srcdir)/parset.html \ diff --git a/src/Makefile.in b/src/Makefile.in index 0495335d..1c8b584b 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -736,7 +736,7 @@ parcat.html: parcat niceload.html || echo "Warning: pod2html not found. Using old parcat.html" rm -f $(srcdir)/pod2htm* -# Depending on niceload.html to avoid stupid pod2html race condition +# Depending on parcat.html to avoid stupid pod2html race condition parset.html: parset.pod parcat.html pod2html --title "GNU parset" $(srcdir)/parset.pod > $(srcdir)/parset.htmln \ && mv $(srcdir)/parset.htmln $(srcdir)/parset.html \ diff --git a/src/parallel b/src/parallel index 110e1f47..9142afa9 100755 --- a/src/parallel +++ b/src/parallel @@ -1217,6 +1217,11 @@ sub parse_options { $opt::decompress_program = "cat >/dev/null"; } } + if(defined $opt::dryrun) { + # Force grouping due to bug #51039: --dry-run --timeout 3600 -u breaks + $opt::ungroup = 0; + $opt::group = 1; + } if(defined $opt::nonall) { # Append a dummy empty argument if there are no arguments # on the command line to avoid reading from STDIN. @@ -1371,7 +1376,7 @@ sub check_invalid_option_combinations { sub init_globals { # Defaults: - $Global::version = 20170512; + $Global::version = 20170515; $Global::progname = 'parallel'; $Global::infinity = 2**31; $Global::debug = 0; diff --git a/src/parallel.pod b/src/parallel.pod index a9161237..9a73c080 100644 --- a/src/parallel.pod +++ b/src/parallel.pod @@ -2300,9 +2300,9 @@ computed as a percentage of the median average runtime of successful jobs. Only values > 100% will make sense. I is normally in seconds, but can be floats postfixed with -s, m, h, or d which would multiply the float by 1, 60, 3600, or -86400. Thus these are equivalent: B<--timeout 100000> and B<--timeout -1d3h46.6m4s>. +B, B, B, or B which would multiply the float by 1, 60, +3600, or 86400. Thus these are equivalent: B<--timeout 100000> and +B<--timeout 1d3.5h16.6m4s>. =item B<--verbose> diff --git a/testsuite/tests-to-run/parallel-local-10s.sh b/testsuite/tests-to-run/parallel-local-10s.sh index aca0c699..ee9dce26 100644 --- a/testsuite/tests-to-run/parallel-local-10s.sh +++ b/testsuite/tests-to-run/parallel-local-10s.sh @@ -317,7 +317,7 @@ par_plus_dyn_repl() { par_slow_total_jobs() { echo 'bug #51006: Slow total_jobs() eats job' (echo a; sleep 6; echo b; sleep 6; seq 2) | - parallel echo '{=total_jobs()=}' + parallel -k echo '{=total_jobs()=}' } export -f $(compgen -A function | grep par_) diff --git a/testsuite/tests-to-run/parallel-local-3s.sh b/testsuite/tests-to-run/parallel-local-3s.sh index af09b441..7f700fa5 100644 --- a/testsuite/tests-to-run/parallel-local-3s.sh +++ b/testsuite/tests-to-run/parallel-local-3s.sh @@ -113,6 +113,10 @@ echo finish {}' echo finish {}' } +par_dryrun_timeout_ungroup() { + echo 'bug #51039: --dry-run --timeout 3600 -u breaks' + seq 1000 | stdout parallel --dry-run --timeout 100 -u --jobs 10 echo | wc +} export -f $(compgen -A function | grep par_) compgen -A function | grep par_ | sort | parallel -j6 --tag -k '{} 2>&1' diff --git a/testsuite/tests-to-run/parallel-tutorial.sh b/testsuite/tests-to-run/parallel-tutorial.sh index 2fc00b5b..5ed53572 100644 --- a/testsuite/tests-to-run/parallel-tutorial.sh +++ b/testsuite/tests-to-run/parallel-tutorial.sh @@ -80,7 +80,8 @@ perl -ne '$/="\n\n"; /^Output/../^[^O]\S/ and next; /^ / and print;' ../../src/ s/cat: input_file: No such file or directory\n//; s{rsync: link_stat ".*/home/parallel/input_file.out" .*\n}{}; s{rsync error: some files/attrs were not transferred .*\n}{}; -' +' | + uniq # 3+3 .par files (from --files), 1 .tms-file from tmux attach find {$TMPDIR,/var/tmp,/tmp}/{fif,tms,par[^a]}* -mmin -10 2>/dev/null | wc -l find {$TMPDIR,/var/tmp,/tmp}/{fif,tms,par[^a]}* -mmin -10 2>/dev/null | parallel rm diff --git a/testsuite/wanted-results/parallel-local-3s b/testsuite/wanted-results/parallel-local-3s index 66f5c6e6..0ac3085e 100644 --- a/testsuite/wanted-results/parallel-local-3s +++ b/testsuite/wanted-results/parallel-local-3s @@ -4,6 +4,8 @@ par_children_receive_sig Got TERM par_children_receive_sig Got TERM par_children_receive_sig Got INT par_children_receive_sig Got TERM +par_dryrun_timeout_ungroup bug #51039: --dry-run --timeout 3600 -u breaks +par_dryrun_timeout_ungroup 1000 2000 8893 par_keeporder_roundrobin bug #50081: --keep-order --round-robin should give predictable results par_keeporder_roundrobin OK par_kill_int_twice ### Are children killed if GNU Parallel receives INT twice? There should be no sleep at the end diff --git a/testsuite/wanted-results/parallel-tutorial b/testsuite/wanted-results/parallel-tutorial index af3c4e70..2f43fb65 100644 --- a/testsuite/wanted-results/parallel-tutorial +++ b/testsuite/wanted-results/parallel-tutorial @@ -15,14 +15,6 @@ (echo %head1; echo %head2; \ sleep .3 /bin/bash: -c: line 2: syntax error: unexpected end of file -sleep .3 -sleep .3 -sleep .3 -sleep .3 -sleep .3 -sleep .3 -sleep .3 -sleep .3 sleep .3 perl -e 'for(1..10){print "$_\n"}') > num_%header /bin/bash: -c: line 0: syntax error near unexpected token `)' @@ -155,7 +147,6 @@ C A B (echo 1; echo; echo 2) | parallel --no-run-if-empty echo -9 9 parallel ::: ls 'echo foo' pwd abc-file @@ -191,12 +182,8 @@ A parallel echo {/.} ::: A/B.C B parallel echo {#} ::: A B C -9 -9 9 parallel -j 2 echo {%} ::: A B C -9 -9 9 parallel -I ,, echo ,, ::: A/B.C A/B.C @@ -209,12 +196,8 @@ A parallel --basenameextensionreplace ,, echo ,, ::: A/B.C B parallel --seqreplace ,, echo ,, ::: A B C -9 -9 9 parallel -j2 --slotreplace ,, echo ,, ::: A B C -9 -9 9 parallel echo '{= s:\.[^.]+$::;s:\.[^.]+$::; =}' ::: foo.tar.gz foo @@ -227,9 +210,6 @@ Job 5 of 5 parallel echo {} shell quoted is {= '$_=Q($_)' =} ::: '*/!#$' */!#$ shell quoted is \*/\!\#\$ parallel echo {= 'if($_==3) { skip() }' =} ::: {1..5} -9 -9 -9 9 parallel echo {= 'if($arg[1]==$arg[2]) { skip() }' =} \ ::: {1..3} ::: {1..3} @@ -313,13 +293,6 @@ f1=C f2=D parallel --plus echo {+/}/{/..}.{+..} ::: dir/sub/file.ex1.ex2.ex3 parallel --plus echo {...}.{+...} ::: dir/sub/file.ex1.ex2.ex3 parallel --plus echo {+/}/{/...}.{+...} ::: dir/sub/file.ex1.ex2.ex3 -dir/sub/file.ex1.ex2.ex3 -dir/sub/file.ex1.ex2.ex3 -dir/sub/file.ex1.ex2.ex3 -dir/sub/file.ex1.ex2.ex3 -dir/sub/file.ex1.ex2.ex3 -dir/sub/file.ex1.ex2.ex3 -dir/sub/file.ex1.ex2.ex3 dir/sub/file.ex1.ex2.ex3 parallel --plus echo Job {#} of {##} ::: {1..5} Job 1 of 5 @@ -330,53 +303,38 @@ Job 5 of 5 unset myvar echo ${myvar:-myval} parallel --plus echo {:-myval} ::: "$myvar" -myval myval myvar=abcAaAdef echo ${myvar:2} parallel --plus echo {:2} ::: "$myvar" -cAaAdef cAaAdef echo ${myvar:2:3} parallel --plus echo {:2:3} ::: "$myvar" - echo ${myvar#bc} parallel --plus echo {#bc} ::: "$myvar" echo ${myvar#abc} parallel --plus echo {#abc} ::: "$myvar" - - - echo ${myvar%de} parallel --plus echo {%de} ::: "$myvar" echo ${myvar%def} parallel --plus echo {%def} ::: "$myvar" - - - echo ${myvar/def/ghi} parallel --plus echo {/def/ghi} ::: "$myvar" - echo ${myvar^a} parallel --plus echo {^a} ::: "$myvar" echo ${myvar^^a} parallel --plus echo {^^a} ::: "$myvar" - - - myvar=AbcAaAdef echo ${myvar,A} parallel --plus echo '{,A}' ::: "$myvar" echo ${myvar,,A} parallel --plus echo '{,,A}' ::: "$myvar" abcAaAdef -abcAaAdef -abcaaadef abcaaadef cat num30000 | parallel --xargs echo | wc -l 9 @@ -403,8 +361,6 @@ G H 1=D 2=E 3=F 1=G 2=H 3= parallel -N0 echo foo ::: 1 2 3 -foo -foo foo perl -e 'print "@ARGV\n"' A A @@ -508,12 +464,8 @@ echo C 1-middle 1-end parallel --files echo ::: A B C -/tmp/parallel-tutorial-tmpdir/parXXXXX.par -/tmp/parallel-tutorial-tmpdir/parXXXXX.par /tmp/parallel-tutorial-tmpdir/parXXXXX.par parallel --tmpdir /var/tmp --files echo ::: A B C -/var/tmp/parXXXXX.par -/var/tmp/parXXXXX.par /var/tmp/parXXXXX.par parallel --results outdir echo ::: A B C A @@ -582,32 +534,6 @@ B D 9 parallel --use-cpus-instead-of-cores -N0 sleep 1 :::: num8 parallel --shuf echo ::: 1 2 3 ::: a b c ::: A B C -123 abc ABC -123 abc ABC -123 abc ABC -123 abc ABC -123 abc ABC -123 abc ABC -123 abc ABC -123 abc ABC -123 abc ABC -123 abc ABC -123 abc ABC -123 abc ABC -123 abc ABC -123 abc ABC -123 abc ABC -123 abc ABC -123 abc ABC -123 abc ABC -123 abc ABC -123 abc ABC -123 abc ABC -123 abc ABC -123 abc ABC -123 abc ABC -123 abc ABC -123 abc ABC 123 abc ABC seq 10 20 | parallel --tmux 'echo start {}; sleep {}; echo done {}' @@ -622,12 +548,8 @@ DATE OUTPUT Starting 3 DATE OUTPUT parallel --timeout 4.1 sleep {}\; echo {} ::: 2 4 6 8 -9 9 parallel --timeout 200% sleep {}\; echo {} ::: 2.1 2.2 3 7 2.3 -9 -9 -9 9 parallel --eta sleep ::: 1 3 2 2 1 3 3 2 1 @@ -697,9 +619,6 @@ Seq Host Starttime JobRuntime Send Receive Exitval Signal Command 2 : TIMESTAMP 9.999 0 0 2 0 exit 2 3 : TIMESTAMP 9.999 0 0 3 0 exit 3 9 -9 -9 -9 parallel: This job failed: echo X; exit X parallel: Starting no more jobs. Waiting for 1 jobs to finish. @@ -707,16 +626,11 @@ parallel: This job failed: echo X; exit X parallel -j2 --halt now,fail=1 echo {}\; exit {} ::: 0 0 1 2 3 9 -9 -9 parallel: This job failed: echo X; exit X parallel -j2 --halt soon,fail=20% echo {}\; exit {} \ ::: 0 1 2 3 4 5 6 7 8 9 9 -9 -9 -9 parallel: This job failed: echo X; exit X parallel: This job failed: @@ -726,11 +640,6 @@ parallel: This job failed: echo X; exit X parallel -j2 --halt now,success=1 echo {}\; exit {} ::: 1 2 3 0 4 5 6 9 -9 -9 -9 -9 -9 parallel: This job succeeded: echo X; exit X parallel -k --retries 3 \ @@ -741,7 +650,6 @@ completed 2 completed 0 tried 1 tried 0 -tried 1 tried 1 show_signals() { perl -e 'for(keys %SIG) { @@ -824,12 +732,8 @@ foo BASE64 /mnt/4tb/home/parallel/.TMPWORKDIR parallel -S $SERVER1 --sshdelay 0.2 echo ::: 1 2 3 -9 -9 9 parallel --controlmaster -S $SERVER1 echo ::: 1 2 3 -9 -9 9 parallel --filter-hosts -S 173.194.32.46,$SERVER1 echo ::: bar bar @@ -1079,48 +983,20 @@ ssh -l parallel lo -- mkdir -p ./.TMPWORKDIR && rsync --protocol 30 -rlDzR -essh seq ::: 10 ::: 12 13 14 cat /tmp/log.csv 99 -99 -99 -99 -99 -99 -99 -99 -99 -99 -99 -99 Seq,Host,Starttime,JobRuntime,Send,Receive,Exitval,_Signal,Command,V1,V2,Stdout,Stderr 1,:,000000000.000,0.000,0,9,0,0,"seq 10 12",10,12,"10 99 -99 ", 2,:,000000000.000,0.000,0,12,0,0,"seq 10 13",10,13,"10 99 -99 -99 ", 3,:,000000000.000,0.000,0,15,0,0,"seq 10 14",10,14,"10 99 -99 -99 -99 ", parallel --results outdir --sqlandworker csv:////%2Ftmp%2Flog2.csv \ seq ::: 10 ::: 12 13 14 cat /tmp/log2.csv 99 -99 -99 -99 -99 -99 -99 -99 -99 -99 -99 -99 Seq,Host,Starttime,JobRuntime,Send,Receive,Exitval,_Signal,Command,V1,V2,Stdout,Stderr 1,:,000000000.000,0.000,0,9,0,0,"seq 10 12",10,12,outdir/1/10/2/12/stdout,outdir/1/10/2/12/stderr 2,:,000000000.000,0.000,0,12,0,0,"seq 10 13",10,13,outdir/1/10/2/13/stdout,outdir/1/10/2/13/stderr @@ -1148,38 +1024,18 @@ parallel: Error: echo is not a valid DBURL parallel --sqlworker $DBURLTABLE echo ::: foo bar ::: baz quuz parallel: Error: echo is not a valid DBURL cat num1000000 | parallel --pipe wc - 999999 999999 9999999 - 999999 999999 9999999 - 999999 999999 9999999 - 999999 999999 9999999 - 999999 999999 9999999 999999 999999 9999999 85352 85352 597465 cat num1000000 | parallel --pipe --block 2M wc - 999999 999999 9999999 - 999999 999999 9999999 999999 999999 9999999 85349 85349 597444 cat num1000000 | parallel --pipe -j4 --round-robin wc - 999999 999999 9999999 - 999999 999999 9999999 - 999999 999999 9999999 999999 999999 9999999 cat num1000000 | parallel --pipe -N140000 wc 140000 140000 868895 - 140000 140000 980000 - 140000 140000 980000 - 140000 140000 980000 - 140000 140000 980000 - 140000 140000 980000 140000 140000 980000 20000 20000 140001 cat num1000000 | parallel --pipe -L75 wc - 999999 999999 9999999 - 999999 999999 9999999 - 999999 999999 9999999 - 999999 999999 9999999 - 999999 999999 9999999 999999 999999 9999999 85500 85500 598500 25 25 176 @@ -1248,7 +1104,6 @@ cat: num_%header: No such file or directory cat: num_%header: No such file or directory parallel --pipepart -a num1000000 --block 3m wc 999999 999999 9999999 - 999999 999999 9999999 126984 126984 888889 cat input_file | parallel command /bin/bash: This is input_file: command not found @@ -1273,7 +1128,6 @@ cat: num_%header: No such file or directory Unescaped left brace in regex is deprecated, passed through in regex; marked by <-- HERE in m/%{ <-- HERE (.*?)}/ at /usr/bin/print line 528. Error: no such file "@ARGV\n" parallel perl_echo ::: foo bar -/bin/bash: perl_echo: command not found /bin/bash: perl_echo: command not found #!/usr/bin/parallel --shebang-wrap /usr/bin/perl print "@ARGV\n" @@ -1288,7 +1142,6 @@ Unescaped left brace in regex is deprecated, passed through in regex; marked by Error: no such file "Arguments @ARGV\n" #!/usr/bin/parallel --shebang-wrap /usr/bin/python - print 'Arguments', str(sys.argv) /bin/bash: -c: line 3: syntax error near unexpected token `(' /bin/bash: -c: line 3: ` print 'Arguments', str(sys.argv)' @@ -1497,10 +1350,8 @@ If you pay 10000 EUR you should feel free to use GNU Parallel without citing. If you send a copy of your published article to tange@gnu.org, it will be mentioned in the release notes of next version of GNU Parallel. - parallel --number-of-cpus parallel --number-of-cores -9 9 echo '--nice 17' > ~/.parallel/nicetimeout echo '--timeout 300%' >> ~/.parallel/nicetimeout