diff --git a/doc/release_new_version b/doc/release_new_version index 3750cd78..cc04d1af 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -213,9 +213,9 @@ cc:Tim Cuthbertson , Ryoichiro Suzuki , Jesse Alama -Subject: GNU Parallel 20160122 ('') released <<[stable]>> +Subject: GNU Parallel 20160222 ('N977GA') released <<[stable]>> -GNU Parallel 20160122 ('') <<[stable]>> has been released. It is available for download at: http://ftp.gnu.org/gnu/parallel/ +GNU Parallel 20160222 ('N977GA') <<[stable]>> has been released. It is available for download at: http://ftp.gnu.org/gnu/parallel/ <> @@ -243,6 +243,8 @@ for Big Data Applications https://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumb * <> +* Daily Quick Tip: parallel https://medium.com/@jacaetevha/daily-quick-tip-parallel-8772435f6582 + * Downloading a list of URLs http://blog.gypsydave5.com/2016/02/04/xargs-and-curl/ * Using GNU Parallel and ImageMagick for Conversions http://blog.fractalcomic.com/tutorials/using-gnu-parallel-and-imagemagick-for-conversions/ diff --git a/src/parallel b/src/parallel index cc96d343..0b578c0c 100755 --- a/src/parallel +++ b/src/parallel @@ -1182,6 +1182,7 @@ sub init_globals { $Global::debug = 0; $Global::verbose = 0; $Global::quoting = 0; + $Global::total_completed = 0; # Read only table with default --rpl values %Global::replace = ( @@ -1460,6 +1461,7 @@ sub open_joblog { if(/$joblog_regexp/o) { # This is 30% faster than set_job_already_run($1); vec($Global::job_already_run,($1||0),1) = 1; + $Global::total_completed++; $group[$1-1] = "true"; } elsif(/(\d+)\s+\S+(\s+[-0-9.]+){6}\s+(.*)$/) { $group[$1-1] = $3 @@ -1495,6 +1497,7 @@ sub open_joblog { if(/$joblog_regexp/o) { # This is 30% faster than set_job_already_run($1); vec($Global::job_already_run,($1||0),1) = 1; + $Global::total_completed++; } elsif(not /\d+\s+[^\s]+\s+([-0-9.]+\s+){6}/) { ::error("Format of '$opt::joblog' is wrong: $_"); ::wait_and_exit(255); @@ -2026,7 +2029,6 @@ sub init_run_jobs { # Returns: N/A $Global::total_running = 0; $Global::total_started = 0; - $Global::total_completed = 0; $Global::tty_taken = 0; $SIG{USR1} = \&list_running_jobs; $SIG{USR2} = \&toggle_progress; @@ -4747,7 +4749,7 @@ sub loadavg_too_high { 'ultrix' => "ps -ax | awk '{print \$3,\$5}'", ); print `$ps{$^O}`; - }; + }); $cmd = "perl -e ".::shell_quote_scalar($ps); } return $cmd; diff --git a/src/parallel.pod b/src/parallel.pod index c1bd23cc..fea5886a 100644 --- a/src/parallel.pod +++ b/src/parallel.pod @@ -2420,6 +2420,14 @@ Find the files in a list that do not exist cat file_list | parallel 'if [ ! -e {} ] ; then echo {}; fi' +=head1 EXAMPLE: Composed command with multiple input sources + +You have a dir with files named as 24 hours in 5 minute intervals: +00:00, 00:05, 00:10 .. 23:55. You want to find the files missing: + + parallel [ -f {1}:{2} ] "||" echo {1}:{2} does not exist ::: {00..23} ::: {00..55..5} + + =head1 EXAMPLE: Calling Bash functions If the composed command is longer than a line, it becomes hard to diff --git a/src/parallel_design.pod b/src/parallel_design.pod index 6bbbbcec..fc981f52 100644 --- a/src/parallel_design.pod +++ b/src/parallel_design.pod @@ -754,8 +754,8 @@ Unfortunately it is not always possible to predict the root cause of the error. Contrary to the obvious B<--load> does not use load average. This is due to load average rising too slowly. Instead it uses B to list -the number of jobs in running or blocked state (state D, O or R). This -gives an instant load. +the number of threads in running or blocked state (state D, O or +R). This gives an instant load. As remote calculation of load can be slow, a process is spawned to run B and put the result in a file, which is then used next time. diff --git a/src/parallel_tutorial.html b/src/parallel_tutorial.html index 96518891..304c1840 100644 --- a/src/parallel_tutorial.html +++ b/src/parallel_tutorial.html @@ -590,6 +590,10 @@
  parallel echo {} shell quoted is {= '$_=Q($_)' =} ::: '*/!#$'
+

Output:

+ +
  */!#$ shell quoted is \*/\!\#\$
+

$job->skip() skips the job:

  parallel echo {= 'if($_==3) { $job->skip() }' =} ::: {1..5}
diff --git a/src/parallel_tutorial.pod b/src/parallel_tutorial.pod index f588402a..04dd2d4e 100644 --- a/src/parallel_tutorial.pod +++ b/src/parallel_tutorial.pod @@ -477,6 +477,10 @@ B shell quotes the string: parallel echo {} shell quoted is {= '$_=Q($_)' =} ::: '*/!#$' +Output: + + */!#$ shell quoted is \*/\!\#\$ + B<$job->>B skips the job: parallel echo {= 'if($_==3) { $job->skip() }' =} ::: {1..5} diff --git a/testsuite/tests-to-run/parallel-local-0.3s.sh b/testsuite/tests-to-run/parallel-local-0.3s.sh index 6d90fc8a..c1fd556b 100644 --- a/testsuite/tests-to-run/parallel-local-0.3s.sh +++ b/testsuite/tests-to-run/parallel-local-0.3s.sh @@ -125,26 +125,26 @@ echo 'bug #41613: --compress --line-buffer - no newline'; parallel --line-buffer echo {} O ::: -n; echo "K" echo 'Compress with failing (de)compressor' - parallel -k --tag --compress --compress-program 'cat;true' --decompress-program 'cat;true' echo ::: tag true true - parallel -k --tag --compress --compress-program 'cat;false' --decompress-program 'cat;true' echo ::: tag false true - parallel -k --tag --compress --compress-program 'cat;false' --decompress-program 'cat;false' echo ::: tag false false - parallel -k --tag --compress --compress-program 'cat;true' --decompress-program 'cat;false' echo ::: tag true false - parallel -k --compress --compress-program 'cat;true' --decompress-program 'cat;true' echo ::: true true - parallel -k --compress --compress-program 'cat;false' --decompress-program 'cat;true' echo ::: false true - parallel -k --compress --compress-program 'cat;false' --decompress-program 'cat;false' echo ::: false false - parallel -k --compress --compress-program 'cat;true' --decompress-program 'cat;false' echo ::: true false - parallel -k --line-buffer --compress --compress-program 'cat;true' --decompress-program 'cat;true' echo ::: line-buffer true true - parallel -k --line-buffer --compress --compress-program 'cat;false' --decompress-program 'cat;true' echo ::: line-buffer false true - parallel -k --line-buffer --compress --compress-program 'cat;false' --decompress-program 'cat;false' echo ::: line-buffer false false + parallel -k --tag --compress --compress-program 'cat;true' --decompress-program 'cat;true' echo ::: tag true true + parallel -k --tag --compress --compress-program 'cat;false' --decompress-program 'cat;true' echo ::: tag false true + parallel -k --tag --compress --compress-program 'cat;false' --decompress-program 'cat;false' echo ::: tag false false + parallel -k --tag --compress --compress-program 'cat;true' --decompress-program 'cat;false' echo ::: tag true false + parallel -k --compress --compress-program 'cat;true' --decompress-program 'cat;true' echo ::: true true + parallel -k --compress --compress-program 'cat;false' --decompress-program 'cat;true' echo ::: false true + parallel -k --compress --compress-program 'cat;false' --decompress-program 'cat;false' echo ::: false false + parallel -k --compress --compress-program 'cat;true' --decompress-program 'cat;false' echo ::: true false + parallel -k --line-buffer --compress --compress-program 'cat;true' --decompress-program 'cat;true' echo ::: line-buffer true true + parallel -k --line-buffer --compress --compress-program 'cat;false' --decompress-program 'cat;true' echo ::: line-buffer false true + parallel -k --line-buffer --compress --compress-program 'cat;false' --decompress-program 'cat;false' echo ::: line-buffer false false parallel -k --tag --line-buffer --compress --compress-program 'cat;true' --decompress-program 'cat;false' echo ::: tag line-buffer true false parallel -k --tag --line-buffer --compress --compress-program 'cat;true' --decompress-program 'cat;true' echo ::: tag line-buffer true true parallel -k --tag --line-buffer --compress --compress-program 'cat;false' --decompress-program 'cat;true' echo ::: tag line-buffer false true parallel -k --tag --line-buffer --compress --compress-program 'cat;false' --decompress-program 'cat;false' echo ::: tag line-buffer false false parallel -k --tag --line-buffer --compress --compress-program 'cat;true' --decompress-program 'cat;false' echo ::: tag line-buffer true false - parallel -k --files --compress --compress-program 'cat;true' --decompress-program 'cat;true' echo ::: files true true | parallel rm - parallel -k --files --compress --compress-program 'cat;false' --decompress-program 'cat;true' echo ::: files false true | parallel rm - parallel -k --files --compress --compress-program 'cat;false' --decompress-program 'cat;false' echo ::: files false false | parallel rm - parallel -k --files --compress --compress-program 'cat;true' --decompress-program 'cat;false' echo ::: files true false | parallel rm + parallel -k --files --compress --compress-program 'cat;true' --decompress-program 'cat;true' echo ::: files true true | parallel rm + parallel -k --files --compress --compress-program 'cat;false' --decompress-program 'cat;true' echo ::: files false true | parallel rm + parallel -k --files --compress --compress-program 'cat;false' --decompress-program 'cat;false' echo ::: files false false | parallel rm + parallel -k --files --compress --compress-program 'cat;true' --decompress-program 'cat;false' echo ::: files true false | parallel rm echo 'bug #44250: pxz complains File format not recognized but decompresses anyway' # The first line dumps core if run from make file. Why?! @@ -263,6 +263,14 @@ echo 'bug #47002: --tagstring with -d \n\n' echo '**' +echo 'bug #47086: [PATCH] Initialize total_completed from joblog' + + rm -f /tmp/parallel-47086; + parallel -j1 --joblog /tmp/parallel-47086 --halt now,fail=1 echo '{= $_=$Global::total_completed =};exit {}' ::: 0 0 0 1 0 0; + parallel -j1 --joblog /tmp/parallel-47086 --halt now,fail=1 --resume echo '{= $_=$Global::total_completed =};exit {}' ::: 0 0 0 1 0 0 + +echo '**' + EOF echo '### 1 .par file from --files expected' diff --git a/testsuite/wanted-results/parallel-local-0.3s b/testsuite/wanted-results/parallel-local-0.3s index 847a1339..e81a6192 100644 --- a/testsuite/wanted-results/parallel-local-0.3s +++ b/testsuite/wanted-results/parallel-local-0.3s @@ -168,56 +168,56 @@ nopipe nocompress notagstring OK echo 'Compress with failing (de)compressor' Compress with failing (de)compressor - parallel -k --tag --compress --compress-program 'cat;true' --decompress-program 'cat;true' echo ::: tag true true + parallel -k --tag --compress --compress-program 'cat;true' --decompress-program 'cat;true' echo ::: tag true true tag tag true true true true - parallel -k --tag --compress --compress-program 'cat;false' --decompress-program 'cat;true' echo ::: tag false true + parallel -k --tag --compress --compress-program 'cat;false' --decompress-program 'cat;true' echo ::: tag false true tag tag false false true true parallel: Error: cat;false failed. parallel: Error: cat;false failed. parallel: Error: cat;false failed. - parallel -k --tag --compress --compress-program 'cat;false' --decompress-program 'cat;false' echo ::: tag false false + parallel -k --tag --compress --compress-program 'cat;false' --decompress-program 'cat;false' echo ::: tag false false tag tag false false false false parallel: Error: cat;false failed. parallel: Error: cat;false failed. parallel: Error: cat;false failed. - parallel -k --tag --compress --compress-program 'cat;true' --decompress-program 'cat;false' echo ::: tag true false + parallel -k --tag --compress --compress-program 'cat;true' --decompress-program 'cat;false' echo ::: tag true false tag tag true true false false - parallel -k --compress --compress-program 'cat;true' --decompress-program 'cat;true' echo ::: true true + parallel -k --compress --compress-program 'cat;true' --decompress-program 'cat;true' echo ::: true true true true - parallel -k --compress --compress-program 'cat;false' --decompress-program 'cat;true' echo ::: false true + parallel -k --compress --compress-program 'cat;false' --decompress-program 'cat;true' echo ::: false true false true parallel: Error: cat;false failed. parallel: Error: cat;false failed. - parallel -k --compress --compress-program 'cat;false' --decompress-program 'cat;false' echo ::: false false + parallel -k --compress --compress-program 'cat;false' --decompress-program 'cat;false' echo ::: false false false false parallel: Error: cat;false failed. parallel: Error: cat;false failed. - parallel -k --compress --compress-program 'cat;true' --decompress-program 'cat;false' echo ::: true false + parallel -k --compress --compress-program 'cat;true' --decompress-program 'cat;false' echo ::: true false true false - parallel -k --line-buffer --compress --compress-program 'cat;true' --decompress-program 'cat;true' echo ::: line-buffer true true + parallel -k --line-buffer --compress --compress-program 'cat;true' --decompress-program 'cat;true' echo ::: line-buffer true true line-buffer true true - parallel -k --line-buffer --compress --compress-program 'cat;false' --decompress-program 'cat;true' echo ::: line-buffer false true + parallel -k --line-buffer --compress --compress-program 'cat;false' --decompress-program 'cat;true' echo ::: line-buffer false true line-buffer false true parallel: Error: cat;false failed. parallel: Error: cat;false failed. parallel: Error: cat;false failed. - parallel -k --line-buffer --compress --compress-program 'cat;false' --decompress-program 'cat;false' echo ::: line-buffer false false + parallel -k --line-buffer --compress --compress-program 'cat;false' --decompress-program 'cat;false' echo ::: line-buffer false false line-buffer false false @@ -257,16 +257,16 @@ tag tag line-buffer line-buffer true true false false - parallel -k --files --compress --compress-program 'cat;true' --decompress-program 'cat;true' echo ::: files true true | parallel rm - parallel -k --files --compress --compress-program 'cat;false' --decompress-program 'cat;true' echo ::: files false true | parallel rm + parallel -k --files --compress --compress-program 'cat;true' --decompress-program 'cat;true' echo ::: files true true | parallel rm + parallel -k --files --compress --compress-program 'cat;false' --decompress-program 'cat;true' echo ::: files false true | parallel rm parallel: Error: cat;false failed. parallel: Error: cat;false failed. parallel: Error: cat;false failed. - parallel -k --files --compress --compress-program 'cat;false' --decompress-program 'cat;false' echo ::: files false false | parallel rm + parallel -k --files --compress --compress-program 'cat;false' --decompress-program 'cat;false' echo ::: files false false | parallel rm parallel: Error: cat;false failed. parallel: Error: cat;false failed. parallel: Error: cat;false failed. - parallel -k --files --compress --compress-program 'cat;true' --decompress-program 'cat;false' echo ::: files true false | parallel rm + parallel -k --files --compress --compress-program 'cat;true' --decompress-program 'cat;false' echo ::: files true false | parallel rm echo 'bug #44250: pxz complains File format not recognized but decompresses anyway' bug #44250: pxz complains File format not recognized but decompresses anyway # The first line dumps core if run from make file. Why?! @@ -575,5 +575,18 @@ bug #47002: --tagstring with -d \n\n 2 echo '**' ** +echo 'bug #47086: [PATCH] Initialize total_completed from joblog' +bug #47086: [PATCH] Initialize total_completed from joblog + rm -f /tmp/parallel-47086; parallel -j1 --joblog /tmp/parallel-47086 --halt now,fail=1 echo '{= $_=$Global::total_completed =};exit {}' ::: 0 0 0 1 0 0; parallel -j1 --joblog /tmp/parallel-47086 --halt now,fail=1 --resume echo '{= $_=$Global::total_completed =};exit {}' ::: 0 0 0 1 0 0 +0 +1 +2 +3 +4 +5 +parallel: This job failed: +echo 3;exit 1 +echo '**' +** ### 1 .par file from --files expected 1