Fixed bug #51006: Slow total_jobs() eats job

doc/release_new_version

@@ -196,9 +196,9 @@ file:///home/tange/privat/parallel/doc/release_new_version
 from:tange@gnu.org
 to:parallel@gnu.org, bug-parallel@gnu.org
 
-Subject: GNU Parallel 20170522 ('') released <<[stable]>>
+Subject: GNU Parallel 20170522 ('Macron') released <<[stable]>>
 
-GNU Parallel 20170522 ('') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/
+GNU Parallel 20170522 ('Macron') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/
 
 <<No new functionality was introduced so this is a good candidate for a stable release.>>
 
@@ -214,10 +214,14 @@ New in this release:
   http://meta.askubuntu.com/a/16750/22307
   http://meta.serverfault.com/a/9040/45704
 
+https://github.com/jupiter126/Create_Speech_Dataset
+
 http://pubs.rsc.org/-/content/articlelanding/2017/cp/c7cp01662j#!divAbstract
 
 ftp://frapftp.fire.ca.gov/outgoing/transfer/PUC_Map1/Utility%20Fire%20map%201%20final%20review%20and%20development%20report21616.pdf
 
+https://dzone.com/articles/batch-convert-images-from-png-to-jpeg
+
 https://arxiv.org/pdf/1704.02257.pdf
 
 http://www.sciencedirect.com/science/article/pii/S0044848616312194
@@ -232,9 +236,15 @@ http://biorxiv.org/content/biorxiv/early/2017/04/27/131680.full.pdf
 
 http://www.nature.com/nprot/journal/v12/n5/full/nprot.2017.022.html?WT.feed_name=subjects_biotechnology
 
+https://128.84.21.199/pdf/1704.08997.pdf
+
+<<Citation not ok: https://arxiv.org/pdf/1704.08997.pdf>>
+
 
 * <<Citation needed: https://sdstrowes.co.uk/publications/sdstrowes-rdns-aaaa.pdf>>
 
+* <<Citation not ok: https://www.nature.com/articles/s41598-017-01703-6>>
+
 * <<Possibly http://link.springer.com/chapter/10.1007%2F978-3-319-22053-6_46>>
 
 * <<Possibly http://link.springer.com/article/10.1007/s12021-015-9290-5>>

src/Makefile.am

@@ -76,13 +76,13 @@ niceload.1: niceload.pod
 
 parcat.1: parcat.pod
 	pod2man --release='$(PACKAGE_VERSION)' --center='$(PACKAGE_NAME)' \
-		--section=1 $(srcdir)/parcat > $(srcdir)/parcat.1n \
+		--section=1 $(srcdir)/parcat.pod > $(srcdir)/parcat.1n \
 	&& mv $(srcdir)/parcat.1n $(srcdir)/parcat.1 \
 	|| echo "Warning: pod2man not found. Using old parcat.1"
 
 parset.1: parset.pod
 	pod2man --release='$(PACKAGE_VERSION)' --center='$(PACKAGE_NAME)' \
-		--section=1 $(srcdir)/parset > $(srcdir)/parset.1n \
+		--section=1 $(srcdir)/parset.pod > $(srcdir)/parset.1n \
 	&& mv $(srcdir)/parset.1n $(srcdir)/parset.1 \
 	|| echo "Warning: pod2man not found. Using old parset.1"
 
@@ -149,8 +149,8 @@ parcat.html: parcat niceload.html
 	rm -f $(srcdir)/pod2htm*
 
 # Depending on niceload.html to avoid stupid pod2html race condition
-parset.html: parset parcat.html
+parset.html: parset.pod parcat.html
-	pod2html --title "GNU parset" $(srcdir)/parset > $(srcdir)/parset.htmln \
+	pod2html --title "GNU parset" $(srcdir)/parset.pod > $(srcdir)/parset.htmln \
 	&& mv $(srcdir)/parset.htmln $(srcdir)/parset.html \
 	|| echo "Warning: pod2html not found. Using old parset.html"
 	rm -f $(srcdir)/pod2htm*
@@ -192,7 +192,7 @@ parcat.texi: parcat
 	|| echo "Warning: pod2texi not found. Using old parcat.texi"
 
 parset.texi: parset.pod
-	pod2texi --output=$(srcdir)/parset.texi $(srcdir)/parset \
+	pod2texi --output=$(srcdir)/parset.texi $(srcdir)/parset.pod \
 	|| echo "Warning: pod2texi not found. Using old parset.texi"
 
 parallel.pdf: parallel.pod
@@ -232,7 +232,7 @@ parcat.pdf: parcat
 	|| echo "Warning: pod2pdf not found. Using old parcat.pdf"
 
 parset.pdf: parset.pod
-	pod2pdf --output-file $(srcdir)/parset.pdf $(srcdir)/parset --title "GNU parset" \
+	pod2pdf --output-file $(srcdir)/parset.pdf $(srcdir)/parset.pod --title "GNU parset" \
 	|| echo "Warning: pod2pdf not found. Using old parset.pdf"
 
 sem: parallel

src/Makefile.in

@@ -664,13 +664,13 @@ niceload.1: niceload.pod
 
 parcat.1: parcat.pod
 	pod2man --release='$(PACKAGE_VERSION)' --center='$(PACKAGE_NAME)' \
-		--section=1 $(srcdir)/parcat > $(srcdir)/parcat.1n \
+		--section=1 $(srcdir)/parcat.pod > $(srcdir)/parcat.1n \
 	&& mv $(srcdir)/parcat.1n $(srcdir)/parcat.1 \
 	|| echo "Warning: pod2man not found. Using old parcat.1"
 
 parset.1: parset.pod
 	pod2man --release='$(PACKAGE_VERSION)' --center='$(PACKAGE_NAME)' \
-		--section=1 $(srcdir)/parset > $(srcdir)/parset.1n \
+		--section=1 $(srcdir)/parset.pod > $(srcdir)/parset.1n \
 	&& mv $(srcdir)/parset.1n $(srcdir)/parset.1 \
 	|| echo "Warning: pod2man not found. Using old parset.1"
 
@@ -737,8 +737,8 @@ parcat.html: parcat niceload.html
 	rm -f $(srcdir)/pod2htm*
 
 # Depending on niceload.html to avoid stupid pod2html race condition
-parset.html: parset parcat.html
+parset.html: parset.pod parcat.html
-	pod2html --title "GNU parset" $(srcdir)/parset > $(srcdir)/parset.htmln \
+	pod2html --title "GNU parset" $(srcdir)/parset.pod > $(srcdir)/parset.htmln \
 	&& mv $(srcdir)/parset.htmln $(srcdir)/parset.html \
 	|| echo "Warning: pod2html not found. Using old parset.html"
 	rm -f $(srcdir)/pod2htm*
@@ -780,7 +780,7 @@ parcat.texi: parcat
 	|| echo "Warning: pod2texi not found. Using old parcat.texi"
 
 parset.texi: parset.pod
-	pod2texi --output=$(srcdir)/parset.texi $(srcdir)/parset \
+	pod2texi --output=$(srcdir)/parset.texi $(srcdir)/parset.pod \
 	|| echo "Warning: pod2texi not found. Using old parset.texi"
 
 parallel.pdf: parallel.pod
@@ -820,7 +820,7 @@ parcat.pdf: parcat
 	|| echo "Warning: pod2pdf not found. Using old parcat.pdf"
 
 parset.pdf: parset.pod
-	pod2pdf --output-file $(srcdir)/parset.pdf $(srcdir)/parset --title "GNU parset" \
+	pod2pdf --output-file $(srcdir)/parset.pdf $(srcdir)/parset.pod --title "GNU parset" \
 	|| echo "Warning: pod2pdf not found. Using old parset.pdf"
 
 sem: parallel

src/parallel

@@ -6525,6 +6525,7 @@ sub total_jobs {
 	    my $record_queue = $self->{'commandlinequeue'}{'arg_queue'};
 	    my $start = time;
 	    while($record = $record_queue->get()) {
+		push @arg_records, $record;
 		if(time - $start > 10) {
 		    ::warning("Reading ".scalar(@arg_records).
 			      " arguments took longer than 10 seconds.");
@@ -6533,7 +6534,6 @@ sub total_jobs {
 		    $opt::shuf && ::warning("Consider removing --shuf.");
 		    last;
 		}
-		push @arg_records, $record;
 	    }
 	    while($record = $record_queue->get()) {
 		push @arg_records, $record;

src/parallel_alternatives.pod

@@ -859,10 +859,14 @@ report this, but finishes with success - thereby risking data loss.
 
 =head2 DIFFERENCES BETWEEN Rush AND GNU Parallel
 
-Rush (https://github.com/shenwei356/rush) is written in Go and based
+B<rush> (https://github.com/shenwei356/rush) is written in Go and
-on gargs.
+based on B<gargs>.
 
-Rush has some string manipulations that can be emulated by putting
+Just like GNU B<parallel> B<rush> buffer in temporary files. But
+opposite GNU B<parallel> B<rush> does not clean up, if the process
+dies abnormally.
+
+B<rush> has some string manipulations that can be emulated by putting
 this into ~/.parallel/config (% is used instead of ^):
 
   --rpl '{:} s:(\.[^/]+)*$::'
@@ -929,6 +933,113 @@ B<12> Followed by:
 
 B<12> seq 1 3 | parallel --joblog mylog --retry-failed 'sleep {}; echo {}'
 
+B<rush> has:
+
+=over 4
+
+=item * B<awk -v> like custom defined variables (B<-v>)
+
+With GNU B<parallel> you would simply simply set a shell variable:
+
+   parallel 'v={}; echo "$v"' ::: foo
+   echo foo | rush -v v={} 'echo {v}'
+
+Also B<rush> does not like special chars. So these do not work:
+
+   echo does not work | rush -v v=\" 'echo {v}'
+   echo "My  brother's  12\"  records" | rush -v v={} 'echo {v}'
+
+Whereas the corresponding GNU B<parallel> version works:
+
+   parallel 'v=\"; echo "$v"' ::: works
+   parallel 'v={}; echo "$v"' ::: "My  brother's  12\"  records"
+
+=item * Exit on first error(s) (-e)
+
+This is called B<--halt now,fail=1> (or shorter: B<--halt 2>) when
+used with GNU B<parallel>.
+
+=item * Settable records sending to every command (B<-n>, default 1)
+
+This is also called B<-n> in GNU B<parallel>.
+
+=item * Practical replacement strings
+
+=over 4
+
+=item {:} remove any extension
+
+With GNU B<parallel> this can be emulated by:
+
+  parallel --plus echo '{/\..*/}' ::: foo.ext.bar.gz
+
+=item {^suffix}, remove suffix
+
+With GNU B<parallel> this can be emulated by:
+
+  parallel --plus echo '{%.bar.gz}' ::: foo.ext.bar.gz
+
+=item {%.}, {%:}, basename without extension
+
+With GNU B<parallel> this can be emulated by:
+
+  parallel echo '{= s:.*/::;s/\..*// =}' ::: dir/foo.bar.gz
+
+And if you need it often, you define a B<--rpl> in
+B<$HOME/.parallel/config>:
+
+  --rpl '{%.} s:.*/::;s/\..*//'
+  --rpl '{%:} s:.*/::;s/\..*//'
+
+Then you can use them as:
+
+  parallel echo {%.} {%:} ::: dir/foo.bar.gz
+
+=back
+
+=item * Preset variable (macro)
+
+E.g.
+
+  echo foosuffix | rush -v p={^suffix} 'echo {p}_new_suffix'
+
+With GNU B<parallel> this can be emulated by:
+
+  echo foosuffix | parallel --plus 'p={%suffix}; echo ${p}_new_suffix'
+
+Opposite B<rush> GNU B<parallel> works fine if the input contains
+double space, ' and ":
+
+  echo "1'6\"  foosuffix" |
+    parallel --plus 'p={%suffix}; echo "${p}"_new_suffix'
+
+
+=item * Commands of multi-lines
+
+To improve readibilty GNU B<parallel> encourages not to use multi-line
+commands. In most cases it can be written as a function:
+
+  seq 1 3 | parallel --timeout 2 --joblog my.log 'sleep {}; echo {}; \
+  echo finish {}'
+
+Could be written as:
+
+  doit() {
+    sleep "$1"
+    echo "$1"
+    echo finish "$1"
+  }
+  export -f doit
+  seq 1 3 | parallel --timeout 2 --joblog my.log doit
+
+The failed commands can be resumed with:
+
+  seq 1 3 |
+    parallel --resume-failed --joblog my.log 'sleep {}; echo {};\
+  echo finish {}'
+
+=back
+
 
 =head2 DIFFERENCES BETWEEN machma AND GNU Parallel
 

testsuite/tests-to-run/parallel-local-10s.sh

@@ -314,6 +314,12 @@ par_plus_dyn_repl() {
     parallel --plus echo '{-2,,A}' ::: "wrong" ::: "$myvar" ::: "wrong"
 }
 
+par_slow_total_jobs() {
+    echo 'bug #51006: Slow total_jobs() eats job'
+    (echo a; sleep 6; echo b; sleep 6; seq 2) |
+	parallel echo '{=total_jobs()=}'
+}
+
 export -f $(compgen -A function | grep par_)
 compgen -A function | grep par_ | sort |
     parallel --joblog /tmp/jl-`basename $0` -j10 --tag -k '{} 2>&1'

testsuite/tests-to-run/parallel-tutorial.sh

@@ -45,7 +45,7 @@ perl -ne '$/="\n\n"; /^Output/../^[^O]\S/ and next; /^  / and print;' ../../src/
             # Remote script
             s/(PARALLEL_PID\D+)\d+/${1}000000/g;
             # sql timing
-            s/,:,\d+.\d+,\d+.\d+/,:,000000000.000,0.000/g;
+            s/,[a-z]*,\d+.\d+,\d+.\d+/,:,000000000.000,0.000/g;
             # /usr/bin/time -f %e
             s/^(\d+)\.\d+$/$1/;
             # Base 64 string

testsuite/wanted-results/parallel-install

@@ -81,6 +81,18 @@ pod2man --release='00000000' --center='parallel' \
 || echo "Warning: pod2man not found. Using old parallel_alternatives.7"
 /bin/bash: pod2man: command not found
 Warning: pod2man not found. Using old parallel_alternatives.7
+pod2man --release='00000000' --center='parallel' \
+	--section=1 ./parcat.pod > ./parcat.1n \
+&& mv ./parcat.1n ./parcat.1 \
+|| echo "Warning: pod2man not found. Using old parcat.1"
+/bin/bash: pod2man: command not found
+Warning: pod2man not found. Using old parcat.1
+pod2man --release='00000000' --center='parallel' \
+	--section=1 ./parset.pod > ./parset.1n \
+&& mv ./parset.1n ./parset.1 \
+|| echo "Warning: pod2man not found. Using old parset.1"
+/bin/bash: pod2man: command not found
+Warning: pod2man not found. Using old parset.1
 pod2html --title "GNU Parallel" ./parallel.pod > ./parallel.htmln \
 && mv ./parallel.htmln ./parallel.html \
 || echo "Warning: pod2html not found. Using old parallel.html"
@@ -161,6 +173,10 @@ pod2texi --output=./parallel_alternatives.texi ./parallel_alternatives.pod \
 || echo "Warning: pod2texi not found. Using old parallel_alternatives.texi"
 /bin/bash: pod2texi: command not found
 Warning: pod2texi not found. Using old parallel_alternatives.texi
+pod2texi --output=./parset.texi ./parset.pod \
+|| echo "Warning: pod2texi not found. Using old parset.texi"
+/bin/bash: pod2texi: command not found
+Warning: pod2texi not found. Using old parset.texi
 pod2pdf --output-file ./parallel.pdf ./parallel.pod --title "GNU Parallel" \
 || echo "Warning: pod2pdf not found. Using old parallel.pdf"
 /bin/bash: pod2pdf: command not found
@@ -193,6 +209,10 @@ pod2pdf --output-file ./parallel_alternatives.pdf ./parallel_alternatives.pod --
 || echo "Warning: pod2pdf not found. Using old parallel_alternatives.pdf"
 /bin/bash: pod2pdf: command not found
 Warning: pod2pdf not found. Using old parallel_alternatives.pdf
+pod2pdf --output-file ./parset.pdf ./parset.pod --title "GNU parset" \
+|| echo "Warning: pod2pdf not found. Using old parset.pdf"
+/bin/bash: pod2pdf: command not found
+Warning: pod2pdf not found. Using old parset.pdf
 make[0]: Entering directory '/tmp/parallel-00000000/src'
  /bin/mkdir -p '/usr/local/bin'
  /usr/bin/install -c parallel sql niceload parcat env_parallel env_parallel.ash env_parallel.bash env_parallel.csh env_parallel.dash env_parallel.fish env_parallel.ksh env_parallel.pdksh env_parallel.sh env_parallel.tcsh env_parallel.zsh '/usr/local/bin'
@@ -281,6 +301,10 @@ pod2texi --output=./parallel_alternatives.texi ./parallel_alternatives.pod \
 || echo "Warning: pod2texi not found. Using old parallel_alternatives.texi"
 /bin/bash: pod2texi: command not found
 Warning: pod2texi not found. Using old parallel_alternatives.texi
+pod2texi --output=./parset.texi ./parset.pod \
+|| echo "Warning: pod2texi not found. Using old parset.texi"
+/bin/bash: pod2texi: command not found
+Warning: pod2texi not found. Using old parset.texi
 pod2pdf --output-file ./parallel.pdf ./parallel.pod --title "GNU Parallel" \
 || echo "Warning: pod2pdf not found. Using old parallel.pdf"
 /bin/bash: pod2pdf: command not found
@@ -313,6 +337,10 @@ pod2pdf --output-file ./parallel_alternatives.pdf ./parallel_alternatives.pod --
 || echo "Warning: pod2pdf not found. Using old parallel_alternatives.pdf"
 /bin/bash: pod2pdf: command not found
 Warning: pod2pdf not found. Using old parallel_alternatives.pdf
+pod2pdf --output-file ./parset.pdf ./parset.pod --title "GNU parset" \
+|| echo "Warning: pod2pdf not found. Using old parset.pdf"
+/bin/bash: pod2pdf: command not found
+Warning: pod2pdf not found. Using old parset.pdf
  /bin/mkdir -p '/usr/local/share/doc/parallel'
  /usr/bin/install -c -m 644 parallel.html env_parallel.html sem.html sql.html niceload.html parallel_tutorial.html parallel_design.html parallel_alternatives.html parcat.html parallel.texi env_parallel.texi sem.texi sql.texi niceload.texi parallel_tutorial.texi parallel_design.texi parallel_alternatives.texi parcat.texi parset.texi parallel.pdf env_parallel.pdf sem.pdf sql.pdf niceload.pdf parallel_tutorial.pdf parallel_design.pdf parallel_alternatives.pdf parcat.pdf parset.pdf '/usr/local/share/doc/parallel'
 pod2man --release='00000000' --center='parallel' \
@@ -363,6 +391,18 @@ pod2man --release='00000000' --center='parallel' \
 || echo "Warning: pod2man not found. Using old parallel_alternatives.7"
 /bin/bash: pod2man: command not found
 Warning: pod2man not found. Using old parallel_alternatives.7
+pod2man --release='00000000' --center='parallel' \
+	--section=1 ./parcat.pod > ./parcat.1n \
+&& mv ./parcat.1n ./parcat.1 \
+|| echo "Warning: pod2man not found. Using old parcat.1"
+/bin/bash: pod2man: command not found
+Warning: pod2man not found. Using old parcat.1
+pod2man --release='00000000' --center='parallel' \
+	--section=1 ./parset.pod > ./parset.1n \
+&& mv ./parset.1n ./parset.1 \
+|| echo "Warning: pod2man not found. Using old parset.1"
+/bin/bash: pod2man: command not found
+Warning: pod2man not found. Using old parset.1
  /bin/mkdir -p '/usr/local/share/man/man1'
  /usr/bin/install -c -m 644 parallel.1 env_parallel.1 sem.1 sql.1 niceload.1 parcat.1 parset.1 '/usr/local/share/man/man1'
  /bin/mkdir -p '/usr/local/share/man/man7'

testsuite/wanted-results/parallel-local-10s

@@ -932,6 +932,12 @@ par_sigterm	parallel: sleep 10; echo 6
 par_sigterm	parallel: sleep 10; echo 7
 par_sigterm	parallel: sleep 10; echo 8
 par_sigterm	parallel: sleep 10; echo 9
+par_slow_total_jobs	bug #51006: Slow total_jobs() eats job
+par_slow_total_jobs	parallel: Warning: Reading 2 arguments took longer than 10 seconds.
+par_slow_total_jobs	a
+par_slow_total_jobs	b
+par_slow_total_jobs	1
+par_slow_total_jobs	2
 par_tmux_fg	bug #50107: --tmux --fg should also write how to access it
 par_tmux_fg	See output with: tmux -S tmp attach
 par_tmux_fg	open terminal failed: not a terminal

testsuite/wanted-results/parallel-tutorial

@@ -918,6 +918,7 @@ DEBFULLNAME
 DEFAULTS_PATH
 DESKTOP_SESSION
 DISPLAY
+GOPATH
 GPG_AGENT_INFO
 GS_LIB
 GTK2_RC_FILES