tange/parallel
1
0
Fork 0
mirror of https://git.savannah.gnu.org/git/parallel.git synced 2024-11-22 14:07:55 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fixed bug #51006: Slow total_jobs() eats job

Browse source
This commit is contained in:
Ole Tange 2017-05-11 22:57:46 +02:00
parent a7a9e522dd
commit 61525c90e1
11 changed files with 1371 additions and 1197 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
doc/release_new_version
View file

@ -196,9 +196,9 @@ file:///home/tange/privat/parallel/doc/release_new_version
from:tange@gnu.org
to:parallel@gnu.org, bug-parallel@gnu.org
Subject: GNU Parallel 20170522 ('') released <<[stable]>>
Subject: GNU Parallel 20170522 ('Macron') released <<[stable]>>
GNU Parallel 20170522 ('') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/
GNU Parallel 20170522 ('Macron') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/
<<No new functionality was introduced so this is a good candidate for a stable release.>>
@ -214,10 +214,14 @@ New in this release:
http://meta.askubuntu.com/a/16750/22307
http://meta.serverfault.com/a/9040/45704
https://github.com/jupiter126/Create_Speech_Dataset
http://pubs.rsc.org/-/content/articlelanding/2017/cp/c7cp01662j#!divAbstract
ftp://frapftp.fire.ca.gov/outgoing/transfer/PUC_Map1/Utility%20Fire%20map%201%20final%20review%20and%20development%20report21616.pdf
https://dzone.com/articles/batch-convert-images-from-png-to-jpeg
https://arxiv.org/pdf/1704.02257.pdf
http://www.sciencedirect.com/science/article/pii/S0044848616312194
@ -232,9 +236,15 @@ http://biorxiv.org/content/biorxiv/early/2017/04/27/131680.full.pdf
http://www.nature.com/nprot/journal/v12/n5/full/nprot.2017.022.html?WT.feed_name=subjects_biotechnology
https://128.84.21.199/pdf/1704.08997.pdf
<<Citation not ok: https://arxiv.org/pdf/1704.08997.pdf>>
* <<Citation needed: https://sdstrowes.co.uk/publications/sdstrowes-rdns-aaaa.pdf>>
* <<Citation not ok: https://www.nature.com/articles/s41598-017-01703-6>>
* <<Possibly http://link.springer.com/chapter/10.1007%2F978-3-319-22053-6_46>>
* <<Possibly http://link.springer.com/article/10.1007/s12021-015-9290-5>>

12
src/Makefile.am
View file

@ -76,13 +76,13 @@ niceload.1: niceload.pod
parcat.1: parcat.pod
pod2man --release='$(PACKAGE_VERSION)' --center='$(PACKAGE_NAME)' \
--section=1 $(srcdir)/parcat > $(srcdir)/parcat.1n \
--section=1 $(srcdir)/parcat.pod > $(srcdir)/parcat.1n \
&& mv $(srcdir)/parcat.1n $(srcdir)/parcat.1 \
|| echo "Warning: pod2man not found. Using old parcat.1"
parset.1: parset.pod
pod2man --release='$(PACKAGE_VERSION)' --center='$(PACKAGE_NAME)' \
--section=1 $(srcdir)/parset > $(srcdir)/parset.1n \
--section=1 $(srcdir)/parset.pod > $(srcdir)/parset.1n \
&& mv $(srcdir)/parset.1n $(srcdir)/parset.1 \
|| echo "Warning: pod2man not found. Using old parset.1"
@ -149,8 +149,8 @@ parcat.html: parcat niceload.html
rm -f $(srcdir)/pod2htm*
# Depending on niceload.html to avoid stupid pod2html race condition
parset.html: parset parcat.html
pod2html --title "GNU parset" $(srcdir)/parset > $(srcdir)/parset.htmln \
parset.html: parset.pod parcat.html
pod2html --title "GNU parset" $(srcdir)/parset.pod > $(srcdir)/parset.htmln \
&& mv $(srcdir)/parset.htmln $(srcdir)/parset.html \
|| echo "Warning: pod2html not found. Using old parset.html"
rm -f $(srcdir)/pod2htm*
@ -192,7 +192,7 @@ parcat.texi: parcat
|| echo "Warning: pod2texi not found. Using old parcat.texi"
parset.texi: parset.pod
pod2texi --output=$(srcdir)/parset.texi $(srcdir)/parset \
pod2texi --output=$(srcdir)/parset.texi $(srcdir)/parset.pod \
|| echo "Warning: pod2texi not found. Using old parset.texi"
parallel.pdf: parallel.pod
@ -232,7 +232,7 @@ parcat.pdf: parcat
|| echo "Warning: pod2pdf not found. Using old parcat.pdf"
parset.pdf: parset.pod
pod2pdf --output-file $(srcdir)/parset.pdf $(srcdir)/parset --title "GNU parset" \
pod2pdf --output-file $(srcdir)/parset.pdf $(srcdir)/parset.pod --title "GNU parset" \
|| echo "Warning: pod2pdf not found. Using old parset.pdf"
sem: parallel

12
src/Makefile.in
View file

@ -664,13 +664,13 @@ niceload.1: niceload.pod
parcat.1: parcat.pod
pod2man --release='$(PACKAGE_VERSION)' --center='$(PACKAGE_NAME)' \
--section=1 $(srcdir)/parcat > $(srcdir)/parcat.1n \
--section=1 $(srcdir)/parcat.pod > $(srcdir)/parcat.1n \
&& mv $(srcdir)/parcat.1n $(srcdir)/parcat.1 \
|| echo "Warning: pod2man not found. Using old parcat.1"
parset.1: parset.pod
pod2man --release='$(PACKAGE_VERSION)' --center='$(PACKAGE_NAME)' \
--section=1 $(srcdir)/parset > $(srcdir)/parset.1n \
--section=1 $(srcdir)/parset.pod > $(srcdir)/parset.1n \
&& mv $(srcdir)/parset.1n $(srcdir)/parset.1 \
|| echo "Warning: pod2man not found. Using old parset.1"
@ -737,8 +737,8 @@ parcat.html: parcat niceload.html
rm -f $(srcdir)/pod2htm*
# Depending on niceload.html to avoid stupid pod2html race condition
parset.html: parset parcat.html
pod2html --title "GNU parset" $(srcdir)/parset > $(srcdir)/parset.htmln \
parset.html: parset.pod parcat.html
pod2html --title "GNU parset" $(srcdir)/parset.pod > $(srcdir)/parset.htmln \
&& mv $(srcdir)/parset.htmln $(srcdir)/parset.html \
|| echo "Warning: pod2html not found. Using old parset.html"
rm -f $(srcdir)/pod2htm*
@ -780,7 +780,7 @@ parcat.texi: parcat
|| echo "Warning: pod2texi not found. Using old parcat.texi"
parset.texi: parset.pod
pod2texi --output=$(srcdir)/parset.texi $(srcdir)/parset \
pod2texi --output=$(srcdir)/parset.texi $(srcdir)/parset.pod \
|| echo "Warning: pod2texi not found. Using old parset.texi"
parallel.pdf: parallel.pod
@ -820,7 +820,7 @@ parcat.pdf: parcat
|| echo "Warning: pod2pdf not found. Using old parcat.pdf"
parset.pdf: parset.pod
pod2pdf --output-file $(srcdir)/parset.pdf $(srcdir)/parset --title "GNU parset" \
pod2pdf --output-file $(srcdir)/parset.pdf $(srcdir)/parset.pod --title "GNU parset" \
|| echo "Warning: pod2pdf not found. Using old parset.pdf"
sem: parallel

2
src/parallel
View file

@ -6525,6 +6525,7 @@ sub total_jobs {
my $record_queue = $self->{'commandlinequeue'}{'arg_queue'};
my $start = time;
while($record = $record_queue->get()) {
push @arg_records, $record;
if(time - $start > 10) {
::warning("Reading ".scalar(@arg_records).
" arguments took longer than 10 seconds.");
@ -6533,7 +6534,6 @@ sub total_jobs {
$opt::shuf && ::warning("Consider removing --shuf.");
last;
}
push @arg_records, $record;
}
while($record = $record_queue->get()) {
push @arg_records, $record;

117
src/parallel_alternatives.pod
View file

@ -859,10 +859,14 @@ report this, but finishes with success - thereby risking data loss.
=head2 DIFFERENCES BETWEEN Rush AND GNU Parallel
Rush (https://github.com/shenwei356/rush) is written in Go and based
on gargs.
B<rush> (https://github.com/shenwei356/rush) is written in Go and
based on B<gargs>.
Rush has some string manipulations that can be emulated by putting
Just like GNU B<parallel> B<rush> buffer in temporary files. But
opposite GNU B<parallel> B<rush> does not clean up, if the process
dies abnormally.
B<rush> has some string manipulations that can be emulated by putting
this into ~/.parallel/config (% is used instead of ^):
--rpl '{:} s:(\.[^/]+)*$::'
@ -929,6 +933,113 @@ B<12> Followed by:
B<12> seq 1 3 | parallel --joblog mylog --retry-failed 'sleep {}; echo {}'
B<rush> has:
=over 4
=item * B<awk -v> like custom defined variables (B<-v>)
With GNU B<parallel> you would simply simply set a shell variable:
parallel 'v={}; echo "$v"' ::: foo
echo foo | rush -v v={} 'echo {v}'
Also B<rush> does not like special chars. So these do not work:
echo does not work | rush -v v=\" 'echo {v}'
echo "My brother's 12\" records" | rush -v v={} 'echo {v}'
Whereas the corresponding GNU B<parallel> version works:
parallel 'v=\"; echo "$v"' ::: works
parallel 'v={}; echo "$v"' ::: "My brother's 12\" records"
=item * Exit on first error(s) (-e)
This is called B<--halt now,fail=1> (or shorter: B<--halt 2>) when
used with GNU B<parallel>.
=item * Settable records sending to every command (B<-n>, default 1)
This is also called B<-n> in GNU B<parallel>.
=item * Practical replacement strings
=over 4
=item {:} remove any extension
With GNU B<parallel> this can be emulated by:
parallel --plus echo '{/\..*/}' ::: foo.ext.bar.gz
=item {^suffix}, remove suffix
With GNU B<parallel> this can be emulated by:
parallel --plus echo '{%.bar.gz}' ::: foo.ext.bar.gz
=item {%.}, {%:}, basename without extension
With GNU B<parallel> this can be emulated by:
parallel echo '{= s:.*/::;s/\..*// =}' ::: dir/foo.bar.gz
And if you need it often, you define a B<--rpl> in
B<$HOME/.parallel/config>:
--rpl '{%.} s:.*/::;s/\..*//'
--rpl '{%:} s:.*/::;s/\..*//'
Then you can use them as:
parallel echo {%.} {%:} ::: dir/foo.bar.gz
=back
=item * Preset variable (macro)
E.g.
echo foosuffix | rush -v p={^suffix} 'echo {p}_new_suffix'
With GNU B<parallel> this can be emulated by:
echo foosuffix | parallel --plus 'p={%suffix}; echo ${p}_new_suffix'
Opposite B<rush> GNU B<parallel> works fine if the input contains
double space, ' and ":
echo "1'6\" foosuffix" |
parallel --plus 'p={%suffix}; echo "${p}"_new_suffix'
=item * Commands of multi-lines
To improve readibilty GNU B<parallel> encourages not to use multi-line
commands. In most cases it can be written as a function:
seq 1 3 | parallel --timeout 2 --joblog my.log 'sleep {}; echo {}; \
echo finish {}'
Could be written as:
doit() {
sleep "$1"
echo "$1"
echo finish "$1"
}
export -f doit
seq 1 3 | parallel --timeout 2 --joblog my.log doit
The failed commands can be resumed with:
seq 1 3 |
parallel --resume-failed --joblog my.log 'sleep {}; echo {};\
echo finish {}'
=back
=head2 DIFFERENCES BETWEEN machma AND GNU Parallel

6
testsuite/tests-to-run/parallel-local-10s.sh
View file

@ -314,6 +314,12 @@ par_plus_dyn_repl() {
parallel --plus echo '{-2,,A}' ::: "wrong" ::: "$myvar" ::: "wrong"
}
par_slow_total_jobs() {
echo 'bug #51006: Slow total_jobs() eats job'
(echo a; sleep 6; echo b; sleep 6; seq 2) |
parallel echo '{=total_jobs()=}'
}
export -f $(compgen -A function | grep par_)
compgen -A function | grep par_ | sort |
parallel --joblog /tmp/jl-`basename $0` -j10 --tag -k '{} 2>&1'

2
testsuite/tests-to-run/parallel-tutorial.sh
View file

@ -45,7 +45,7 @@ perl -ne '$/="\n\n"; /^Output/../^[^O]\S/ and next; /^ / and print;' ../../src/
# Remote script
s/(PARALLEL_PID\D+)\d+/${1}000000/g;
# sql timing
s/,:,\d+.\d+,\d+.\d+/,:,000000000.000,0.000/g;
s/,[a-z]*,\d+.\d+,\d+.\d+/,:,000000000.000,0.000/g;
# /usr/bin/time -f %e
s/^(\d+)\.\d+$/$1/;
# Base 64 string

40
testsuite/wanted-results/parallel-install
View file

@ -81,6 +81,18 @@ pod2man --release='00000000' --center='parallel' \
|| echo "Warning: pod2man not found. Using old parallel_alternatives.7"
/bin/bash: pod2man: command not found
Warning: pod2man not found. Using old parallel_alternatives.7
pod2man --release='00000000' --center='parallel' \
--section=1 ./parcat.pod > ./parcat.1n \
&& mv ./parcat.1n ./parcat.1 \
|| echo "Warning: pod2man not found. Using old parcat.1"
/bin/bash: pod2man: command not found
Warning: pod2man not found. Using old parcat.1
pod2man --release='00000000' --center='parallel' \
--section=1 ./parset.pod > ./parset.1n \
&& mv ./parset.1n ./parset.1 \
|| echo "Warning: pod2man not found. Using old parset.1"
/bin/bash: pod2man: command not found
Warning: pod2man not found. Using old parset.1
pod2html --title "GNU Parallel" ./parallel.pod > ./parallel.htmln \
&& mv ./parallel.htmln ./parallel.html \
|| echo "Warning: pod2html not found. Using old parallel.html"
@ -161,6 +173,10 @@ pod2texi --output=./parallel_alternatives.texi ./parallel_alternatives.pod \
|| echo "Warning: pod2texi not found. Using old parallel_alternatives.texi"
/bin/bash: pod2texi: command not found
Warning: pod2texi not found. Using old parallel_alternatives.texi
pod2texi --output=./parset.texi ./parset.pod \
|| echo "Warning: pod2texi not found. Using old parset.texi"
/bin/bash: pod2texi: command not found
Warning: pod2texi not found. Using old parset.texi
pod2pdf --output-file ./parallel.pdf ./parallel.pod --title "GNU Parallel" \
|| echo "Warning: pod2pdf not found. Using old parallel.pdf"
/bin/bash: pod2pdf: command not found
@ -193,6 +209,10 @@ pod2pdf --output-file ./parallel_alternatives.pdf ./parallel_alternatives.pod --
|| echo "Warning: pod2pdf not found. Using old parallel_alternatives.pdf"
/bin/bash: pod2pdf: command not found
Warning: pod2pdf not found. Using old parallel_alternatives.pdf
pod2pdf --output-file ./parset.pdf ./parset.pod --title "GNU parset" \
|| echo "Warning: pod2pdf not found. Using old parset.pdf"
/bin/bash: pod2pdf: command not found
Warning: pod2pdf not found. Using old parset.pdf
make[0]: Entering directory '/tmp/parallel-00000000/src'
/bin/mkdir -p '/usr/local/bin'
/usr/bin/install -c parallel sql niceload parcat env_parallel env_parallel.ash env_parallel.bash env_parallel.csh env_parallel.dash env_parallel.fish env_parallel.ksh env_parallel.pdksh env_parallel.sh env_parallel.tcsh env_parallel.zsh '/usr/local/bin'
@ -281,6 +301,10 @@ pod2texi --output=./parallel_alternatives.texi ./parallel_alternatives.pod \
|| echo "Warning: pod2texi not found. Using old parallel_alternatives.texi"
/bin/bash: pod2texi: command not found
Warning: pod2texi not found. Using old parallel_alternatives.texi
pod2texi --output=./parset.texi ./parset.pod \
|| echo "Warning: pod2texi not found. Using old parset.texi"
/bin/bash: pod2texi: command not found
Warning: pod2texi not found. Using old parset.texi
pod2pdf --output-file ./parallel.pdf ./parallel.pod --title "GNU Parallel" \
|| echo "Warning: pod2pdf not found. Using old parallel.pdf"
/bin/bash: pod2pdf: command not found
@ -313,6 +337,10 @@ pod2pdf --output-file ./parallel_alternatives.pdf ./parallel_alternatives.pod --
|| echo "Warning: pod2pdf not found. Using old parallel_alternatives.pdf"
/bin/bash: pod2pdf: command not found
Warning: pod2pdf not found. Using old parallel_alternatives.pdf
pod2pdf --output-file ./parset.pdf ./parset.pod --title "GNU parset" \
|| echo "Warning: pod2pdf not found. Using old parset.pdf"
/bin/bash: pod2pdf: command not found
Warning: pod2pdf not found. Using old parset.pdf
/bin/mkdir -p '/usr/local/share/doc/parallel'
/usr/bin/install -c -m 644 parallel.html env_parallel.html sem.html sql.html niceload.html parallel_tutorial.html parallel_design.html parallel_alternatives.html parcat.html parallel.texi env_parallel.texi sem.texi sql.texi niceload.texi parallel_tutorial.texi parallel_design.texi parallel_alternatives.texi parcat.texi parset.texi parallel.pdf env_parallel.pdf sem.pdf sql.pdf niceload.pdf parallel_tutorial.pdf parallel_design.pdf parallel_alternatives.pdf parcat.pdf parset.pdf '/usr/local/share/doc/parallel'
pod2man --release='00000000' --center='parallel' \
@ -363,6 +391,18 @@ pod2man --release='00000000' --center='parallel' \
|| echo "Warning: pod2man not found. Using old parallel_alternatives.7"
/bin/bash: pod2man: command not found
Warning: pod2man not found. Using old parallel_alternatives.7
pod2man --release='00000000' --center='parallel' \
--section=1 ./parcat.pod > ./parcat.1n \
&& mv ./parcat.1n ./parcat.1 \
|| echo "Warning: pod2man not found. Using old parcat.1"
/bin/bash: pod2man: command not found
Warning: pod2man not found. Using old parcat.1
pod2man --release='00000000' --center='parallel' \
--section=1 ./parset.pod > ./parset.1n \
&& mv ./parset.1n ./parset.1 \
|| echo "Warning: pod2man not found. Using old parset.1"
/bin/bash: pod2man: command not found
Warning: pod2man not found. Using old parset.1
/bin/mkdir -p '/usr/local/share/man/man1'
/usr/bin/install -c -m 644 parallel.1 env_parallel.1 sem.1 sql.1 niceload.1 parcat.1 parset.1 '/usr/local/share/man/man1'
/bin/mkdir -p '/usr/local/share/man/man7'

6
testsuite/wanted-results/parallel-local-10s
View file

@ -932,6 +932,12 @@ par_sigterm parallel: sleep 10; echo 6
par_sigterm parallel: sleep 10; echo 7
par_sigterm parallel: sleep 10; echo 8
par_sigterm parallel: sleep 10; echo 9
par_slow_total_jobs bug #51006: Slow total_jobs() eats job
par_slow_total_jobs parallel: Warning: Reading 2 arguments took longer than 10 seconds.
par_slow_total_jobs a
par_slow_total_jobs b
par_slow_total_jobs 1
par_slow_total_jobs 2
par_tmux_fg bug #50107: --tmux --fg should also write how to access it
par_tmux_fg See output with: tmux -S tmp attach
par_tmux_fg open terminal failed: not a terminal

2354
testsuite/wanted-results/parallel-local-sql
View file

File diff suppressed because it is too large Load diff

1
testsuite/wanted-results/parallel-tutorial
View file

@ -918,6 +918,7 @@ DEBFULLNAME
DEFAULTS_PATH
DESKTOP_SESSION
DISPLAY
GOPATH
GPG_AGENT_INFO
GS_LIB
GTK2_RC_FILES