mirror of
https://git.savannah.gnu.org/git/parallel.git
synced 2024-11-25 23:47:53 +00:00
-l 0 no longer works (and never will) because of --tollef.
exponential backoff in --semaphore mode. --regexp test. testsuite passes.
This commit is contained in:
Ole Tange
parent
69201240dc
commit
11b21d638c
|
|
@ -147,10 +147,25 @@ at: http://ftp.gnu.org/gnu/parallel/
|
|||
|
||||
New in this release:
|
||||
|
||||
* --tollef to be switch compatible with Tollef's parallel.
|
||||
* --tollef to be switch compatible with Tollef's parallel. This will
|
||||
cause -l to mean --load, and the argument separator will be --
|
||||
instead of :::
|
||||
|
||||
* --gnu will force GNU Parallel to behave like GNU Parallel even if
|
||||
--tollef is set.
|
||||
|
||||
* Site wide config file: /etc/parallel/config
|
||||
|
||||
By putting --tollef in the site wide config file you can deinstall
|
||||
Tollef's parallel and install GNU Parallel instead without any
|
||||
change for users or scripts. This is useful for packagers that do
|
||||
not distribute GNU Parallel because the command name conflicts with
|
||||
Tollef's parallel.
|
||||
|
||||
* -L 0 -n 0, and -N 0 implemented. They will read one argument,
|
||||
but insert 0 arguments on the command line. Useful if you just want
|
||||
to run the same command with the same arguments a number of times.
|
||||
|
||||
* Opscode Cookbook for Chef. Thanks to Joshua Timberman.
|
||||
https://github.com/opscode/cookbooks/tree/master/gnu_parallel
|
||||
|
||||
|
|
@ -166,8 +181,12 @@ New in this release:
|
|||
* A small example of grepping maillogs. Thanks to François Maillet.
|
||||
http://www.francoismaillet.com/blog/?p=399
|
||||
|
||||
* Using GNU Parallel instead of xargs. Thanks to James Cuff.
|
||||
http://blog.jcuff.net/2011/02/on-train-ride-in.html
|
||||
|
||||
* Bug fixes and man page updates.
|
||||
|
||||
|
||||
= About GNU Parallel =
|
||||
|
||||
GNU Parallel is a shell tool for executing jobs in parallel using one
|
||||
|
|
|
|||
41
src/parallel
41
src/parallel
|
|
@ -304,7 +304,7 @@ sub get_options_from_array {
|
|||
"basenamereplace|bnr=s" => \$::opt_basenamereplace,
|
||||
"basenameextensionreplace=s" => \$::opt_basenameextensionreplace,
|
||||
"jobs|j=s" => \$::opt_P,
|
||||
"load=s" => \$::opt_load,
|
||||
"load=f" => \$::opt_load,
|
||||
"max-line-length-allowed" => \$::opt_max_line_length_allowed,
|
||||
"number-of-cpus" => \$::opt_number_of_cpus,
|
||||
"number-of-cores" => \$::opt_number_of_cores,
|
||||
|
|
@ -322,7 +322,7 @@ sub get_options_from_array {
|
|||
"tmpdir=s" => \$::opt_tmpdir,
|
||||
"tempdir=s" => \$::opt_tmpdir,
|
||||
"tty|T" => \$::opt_tty,
|
||||
"halt-on-error|H=s" => \$::opt_halt_on_error,
|
||||
"halt-on-error|H=i" => \$::opt_halt_on_error,
|
||||
"retries=i" => \$::opt_retries,
|
||||
"dry-run|dryrun" => \$::opt_dryrun,
|
||||
"progress" => \$::opt_progress,
|
||||
|
|
@ -353,8 +353,8 @@ sub get_options_from_array {
|
|||
"max-replace-args|N=i" => \$::opt_N,
|
||||
"colsep|col-sep|C=s" => \$::opt_colsep,
|
||||
"help|h" => \$::opt_help,
|
||||
"L=i" => \$::opt_L,
|
||||
"max-lines|l:s" => \$::opt_l,
|
||||
"L=f" => \$::opt_L,
|
||||
"max-lines|l:f" => \$::opt_l,
|
||||
"interactive|p" => \$::opt_p,
|
||||
"verbose|t" => \$::opt_verbose,
|
||||
"version|V" => \$::opt_version,
|
||||
|
|
@ -381,7 +381,7 @@ sub get_options_from_array {
|
|||
sub parse_options {
|
||||
# Returns: N/A
|
||||
# Defaults:
|
||||
$Global::version = 20110218;
|
||||
$Global::version = 20110224;
|
||||
$Global::progname = 'parallel';
|
||||
$Global::infinity = 2**31;
|
||||
$Global::debug = 0;
|
||||
|
|
@ -484,14 +484,23 @@ sub parse_options {
|
|||
$Global::arg_sep = "--";
|
||||
}
|
||||
}
|
||||
for my $nlines ($::opt_L, $::opt_l) {
|
||||
if(defined $nlines) {
|
||||
if($nlines eq "") {
|
||||
$nlines = 1;
|
||||
if(defined $::opt_l) {
|
||||
if($::opt_l eq "-0") {
|
||||
# -l -0 (swallowed -0)
|
||||
$::opt_l = 1;
|
||||
$::opt_0 = 1;
|
||||
$/ = "\0";
|
||||
} elsif ($::opt_l == 0) {
|
||||
# If not given (or if 0 is given) => 1
|
||||
$::opt_l = 1;
|
||||
}
|
||||
$Global::max_lines = $nlines;
|
||||
$Global::max_lines = $::opt_l;
|
||||
$Global::max_number_of_args ||= $Global::max_lines;
|
||||
}
|
||||
|
||||
if(defined $::opt_L) {
|
||||
$Global::max_lines = $::opt_L;
|
||||
$Global::max_number_of_args ||= $Global::max_lines;
|
||||
}
|
||||
%Global::replace_rev = reverse %Global::replace;
|
||||
|
||||
|
|
@ -3953,19 +3962,22 @@ sub new {
|
|||
'id' => $id,
|
||||
'idfile' => $lockdir."/".$id,
|
||||
'pid' => $$,
|
||||
'pidfile' => $lockdir."/".$$,
|
||||
'pidfile' => $lockdir."/".$$.'@'.::hostname(),
|
||||
'count' => $count + 1 # nlinks returns a link for the 'id-' as well
|
||||
}, ref($class) || $class;
|
||||
}
|
||||
|
||||
sub acquire {
|
||||
my $self = shift;
|
||||
my $exponential_backoff = 1;
|
||||
while(1) {
|
||||
$self->atomic_link_if_count_less_than() and last;
|
||||
::debug("Remove dead locks");
|
||||
my $lockdir = $self->{'lockdir'};
|
||||
for my $d (<$lockdir/*>) {
|
||||
$d =~ m:$lockdir/([0-9]+)$:o or next;
|
||||
$d =~ m:$lockdir/([0-9]+)\@([-\._a-z0-9])$:o or next;
|
||||
my ($pid, $host) = ($1,$2);
|
||||
if($host eq ::hostname()) {
|
||||
if(not kill 0, $1) {
|
||||
::debug("Dead: $d");
|
||||
unlink $d;
|
||||
|
|
@ -3973,9 +3985,12 @@ sub acquire {
|
|||
::debug("Alive: $d");
|
||||
}
|
||||
}
|
||||
}
|
||||
# try again
|
||||
$self->atomic_link_if_count_less_than() and last;
|
||||
sleep 1;
|
||||
::usleep(rand()*$exponential_backoff);
|
||||
# Retry slower and slower up to 1 second
|
||||
$exponential_backoff = ($exponential_backoff < 1000) ? ($exponential_backoff * 1.1) : ($exponential_backoff);
|
||||
# TODO if timeout: last
|
||||
}
|
||||
::debug("acquired $self->{'pid'}\n");
|
||||
|
|
|
|||
|
|
@ -46,7 +46,8 @@ B<parallel> is capable of.
|
|||
|
||||
You can also watch the intro video for a quick introduction:
|
||||
http://www.youtube.com/watch?v=OpaiGYxkSuQ or at
|
||||
http://tinyogg.com/watch/TORaR/ and http://tinyogg.com/watch/hfxKj/
|
||||
http://tinyogg.com/watch/TORaR/ http://tinyogg.com/watch/hfxKj/ and
|
||||
http://tinyogg.com/watch/YQuXd/
|
||||
|
||||
=head1 OPTIONS
|
||||
|
||||
|
|
@ -82,7 +83,7 @@ B<{.}> can be used the same places as B<{}>. The replacement string
|
|||
B<{.}> can be changed with B<-U>.
|
||||
|
||||
|
||||
=item B<{/}> (beta testing)
|
||||
=item B<{/}>
|
||||
|
||||
Basename of input line. This is a specialized replacement string
|
||||
with the directory part removed.
|
||||
|
|
@ -91,7 +92,7 @@ B<{/}> can be used the same places as B<{}>. The replacement string
|
|||
B<{/}> can be changed with B<--basenamereplace>.
|
||||
|
||||
|
||||
=item B<{/.}> (beta testing)
|
||||
=item B<{/.}>
|
||||
|
||||
Basename of input line without extension. This is a specialized
|
||||
replacement string with the directory and extension part removed. It
|
||||
|
|
@ -117,7 +118,7 @@ extension. It is a combination of B<{>I<n>B<}> and B<{.}>.
|
|||
B<{>I<n>.B<}> can be used the same places as B<{>I<n>B<}>.
|
||||
|
||||
|
||||
=item B<{>I<n>/B<}> (beta testing)
|
||||
=item B<{>I<n>/B<}>
|
||||
|
||||
Basename of argument from argument file I<n> or the I<n>'th argument.
|
||||
It is a combination of B<{>I<n>B<}> and B<{/}>. See B<-a> and B<-N>.
|
||||
|
|
@ -125,7 +126,7 @@ It is a combination of B<{>I<n>B<}> and B<{/}>. See B<-a> and B<-N>.
|
|||
B<{>I<n>/B<}> can be used the same places as B<{>I<n>B<}>.
|
||||
|
||||
|
||||
=item B<{>I<n>/.B<}> (beta testing)
|
||||
=item B<{>I<n>/.B<}>
|
||||
|
||||
Basename of argument from argument file I<n> or the I<n>'th argument
|
||||
without extension. It is a combination of B<{>I<n>B<}>, B<{/}>, and
|
||||
|
|
@ -228,7 +229,7 @@ I<file> will be transferred the same way as B<--transfer>.
|
|||
Use the replacement string I<replace-str> instead of B<{/}> for basename of input line.
|
||||
|
||||
|
||||
=item B<--basenameextensionreplace> I<replace-str> (beta testing)
|
||||
=item B<--basenameextensionreplace> I<replace-str>
|
||||
|
||||
Use the replacement string I<replace-str> instead of B<{/.}> for basename of input line without extension.
|
||||
|
||||
|
|
@ -527,8 +528,7 @@ is optional. If I<max-lines> is not specified, it defaults to one.
|
|||
The B<-l> option is deprecated since the POSIX standard specifies
|
||||
B<-L> instead.
|
||||
|
||||
B<-l 0> means read one line, but insert 0 arguments on the command
|
||||
line.
|
||||
B<-l 0> is an alias for B<-l 1>.
|
||||
|
||||
Implies B<-X> unless B<-m> is set.
|
||||
|
||||
|
|
@ -675,7 +675,7 @@ Print the number of CPU cores and exit (used by GNU B<parallel> itself
|
|||
to determine the number of CPU cores on remote computers).
|
||||
|
||||
|
||||
=item B<--nice> I<niceness> (beta testing)
|
||||
=item B<--nice> I<niceness>
|
||||
|
||||
Run the command at this niceness. For simple commands you can just add
|
||||
B<nice> in front of the command. But if the command consists of more
|
||||
|
|
@ -750,11 +750,11 @@ Use B<--regexp> to interpret B<--recstart> and B<--recend> as regular
|
|||
expressions. This is slow, however.
|
||||
|
||||
|
||||
=item B<--remove-rec-sep> (alpha testing)
|
||||
=item B<--remove-rec-sep> (beta testing)
|
||||
|
||||
=item B<--removerecsep> (alpha testing)
|
||||
=item B<--removerecsep> (beta testing)
|
||||
|
||||
=item B<--rrs> (alpha testing)
|
||||
=item B<--rrs> (beta testing)
|
||||
|
||||
Remove the text matched by B<--recstart> and B<--recend> before piping
|
||||
it to the command.
|
||||
|
|
@ -1072,9 +1072,9 @@ Use B<-v> B<-v> to print the wrapping ssh command when running remotely.
|
|||
Print the version GNU B<parallel> and exit.
|
||||
|
||||
|
||||
=item B<--workdir> I<mydir> (beta testing)
|
||||
=item B<--workdir> I<mydir>
|
||||
|
||||
=item B<-W> I<mydir> (beta testing)
|
||||
=item B<-W> I<mydir>
|
||||
|
||||
Files transferred using B<--transfer> and B<--return> will be relative
|
||||
to I<mydir> on remote computers, and the command will be executed in
|
||||
|
|
|
|||
|
|
@ -64,10 +64,9 @@ egrep -v '^0$'
|
|||
echo '### Test --rrs -N1 --recend single'
|
||||
echo 12a34a45a6 |
|
||||
parallel -k --pipe --recend a -N1 --rrs 'echo -n "$PARALLEL_SEQ>"; cat; echo; sleep 0.1'
|
||||
# Broken
|
||||
#echo '### Test --rrs -N1 --recend alternate'
|
||||
#echo 12a34b45a6 |
|
||||
# parallel -k --pipe --recend 'a|b' -N1 --rrs 'echo -n "$PARALLEL_SEQ>"; cat; echo; sleep 0.1'
|
||||
echo '### Test --rrs -N1 --regexp --recend alternate'
|
||||
echo 12a34b45a6 |
|
||||
parallel -k --pipe --regexp --recend 'a|b' -N1 --rrs 'echo -n "$PARALLEL_SEQ>"; cat; echo; sleep 0.1'
|
||||
echo '### Test --rrs -N1 --recend single'
|
||||
echo 12a34b45a6 |
|
||||
parallel -k --pipe --recend 'b' -N1 --rrs 'echo -n "$PARALLEL_SEQ>"; cat; echo; sleep 0.1'
|
||||
|
|
@ -75,11 +74,10 @@ echo 12a34b45a6 |
|
|||
echo '### Test --rrs --recend single'
|
||||
echo 12a34a45a6 |
|
||||
parallel -k --pipe --recend a --rrs 'echo -n "$PARALLEL_SEQ>"; cat; echo; sleep 0.1'
|
||||
# Broken
|
||||
#echo '### Test --rrs -N1 --recend alternate'
|
||||
#echo 12a34b45a6 |
|
||||
# parallel -k --pipe --recend 'a|b' --rrs 'echo -n "$PARALLEL_SEQ>"; cat; echo; sleep 0.1'
|
||||
echo '### Test --rrs -N1 --recend single'
|
||||
echo '### Test --rrs --regexp --recend alternate'
|
||||
echo 12a34b45a6 |
|
||||
parallel -k --pipe --regexp --recend 'a|b' --rrs 'echo -n "$PARALLEL_SEQ>"; cat; echo; sleep 0.1'
|
||||
echo '### Test --rrs --recend single'
|
||||
echo 12a34b45a6 |
|
||||
parallel -k --pipe --recend 'b' --rrs 'echo -n "$PARALLEL_SEQ>"; cat; echo; sleep 0.1'
|
||||
|
||||
|
|
|
|||
|
|
@ -1,13 +1,14 @@
|
|||
#!/bin/bash
|
||||
|
||||
echo '### Test 0-arguments'
|
||||
seq 1 2 | parallel -n0 echo n0
|
||||
seq 1 2 | parallel -L0 echo L0
|
||||
seq 1 2 | parallel -l0 echo l0
|
||||
seq 1 2 | parallel -N0 echo N0
|
||||
seq 1 2 | parallel -k -n0 echo n0
|
||||
seq 1 2 | parallel -k -L0 echo L0
|
||||
seq 1 2 | parallel -k -N0 echo N0
|
||||
echo '### Because of --tollef -l, then -l0 == -l1, sorry'
|
||||
seq 1 2 | parallel -k -l0 echo l0
|
||||
|
||||
echo '### Test replace {}'
|
||||
seq 1 2 | parallel -N0 echo replace {} curlies
|
||||
seq 1 2 | parallel -k -N0 echo replace {} curlies
|
||||
|
||||
echo '### Test arguments on commandline'
|
||||
parallel -N0 echo args on cmdline ::: 1 2
|
||||
parallel -k -N0 echo args on cmdline ::: 1 2
|
||||
|
|
|
|||
28
testsuite/tests-to-run/test54.sh
Normal file
28
testsuite/tests-to-run/test54.sh
Normal file
|
|
@ -0,0 +1,28 @@
|
|||
#!/bin/bash
|
||||
|
||||
echo '### Test https://savannah.gnu.org/bugs/index.php?31716'
|
||||
seq 1 5 | stdout parallel -k -l echo {} OK
|
||||
seq 1 5 | stdout parallel -k -l 1 echo {} OK
|
||||
echo '### -k -l -0'
|
||||
printf '1\0002\0003\0004\0005\000' | stdout parallel -k -l -0 echo {} OK
|
||||
echo '### -k -0 -l'
|
||||
printf '1\0002\0003\0004\0005\000' | stdout parallel -k -0 -l echo {} OK
|
||||
echo '### -k -0 -l 1'
|
||||
printf '1\0002\0003\0004\0005\000' | stdout parallel -k -0 -l 1 echo {} OK
|
||||
echo '### -k -0 -l 0'
|
||||
printf '1\0002\0003\0004\0005\000' | stdout parallel -k -0 -l 0 echo {} OK
|
||||
echo '### -k -0 -L -0 - -0 is argument for -L'
|
||||
printf '1\0002\0003\0004\0005\000' | stdout parallel -k -0 -L -0 echo {} OK
|
||||
echo '### -k -0 -L 0 - -L always takes arg'
|
||||
printf '1\0002\0003\0004\0005\000' | stdout parallel -k -0 -L 0 echo {} OK
|
||||
echo '### -k -0 -L 0 - -L always takes arg'
|
||||
printf '1\0002\0003\0004\0005\000' | stdout parallel -k -L 0 -0 echo {} OK
|
||||
echo '### -k -e -0'
|
||||
printf '1\0002\0003\0004\0005\000' | stdout parallel -k -e -0 echo {} OK
|
||||
echo '### -k -0 -e eof'
|
||||
printf '1\0002\0003\0004\0005\000' | stdout parallel -k -0 -e eof echo {} OK
|
||||
echo '### -k -i -0'
|
||||
printf '1\0002\0003\0004\0005\000' | stdout parallel -k -i -0 echo {} OK
|
||||
echo '### -k -0 -i repl'
|
||||
printf '1\0002\0003\0004\0005\000' | stdout parallel -k -0 -i repl echo repl OK
|
||||
|
||||
|
|
@ -141,6 +141,12 @@ d
|
|||
3>45
|
||||
4>6
|
||||
|
||||
### Test --rrs -N1 --regexp --recend alternate
|
||||
1>12
|
||||
2>34
|
||||
3>45
|
||||
4>6
|
||||
|
||||
### Test --rrs -N1 --recend single
|
||||
1>12a34
|
||||
2>45a6
|
||||
|
|
@ -149,7 +155,11 @@ d
|
|||
1>123445
|
||||
2>6
|
||||
|
||||
### Test --rrs -N1 --recend single
|
||||
### Test --rrs --regexp --recend alternate
|
||||
1>123445
|
||||
2>6
|
||||
|
||||
### Test --rrs --recend single
|
||||
1>12a34
|
||||
2>45a6
|
||||
|
||||
|
|
|
|||
|
|
@ -3,10 +3,11 @@ n0
|
|||
n0
|
||||
L0
|
||||
L0
|
||||
l0
|
||||
l0
|
||||
N0
|
||||
N0
|
||||
### Because of --tollef -l, then -l0 == -l1, sorry
|
||||
l0 1
|
||||
l0 2
|
||||
### Test replace {}
|
||||
replace curlies
|
||||
replace curlies
|
||||
|
|
|
|||
77
testsuite/wanted-results/test54
Normal file
77
testsuite/wanted-results/test54
Normal file
|
|
@ -0,0 +1,77 @@
|
|||
### Test https://savannah.gnu.org/bugs/index.php?31716
|
||||
1 OK
|
||||
2 OK
|
||||
3 OK
|
||||
4 OK
|
||||
5 OK
|
||||
1 OK
|
||||
2 OK
|
||||
3 OK
|
||||
4 OK
|
||||
5 OK
|
||||
### -k -l -0
|
||||
1 OK
|
||||
2 OK
|
||||
3 OK
|
||||
4 OK
|
||||
5 OK
|
||||
### -k -0 -l
|
||||
1 OK
|
||||
2 OK
|
||||
3 OK
|
||||
4 OK
|
||||
5 OK
|
||||
### -k -0 -l 1
|
||||
1 OK
|
||||
2 OK
|
||||
3 OK
|
||||
4 OK
|
||||
5 OK
|
||||
### -k -0 -l 0
|
||||
1 OK
|
||||
2 OK
|
||||
3 OK
|
||||
4 OK
|
||||
5 OK
|
||||
### -k -0 -L -0 - -0 is argument for -L
|
||||
OK
|
||||
OK
|
||||
OK
|
||||
OK
|
||||
OK
|
||||
### -k -0 -L 0 - -L always takes arg
|
||||
OK
|
||||
OK
|
||||
OK
|
||||
OK
|
||||
OK
|
||||
### -k -0 -L 0 - -L always takes arg
|
||||
OK
|
||||
OK
|
||||
OK
|
||||
OK
|
||||
OK
|
||||
### -k -e -0
|
||||
1 OK
|
||||
2 OK
|
||||
3 OK
|
||||
4 OK
|
||||
5 OK
|
||||
### -k -0 -e eof
|
||||
1 OK
|
||||
2 OK
|
||||
3 OK
|
||||
4 OK
|
||||
5 OK
|
||||
### -k -i -0
|
||||
1 OK
|
||||
2 OK
|
||||
3 OK
|
||||
4 OK
|
||||
5 OK
|
||||
### -k -0 -i repl
|
||||
1 OK
|
||||
2 OK
|
||||
3 OK
|
||||
4 OK
|
||||
5 OK
|
||||
Loading…
Reference in a new issue