-l 0 no longer works (and never will) because of --tollef.

exponential backoff in --semaphore mode.
--regexp test.
testsuite passes.
This commit is contained in:
Ole Tange 2011-03-01 23:04:15 +01:00
parent 69201240dc
commit 11b21d638c
9 changed files with 202 additions and 53 deletions

View file

@ -147,10 +147,25 @@ at: http://ftp.gnu.org/gnu/parallel/
New in this release:
* --tollef to be switch compatible with Tollef's parallel.
* --tollef to be switch compatible with Tollef's parallel. This will
cause -l to mean --load, and the argument separator will be --
instead of :::
* --gnu will force GNU Parallel to behave like GNU Parallel even if
--tollef is set.
* Site wide config file: /etc/parallel/config
By putting --tollef in the site wide config file you can deinstall
Tollef's parallel and install GNU Parallel instead without any
change for users or scripts. This is useful for packagers that do
not distribute GNU Parallel because the command name conflicts with
Tollef's parallel.
* -L 0 -n 0, and -N 0 implemented. They will read one argument,
but insert 0 arguments on the command line. Useful if you just want
to run the same command with the same arguments a number of times.
* Opscode Cookbook for Chef. Thanks to Joshua Timberman.
https://github.com/opscode/cookbooks/tree/master/gnu_parallel
@ -166,8 +181,12 @@ New in this release:
* A small example of grepping maillogs. Thanks to François Maillet.
http://www.francoismaillet.com/blog/?p=399
* Using GNU Parallel instead of xargs. Thanks to James Cuff.
http://blog.jcuff.net/2011/02/on-train-ride-in.html
* Bug fixes and man page updates.
= About GNU Parallel =
GNU Parallel is a shell tool for executing jobs in parallel using one

View file

@ -304,7 +304,7 @@ sub get_options_from_array {
"basenamereplace|bnr=s" => \$::opt_basenamereplace,
"basenameextensionreplace=s" => \$::opt_basenameextensionreplace,
"jobs|j=s" => \$::opt_P,
"load=s" => \$::opt_load,
"load=f" => \$::opt_load,
"max-line-length-allowed" => \$::opt_max_line_length_allowed,
"number-of-cpus" => \$::opt_number_of_cpus,
"number-of-cores" => \$::opt_number_of_cores,
@ -322,7 +322,7 @@ sub get_options_from_array {
"tmpdir=s" => \$::opt_tmpdir,
"tempdir=s" => \$::opt_tmpdir,
"tty|T" => \$::opt_tty,
"halt-on-error|H=s" => \$::opt_halt_on_error,
"halt-on-error|H=i" => \$::opt_halt_on_error,
"retries=i" => \$::opt_retries,
"dry-run|dryrun" => \$::opt_dryrun,
"progress" => \$::opt_progress,
@ -353,8 +353,8 @@ sub get_options_from_array {
"max-replace-args|N=i" => \$::opt_N,
"colsep|col-sep|C=s" => \$::opt_colsep,
"help|h" => \$::opt_help,
"L=i" => \$::opt_L,
"max-lines|l:s" => \$::opt_l,
"L=f" => \$::opt_L,
"max-lines|l:f" => \$::opt_l,
"interactive|p" => \$::opt_p,
"verbose|t" => \$::opt_verbose,
"version|V" => \$::opt_version,
@ -381,7 +381,7 @@ sub get_options_from_array {
sub parse_options {
# Returns: N/A
# Defaults:
$Global::version = 20110218;
$Global::version = 20110224;
$Global::progname = 'parallel';
$Global::infinity = 2**31;
$Global::debug = 0;
@ -484,14 +484,23 @@ sub parse_options {
$Global::arg_sep = "--";
}
}
for my $nlines ($::opt_L, $::opt_l) {
if(defined $nlines) {
if($nlines eq "") {
$nlines = 1;
}
$Global::max_lines = $nlines;
$Global::max_number_of_args ||= $Global::max_lines;
if(defined $::opt_l) {
if($::opt_l eq "-0") {
# -l -0 (swallowed -0)
$::opt_l = 1;
$::opt_0 = 1;
$/ = "\0";
} elsif ($::opt_l == 0) {
# If not given (or if 0 is given) => 1
$::opt_l = 1;
}
$Global::max_lines = $::opt_l;
$Global::max_number_of_args ||= $Global::max_lines;
}
if(defined $::opt_L) {
$Global::max_lines = $::opt_L;
$Global::max_number_of_args ||= $Global::max_lines;
}
%Global::replace_rev = reverse %Global::replace;
@ -3953,29 +3962,35 @@ sub new {
'id' => $id,
'idfile' => $lockdir."/".$id,
'pid' => $$,
'pidfile' => $lockdir."/".$$,
'pidfile' => $lockdir."/".$$.'@'.::hostname(),
'count' => $count + 1 # nlinks returns a link for the 'id-' as well
}, ref($class) || $class;
}
sub acquire {
my $self = shift;
my $exponential_backoff = 1;
while(1) {
$self->atomic_link_if_count_less_than() and last;
::debug("Remove dead locks");
my $lockdir = $self->{'lockdir'};
for my $d (<$lockdir/*>) {
$d =~ m:$lockdir/([0-9]+)$:o or next;
if(not kill 0, $1) {
::debug("Dead: $d");
unlink $d;
} else {
::debug("Alive: $d");
$d =~ m:$lockdir/([0-9]+)\@([-\._a-z0-9])$:o or next;
my ($pid, $host) = ($1,$2);
if($host eq ::hostname()) {
if(not kill 0, $1) {
::debug("Dead: $d");
unlink $d;
} else {
::debug("Alive: $d");
}
}
}
# try again
$self->atomic_link_if_count_less_than() and last;
sleep 1;
::usleep(rand()*$exponential_backoff);
# Retry slower and slower up to 1 second
$exponential_backoff = ($exponential_backoff < 1000) ? ($exponential_backoff * 1.1) : ($exponential_backoff);
# TODO if timeout: last
}
::debug("acquired $self->{'pid'}\n");

View file

@ -46,7 +46,8 @@ B<parallel> is capable of.
You can also watch the intro video for a quick introduction:
http://www.youtube.com/watch?v=OpaiGYxkSuQ or at
http://tinyogg.com/watch/TORaR/ and http://tinyogg.com/watch/hfxKj/
http://tinyogg.com/watch/TORaR/ http://tinyogg.com/watch/hfxKj/ and
http://tinyogg.com/watch/YQuXd/
=head1 OPTIONS
@ -82,7 +83,7 @@ B<{.}> can be used the same places as B<{}>. The replacement string
B<{.}> can be changed with B<-U>.
=item B<{/}> (beta testing)
=item B<{/}>
Basename of input line. This is a specialized replacement string
with the directory part removed.
@ -91,7 +92,7 @@ B<{/}> can be used the same places as B<{}>. The replacement string
B<{/}> can be changed with B<--basenamereplace>.
=item B<{/.}> (beta testing)
=item B<{/.}>
Basename of input line without extension. This is a specialized
replacement string with the directory and extension part removed. It
@ -117,7 +118,7 @@ extension. It is a combination of B<{>I<n>B<}> and B<{.}>.
B<{>I<n>.B<}> can be used the same places as B<{>I<n>B<}>.
=item B<{>I<n>/B<}> (beta testing)
=item B<{>I<n>/B<}>
Basename of argument from argument file I<n> or the I<n>'th argument.
It is a combination of B<{>I<n>B<}> and B<{/}>. See B<-a> and B<-N>.
@ -125,7 +126,7 @@ It is a combination of B<{>I<n>B<}> and B<{/}>. See B<-a> and B<-N>.
B<{>I<n>/B<}> can be used the same places as B<{>I<n>B<}>.
=item B<{>I<n>/.B<}> (beta testing)
=item B<{>I<n>/.B<}>
Basename of argument from argument file I<n> or the I<n>'th argument
without extension. It is a combination of B<{>I<n>B<}>, B<{/}>, and
@ -228,7 +229,7 @@ I<file> will be transferred the same way as B<--transfer>.
Use the replacement string I<replace-str> instead of B<{/}> for basename of input line.
=item B<--basenameextensionreplace> I<replace-str> (beta testing)
=item B<--basenameextensionreplace> I<replace-str>
Use the replacement string I<replace-str> instead of B<{/.}> for basename of input line without extension.
@ -527,8 +528,7 @@ is optional. If I<max-lines> is not specified, it defaults to one.
The B<-l> option is deprecated since the POSIX standard specifies
B<-L> instead.
B<-l 0> means read one line, but insert 0 arguments on the command
line.
B<-l 0> is an alias for B<-l 1>.
Implies B<-X> unless B<-m> is set.
@ -675,7 +675,7 @@ Print the number of CPU cores and exit (used by GNU B<parallel> itself
to determine the number of CPU cores on remote computers).
=item B<--nice> I<niceness> (beta testing)
=item B<--nice> I<niceness>
Run the command at this niceness. For simple commands you can just add
B<nice> in front of the command. But if the command consists of more
@ -750,11 +750,11 @@ Use B<--regexp> to interpret B<--recstart> and B<--recend> as regular
expressions. This is slow, however.
=item B<--remove-rec-sep> (alpha testing)
=item B<--remove-rec-sep> (beta testing)
=item B<--removerecsep> (alpha testing)
=item B<--removerecsep> (beta testing)
=item B<--rrs> (alpha testing)
=item B<--rrs> (beta testing)
Remove the text matched by B<--recstart> and B<--recend> before piping
it to the command.
@ -1072,9 +1072,9 @@ Use B<-v> B<-v> to print the wrapping ssh command when running remotely.
Print the version GNU B<parallel> and exit.
=item B<--workdir> I<mydir> (beta testing)
=item B<--workdir> I<mydir>
=item B<-W> I<mydir> (beta testing)
=item B<-W> I<mydir>
Files transferred using B<--transfer> and B<--return> will be relative
to I<mydir> on remote computers, and the command will be executed in

View file

@ -64,10 +64,9 @@ egrep -v '^0$'
echo '### Test --rrs -N1 --recend single'
echo 12a34a45a6 |
parallel -k --pipe --recend a -N1 --rrs 'echo -n "$PARALLEL_SEQ>"; cat; echo; sleep 0.1'
# Broken
#echo '### Test --rrs -N1 --recend alternate'
#echo 12a34b45a6 |
# parallel -k --pipe --recend 'a|b' -N1 --rrs 'echo -n "$PARALLEL_SEQ>"; cat; echo; sleep 0.1'
echo '### Test --rrs -N1 --regexp --recend alternate'
echo 12a34b45a6 |
parallel -k --pipe --regexp --recend 'a|b' -N1 --rrs 'echo -n "$PARALLEL_SEQ>"; cat; echo; sleep 0.1'
echo '### Test --rrs -N1 --recend single'
echo 12a34b45a6 |
parallel -k --pipe --recend 'b' -N1 --rrs 'echo -n "$PARALLEL_SEQ>"; cat; echo; sleep 0.1'
@ -75,11 +74,10 @@ echo 12a34b45a6 |
echo '### Test --rrs --recend single'
echo 12a34a45a6 |
parallel -k --pipe --recend a --rrs 'echo -n "$PARALLEL_SEQ>"; cat; echo; sleep 0.1'
# Broken
#echo '### Test --rrs -N1 --recend alternate'
#echo 12a34b45a6 |
# parallel -k --pipe --recend 'a|b' --rrs 'echo -n "$PARALLEL_SEQ>"; cat; echo; sleep 0.1'
echo '### Test --rrs -N1 --recend single'
echo '### Test --rrs --regexp --recend alternate'
echo 12a34b45a6 |
parallel -k --pipe --regexp --recend 'a|b' --rrs 'echo -n "$PARALLEL_SEQ>"; cat; echo; sleep 0.1'
echo '### Test --rrs --recend single'
echo 12a34b45a6 |
parallel -k --pipe --recend 'b' --rrs 'echo -n "$PARALLEL_SEQ>"; cat; echo; sleep 0.1'

View file

@ -1,13 +1,14 @@
#!/bin/bash
echo '### Test 0-arguments'
seq 1 2 | parallel -n0 echo n0
seq 1 2 | parallel -L0 echo L0
seq 1 2 | parallel -l0 echo l0
seq 1 2 | parallel -N0 echo N0
seq 1 2 | parallel -k -n0 echo n0
seq 1 2 | parallel -k -L0 echo L0
seq 1 2 | parallel -k -N0 echo N0
echo '### Because of --tollef -l, then -l0 == -l1, sorry'
seq 1 2 | parallel -k -l0 echo l0
echo '### Test replace {}'
seq 1 2 | parallel -N0 echo replace {} curlies
seq 1 2 | parallel -k -N0 echo replace {} curlies
echo '### Test arguments on commandline'
parallel -N0 echo args on cmdline ::: 1 2
parallel -k -N0 echo args on cmdline ::: 1 2

View file

@ -0,0 +1,28 @@
#!/bin/bash
echo '### Test https://savannah.gnu.org/bugs/index.php?31716'
seq 1 5 | stdout parallel -k -l echo {} OK
seq 1 5 | stdout parallel -k -l 1 echo {} OK
echo '### -k -l -0'
printf '1\0002\0003\0004\0005\000' | stdout parallel -k -l -0 echo {} OK
echo '### -k -0 -l'
printf '1\0002\0003\0004\0005\000' | stdout parallel -k -0 -l echo {} OK
echo '### -k -0 -l 1'
printf '1\0002\0003\0004\0005\000' | stdout parallel -k -0 -l 1 echo {} OK
echo '### -k -0 -l 0'
printf '1\0002\0003\0004\0005\000' | stdout parallel -k -0 -l 0 echo {} OK
echo '### -k -0 -L -0 - -0 is argument for -L'
printf '1\0002\0003\0004\0005\000' | stdout parallel -k -0 -L -0 echo {} OK
echo '### -k -0 -L 0 - -L always takes arg'
printf '1\0002\0003\0004\0005\000' | stdout parallel -k -0 -L 0 echo {} OK
echo '### -k -0 -L 0 - -L always takes arg'
printf '1\0002\0003\0004\0005\000' | stdout parallel -k -L 0 -0 echo {} OK
echo '### -k -e -0'
printf '1\0002\0003\0004\0005\000' | stdout parallel -k -e -0 echo {} OK
echo '### -k -0 -e eof'
printf '1\0002\0003\0004\0005\000' | stdout parallel -k -0 -e eof echo {} OK
echo '### -k -i -0'
printf '1\0002\0003\0004\0005\000' | stdout parallel -k -i -0 echo {} OK
echo '### -k -0 -i repl'
printf '1\0002\0003\0004\0005\000' | stdout parallel -k -0 -i repl echo repl OK

View file

@ -141,6 +141,12 @@ d
3>45
4>6
### Test --rrs -N1 --regexp --recend alternate
1>12
2>34
3>45
4>6
### Test --rrs -N1 --recend single
1>12a34
2>45a6
@ -149,7 +155,11 @@ d
1>123445
2>6
### Test --rrs -N1 --recend single
### Test --rrs --regexp --recend alternate
1>123445
2>6
### Test --rrs --recend single
1>12a34
2>45a6

View file

@ -3,10 +3,11 @@ n0
n0
L0
L0
l0
l0
N0
N0
### Because of --tollef -l, then -l0 == -l1, sorry
l0 1
l0 2
### Test replace {}
replace curlies
replace curlies

View file

@ -0,0 +1,77 @@
### Test https://savannah.gnu.org/bugs/index.php?31716
1 OK
2 OK
3 OK
4 OK
5 OK
1 OK
2 OK
3 OK
4 OK
5 OK
### -k -l -0
1 OK
2 OK
3 OK
4 OK
5 OK
### -k -0 -l
1 OK
2 OK
3 OK
4 OK
5 OK
### -k -0 -l 1
1 OK
2 OK
3 OK
4 OK
5 OK
### -k -0 -l 0
1 OK
2 OK
3 OK
4 OK
5 OK
### -k -0 -L -0 - -0 is argument for -L
OK
OK
OK
OK
OK
### -k -0 -L 0 - -L always takes arg
OK
OK
OK
OK
OK
### -k -0 -L 0 - -L always takes arg
OK
OK
OK
OK
OK
### -k -e -0
1 OK
2 OK
3 OK
4 OK
5 OK
### -k -0 -e eof
1 OK
2 OK
3 OK
4 OK
5 OK
### -k -i -0
1 OK
2 OK
3 OK
4 OK
5 OK
### -k -0 -i repl
1 OK
2 OK
3 OK
4 OK
5 OK