From 11b21d638ce0306dcac83b72f624f468969b6f81 Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Tue, 1 Mar 2011 23:04:15 +0100 Subject: [PATCH] -l 0 no longer works (and never will) because of --tollef. exponential backoff in --semaphore mode. --regexp test. testsuite passes. --- doc/release_new_version | 21 ++++++++- src/parallel | 55 ++++++++++++++--------- src/parallel.pod | 28 ++++++------ testsuite/tests-to-run/test51.sh | 16 +++---- testsuite/tests-to-run/test53.sh | 13 +++--- testsuite/tests-to-run/test54.sh | 28 ++++++++++++ testsuite/wanted-results/test51 | 12 ++++- testsuite/wanted-results/test53 | 5 ++- testsuite/wanted-results/test54 | 77 ++++++++++++++++++++++++++++++++ 9 files changed, 202 insertions(+), 53 deletions(-) create mode 100644 testsuite/tests-to-run/test54.sh create mode 100644 testsuite/wanted-results/test54 diff --git a/doc/release_new_version b/doc/release_new_version index 198c4a46..371e4235 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -147,10 +147,25 @@ at: http://ftp.gnu.org/gnu/parallel/ New in this release: -* --tollef to be switch compatible with Tollef's parallel. +* --tollef to be switch compatible with Tollef's parallel. This will + cause -l to mean --load, and the argument separator will be -- + instead of ::: + +* --gnu will force GNU Parallel to behave like GNU Parallel even if + --tollef is set. * Site wide config file: /etc/parallel/config + By putting --tollef in the site wide config file you can deinstall + Tollef's parallel and install GNU Parallel instead without any + change for users or scripts. This is useful for packagers that do + not distribute GNU Parallel because the command name conflicts with + Tollef's parallel. + +* -L 0 -n 0, and -N 0 implemented. They will read one argument, + but insert 0 arguments on the command line. Useful if you just want + to run the same command with the same arguments a number of times. + * Opscode Cookbook for Chef. Thanks to Joshua Timberman. https://github.com/opscode/cookbooks/tree/master/gnu_parallel @@ -166,8 +181,12 @@ New in this release: * A small example of grepping maillogs. Thanks to François Maillet. http://www.francoismaillet.com/blog/?p=399 +* Using GNU Parallel instead of xargs. Thanks to James Cuff. + http://blog.jcuff.net/2011/02/on-train-ride-in.html + * Bug fixes and man page updates. + = About GNU Parallel = GNU Parallel is a shell tool for executing jobs in parallel using one diff --git a/src/parallel b/src/parallel index 78315f75..2f1f0af5 100755 --- a/src/parallel +++ b/src/parallel @@ -304,7 +304,7 @@ sub get_options_from_array { "basenamereplace|bnr=s" => \$::opt_basenamereplace, "basenameextensionreplace=s" => \$::opt_basenameextensionreplace, "jobs|j=s" => \$::opt_P, - "load=s" => \$::opt_load, + "load=f" => \$::opt_load, "max-line-length-allowed" => \$::opt_max_line_length_allowed, "number-of-cpus" => \$::opt_number_of_cpus, "number-of-cores" => \$::opt_number_of_cores, @@ -322,7 +322,7 @@ sub get_options_from_array { "tmpdir=s" => \$::opt_tmpdir, "tempdir=s" => \$::opt_tmpdir, "tty|T" => \$::opt_tty, - "halt-on-error|H=s" => \$::opt_halt_on_error, + "halt-on-error|H=i" => \$::opt_halt_on_error, "retries=i" => \$::opt_retries, "dry-run|dryrun" => \$::opt_dryrun, "progress" => \$::opt_progress, @@ -353,8 +353,8 @@ sub get_options_from_array { "max-replace-args|N=i" => \$::opt_N, "colsep|col-sep|C=s" => \$::opt_colsep, "help|h" => \$::opt_help, - "L=i" => \$::opt_L, - "max-lines|l:s" => \$::opt_l, + "L=f" => \$::opt_L, + "max-lines|l:f" => \$::opt_l, "interactive|p" => \$::opt_p, "verbose|t" => \$::opt_verbose, "version|V" => \$::opt_version, @@ -381,7 +381,7 @@ sub get_options_from_array { sub parse_options { # Returns: N/A # Defaults: - $Global::version = 20110218; + $Global::version = 20110224; $Global::progname = 'parallel'; $Global::infinity = 2**31; $Global::debug = 0; @@ -484,14 +484,23 @@ sub parse_options { $Global::arg_sep = "--"; } } - for my $nlines ($::opt_L, $::opt_l) { - if(defined $nlines) { - if($nlines eq "") { - $nlines = 1; - } - $Global::max_lines = $nlines; - $Global::max_number_of_args ||= $Global::max_lines; + if(defined $::opt_l) { + if($::opt_l eq "-0") { + # -l -0 (swallowed -0) + $::opt_l = 1; + $::opt_0 = 1; + $/ = "\0"; + } elsif ($::opt_l == 0) { + # If not given (or if 0 is given) => 1 + $::opt_l = 1; } + $Global::max_lines = $::opt_l; + $Global::max_number_of_args ||= $Global::max_lines; + } + + if(defined $::opt_L) { + $Global::max_lines = $::opt_L; + $Global::max_number_of_args ||= $Global::max_lines; } %Global::replace_rev = reverse %Global::replace; @@ -3953,29 +3962,35 @@ sub new { 'id' => $id, 'idfile' => $lockdir."/".$id, 'pid' => $$, - 'pidfile' => $lockdir."/".$$, + 'pidfile' => $lockdir."/".$$.'@'.::hostname(), 'count' => $count + 1 # nlinks returns a link for the 'id-' as well }, ref($class) || $class; } sub acquire { my $self = shift; + my $exponential_backoff = 1; while(1) { $self->atomic_link_if_count_less_than() and last; ::debug("Remove dead locks"); my $lockdir = $self->{'lockdir'}; for my $d (<$lockdir/*>) { - $d =~ m:$lockdir/([0-9]+)$:o or next; - if(not kill 0, $1) { - ::debug("Dead: $d"); - unlink $d; - } else { - ::debug("Alive: $d"); + $d =~ m:$lockdir/([0-9]+)\@([-\._a-z0-9])$:o or next; + my ($pid, $host) = ($1,$2); + if($host eq ::hostname()) { + if(not kill 0, $1) { + ::debug("Dead: $d"); + unlink $d; + } else { + ::debug("Alive: $d"); + } } } # try again $self->atomic_link_if_count_less_than() and last; - sleep 1; + ::usleep(rand()*$exponential_backoff); + # Retry slower and slower up to 1 second + $exponential_backoff = ($exponential_backoff < 1000) ? ($exponential_backoff * 1.1) : ($exponential_backoff); # TODO if timeout: last } ::debug("acquired $self->{'pid'}\n"); diff --git a/src/parallel.pod b/src/parallel.pod index dd139645..890c1982 100644 --- a/src/parallel.pod +++ b/src/parallel.pod @@ -46,7 +46,8 @@ B is capable of. You can also watch the intro video for a quick introduction: http://www.youtube.com/watch?v=OpaiGYxkSuQ or at -http://tinyogg.com/watch/TORaR/ and http://tinyogg.com/watch/hfxKj/ +http://tinyogg.com/watch/TORaR/ http://tinyogg.com/watch/hfxKj/ and +http://tinyogg.com/watch/YQuXd/ =head1 OPTIONS @@ -82,7 +83,7 @@ B<{.}> can be used the same places as B<{}>. The replacement string B<{.}> can be changed with B<-U>. -=item B<{/}> (beta testing) +=item B<{/}> Basename of input line. This is a specialized replacement string with the directory part removed. @@ -91,7 +92,7 @@ B<{/}> can be used the same places as B<{}>. The replacement string B<{/}> can be changed with B<--basenamereplace>. -=item B<{/.}> (beta testing) +=item B<{/.}> Basename of input line without extension. This is a specialized replacement string with the directory and extension part removed. It @@ -117,7 +118,7 @@ extension. It is a combination of B<{>IB<}> and B<{.}>. B<{>I.B<}> can be used the same places as B<{>IB<}>. -=item B<{>I/B<}> (beta testing) +=item B<{>I/B<}> Basename of argument from argument file I or the I'th argument. It is a combination of B<{>IB<}> and B<{/}>. See B<-a> and B<-N>. @@ -125,7 +126,7 @@ It is a combination of B<{>IB<}> and B<{/}>. See B<-a> and B<-N>. B<{>I/B<}> can be used the same places as B<{>IB<}>. -=item B<{>I/.B<}> (beta testing) +=item B<{>I/.B<}> Basename of argument from argument file I or the I'th argument without extension. It is a combination of B<{>IB<}>, B<{/}>, and @@ -228,7 +229,7 @@ I will be transferred the same way as B<--transfer>. Use the replacement string I instead of B<{/}> for basename of input line. -=item B<--basenameextensionreplace> I (beta testing) +=item B<--basenameextensionreplace> I Use the replacement string I instead of B<{/.}> for basename of input line without extension. @@ -527,8 +528,7 @@ is optional. If I is not specified, it defaults to one. The B<-l> option is deprecated since the POSIX standard specifies B<-L> instead. -B<-l 0> means read one line, but insert 0 arguments on the command -line. +B<-l 0> is an alias for B<-l 1>. Implies B<-X> unless B<-m> is set. @@ -675,7 +675,7 @@ Print the number of CPU cores and exit (used by GNU B itself to determine the number of CPU cores on remote computers). -=item B<--nice> I (beta testing) +=item B<--nice> I Run the command at this niceness. For simple commands you can just add B in front of the command. But if the command consists of more @@ -750,11 +750,11 @@ Use B<--regexp> to interpret B<--recstart> and B<--recend> as regular expressions. This is slow, however. -=item B<--remove-rec-sep> (alpha testing) +=item B<--remove-rec-sep> (beta testing) -=item B<--removerecsep> (alpha testing) +=item B<--removerecsep> (beta testing) -=item B<--rrs> (alpha testing) +=item B<--rrs> (beta testing) Remove the text matched by B<--recstart> and B<--recend> before piping it to the command. @@ -1072,9 +1072,9 @@ Use B<-v> B<-v> to print the wrapping ssh command when running remotely. Print the version GNU B and exit. -=item B<--workdir> I (beta testing) +=item B<--workdir> I -=item B<-W> I (beta testing) +=item B<-W> I Files transferred using B<--transfer> and B<--return> will be relative to I on remote computers, and the command will be executed in diff --git a/testsuite/tests-to-run/test51.sh b/testsuite/tests-to-run/test51.sh index 06703937..4bc8fc8c 100644 --- a/testsuite/tests-to-run/test51.sh +++ b/testsuite/tests-to-run/test51.sh @@ -64,10 +64,9 @@ egrep -v '^0$' echo '### Test --rrs -N1 --recend single' echo 12a34a45a6 | parallel -k --pipe --recend a -N1 --rrs 'echo -n "$PARALLEL_SEQ>"; cat; echo; sleep 0.1' -# Broken -#echo '### Test --rrs -N1 --recend alternate' -#echo 12a34b45a6 | -# parallel -k --pipe --recend 'a|b' -N1 --rrs 'echo -n "$PARALLEL_SEQ>"; cat; echo; sleep 0.1' +echo '### Test --rrs -N1 --regexp --recend alternate' +echo 12a34b45a6 | + parallel -k --pipe --regexp --recend 'a|b' -N1 --rrs 'echo -n "$PARALLEL_SEQ>"; cat; echo; sleep 0.1' echo '### Test --rrs -N1 --recend single' echo 12a34b45a6 | parallel -k --pipe --recend 'b' -N1 --rrs 'echo -n "$PARALLEL_SEQ>"; cat; echo; sleep 0.1' @@ -75,11 +74,10 @@ echo 12a34b45a6 | echo '### Test --rrs --recend single' echo 12a34a45a6 | parallel -k --pipe --recend a --rrs 'echo -n "$PARALLEL_SEQ>"; cat; echo; sleep 0.1' -# Broken -#echo '### Test --rrs -N1 --recend alternate' -#echo 12a34b45a6 | -# parallel -k --pipe --recend 'a|b' --rrs 'echo -n "$PARALLEL_SEQ>"; cat; echo; sleep 0.1' -echo '### Test --rrs -N1 --recend single' +echo '### Test --rrs --regexp --recend alternate' +echo 12a34b45a6 | + parallel -k --pipe --regexp --recend 'a|b' --rrs 'echo -n "$PARALLEL_SEQ>"; cat; echo; sleep 0.1' +echo '### Test --rrs --recend single' echo 12a34b45a6 | parallel -k --pipe --recend 'b' --rrs 'echo -n "$PARALLEL_SEQ>"; cat; echo; sleep 0.1' diff --git a/testsuite/tests-to-run/test53.sh b/testsuite/tests-to-run/test53.sh index e0992f47..f94caf07 100644 --- a/testsuite/tests-to-run/test53.sh +++ b/testsuite/tests-to-run/test53.sh @@ -1,13 +1,14 @@ #!/bin/bash echo '### Test 0-arguments' -seq 1 2 | parallel -n0 echo n0 -seq 1 2 | parallel -L0 echo L0 -seq 1 2 | parallel -l0 echo l0 -seq 1 2 | parallel -N0 echo N0 +seq 1 2 | parallel -k -n0 echo n0 +seq 1 2 | parallel -k -L0 echo L0 +seq 1 2 | parallel -k -N0 echo N0 +echo '### Because of --tollef -l, then -l0 == -l1, sorry' +seq 1 2 | parallel -k -l0 echo l0 echo '### Test replace {}' -seq 1 2 | parallel -N0 echo replace {} curlies +seq 1 2 | parallel -k -N0 echo replace {} curlies echo '### Test arguments on commandline' -parallel -N0 echo args on cmdline ::: 1 2 +parallel -k -N0 echo args on cmdline ::: 1 2 diff --git a/testsuite/tests-to-run/test54.sh b/testsuite/tests-to-run/test54.sh new file mode 100644 index 00000000..1cbc642a --- /dev/null +++ b/testsuite/tests-to-run/test54.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +echo '### Test https://savannah.gnu.org/bugs/index.php?31716' +seq 1 5 | stdout parallel -k -l echo {} OK +seq 1 5 | stdout parallel -k -l 1 echo {} OK +echo '### -k -l -0' +printf '1\0002\0003\0004\0005\000' | stdout parallel -k -l -0 echo {} OK +echo '### -k -0 -l' +printf '1\0002\0003\0004\0005\000' | stdout parallel -k -0 -l echo {} OK +echo '### -k -0 -l 1' +printf '1\0002\0003\0004\0005\000' | stdout parallel -k -0 -l 1 echo {} OK +echo '### -k -0 -l 0' +printf '1\0002\0003\0004\0005\000' | stdout parallel -k -0 -l 0 echo {} OK +echo '### -k -0 -L -0 - -0 is argument for -L' +printf '1\0002\0003\0004\0005\000' | stdout parallel -k -0 -L -0 echo {} OK +echo '### -k -0 -L 0 - -L always takes arg' +printf '1\0002\0003\0004\0005\000' | stdout parallel -k -0 -L 0 echo {} OK +echo '### -k -0 -L 0 - -L always takes arg' +printf '1\0002\0003\0004\0005\000' | stdout parallel -k -L 0 -0 echo {} OK +echo '### -k -e -0' +printf '1\0002\0003\0004\0005\000' | stdout parallel -k -e -0 echo {} OK +echo '### -k -0 -e eof' +printf '1\0002\0003\0004\0005\000' | stdout parallel -k -0 -e eof echo {} OK +echo '### -k -i -0' +printf '1\0002\0003\0004\0005\000' | stdout parallel -k -i -0 echo {} OK +echo '### -k -0 -i repl' +printf '1\0002\0003\0004\0005\000' | stdout parallel -k -0 -i repl echo repl OK + diff --git a/testsuite/wanted-results/test51 b/testsuite/wanted-results/test51 index 079ccf5e..34729c9f 100644 --- a/testsuite/wanted-results/test51 +++ b/testsuite/wanted-results/test51 @@ -141,6 +141,12 @@ d 3>45 4>6 +### Test --rrs -N1 --regexp --recend alternate +1>12 +2>34 +3>45 +4>6 + ### Test --rrs -N1 --recend single 1>12a34 2>45a6 @@ -149,7 +155,11 @@ d 1>123445 2>6 -### Test --rrs -N1 --recend single +### Test --rrs --regexp --recend alternate +1>123445 +2>6 + +### Test --rrs --recend single 1>12a34 2>45a6 diff --git a/testsuite/wanted-results/test53 b/testsuite/wanted-results/test53 index 6ab97226..83dcd2bd 100644 --- a/testsuite/wanted-results/test53 +++ b/testsuite/wanted-results/test53 @@ -3,10 +3,11 @@ n0 n0 L0 L0 -l0 -l0 N0 N0 +### Because of --tollef -l, then -l0 == -l1, sorry +l0 1 +l0 2 ### Test replace {} replace curlies replace curlies diff --git a/testsuite/wanted-results/test54 b/testsuite/wanted-results/test54 new file mode 100644 index 00000000..6ece5795 --- /dev/null +++ b/testsuite/wanted-results/test54 @@ -0,0 +1,77 @@ +### Test https://savannah.gnu.org/bugs/index.php?31716 +1 OK +2 OK +3 OK +4 OK +5 OK +1 OK +2 OK +3 OK +4 OK +5 OK +### -k -l -0 +1 OK +2 OK +3 OK +4 OK +5 OK +### -k -0 -l +1 OK +2 OK +3 OK +4 OK +5 OK +### -k -0 -l 1 +1 OK +2 OK +3 OK +4 OK +5 OK +### -k -0 -l 0 +1 OK +2 OK +3 OK +4 OK +5 OK +### -k -0 -L -0 - -0 is argument for -L +OK +OK +OK +OK +OK +### -k -0 -L 0 - -L always takes arg +OK +OK +OK +OK +OK +### -k -0 -L 0 - -L always takes arg +OK +OK +OK +OK +OK +### -k -e -0 +1 OK +2 OK +3 OK +4 OK +5 OK +### -k -0 -e eof +1 OK +2 OK +3 OK +4 OK +5 OK +### -k -i -0 +1 OK +2 OK +3 OK +4 OK +5 OK +### -k -0 -i repl +1 OK +2 OK +3 OK +4 OK +5 OK