From fe8a8aa41b3fd3b9c941095e55656f15a02238ed Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Sun, 17 Jul 2011 01:46:02 +0200 Subject: [PATCH] Fixed bug #33630: -L and -N do not do context replace. --bner alias for --basenameextensionreplace. --- src/parallel | 59 +++++++++++++++++++------------- src/parallel.pod | 18 ++++++---- testsuite/Start.sh | 13 +++++-- testsuite/tests-to-run/test42.sh | 8 +++++ testsuite/tests-to-run/test48.sh | 6 ++-- testsuite/tests-to-run/test60.sh | 6 ++-- testsuite/wanted-results/test15 | 14 ++++---- testsuite/wanted-results/test27 | 6 ++-- testsuite/wanted-results/test33 | 22 ++++++++++-- testsuite/wanted-results/test42 | 16 +++++++++ testsuite/wanted-results/test48 | 6 ++++ testsuite/wanted-results/test56 | 2 +- testsuite/wanted-results/test60 | 8 ----- 13 files changed, 123 insertions(+), 61 deletions(-) diff --git a/src/parallel b/src/parallel index ee566009..3007aaf2 100755 --- a/src/parallel +++ b/src/parallel @@ -24,7 +24,7 @@ parse_options(); my $number_of_args; if($Global::max_number_of_args) { $number_of_args=$Global::max_number_of_args; -} elsif ($Global::Xargs or $Global::xargs) { +} elsif ($::opt_X or $::opt_m) { $number_of_args = undef; } else { $number_of_args = 1; @@ -89,7 +89,7 @@ if($::opt_nonall or $::opt_onall) { } $Global::JobQueue = JobQueue->new( - $command,\@fhlist,$Global::Xargs,$number_of_args,\@Global::ret_files); + $command,\@fhlist,$Global::ContextReplace,$number_of_args,\@Global::ret_files); if($::opt_eta) { # Count the number of jobs before starting any $Global::JobQueue->total_jobs(); @@ -336,7 +336,8 @@ sub options_hash { # Returns a hash of the GetOptions config return ("debug|D" => \$::opt_D, - "xargs|m" => \$::opt_m, + "xargs" => \$::opt_xargs, + "m" => \$::opt_m, "X" => \$::opt_X, "v" => \@::opt_v, "joblog=s" => \$::opt_joblog, @@ -350,7 +351,7 @@ sub options_hash { "extensionreplace|U=s" => \$::opt_U, "basenamereplace|bnr=s" => \$::opt_basenamereplace, "dirnamereplace|dnr=s" => \$::opt_dirnamereplace, - "basenameextensionreplace=s" => \$::opt_basenameextensionreplace, + "basenameextensionreplace|bner=s" => \$::opt_basenameextensionreplace, "seqreplace=s" => \$::opt_seqreplace, "jobs|j=s" => \$::opt_P, "load=f" => \$::opt_load, @@ -453,7 +454,7 @@ sub get_options_from_array { sub parse_options { # Returns: N/A # Defaults: - $Global::version = 20110622; + $Global::version = 20110709; $Global::progname = 'parallel'; $Global::infinity = 2**31; $Global::debug = 0; @@ -483,8 +484,7 @@ sub parse_options { if(defined @::opt_v) { $Global::verbose = $#::opt_v+1; } # Convert -v -v to v=2 $Global::debug = (defined $::opt_D); - if(defined $::opt_m) { $Global::xargs = 1; } - if(defined $::opt_X) { $Global::Xargs = 1; } + if(defined $::opt_X) { $Global::ContextReplace = 1; } if(defined $::opt_silent) { $Global::verbose = 0; } if(defined $::opt_k) { $Global::keeporder = 1; } if(defined $::opt_g) { $Global::grouped = 1; } @@ -510,7 +510,6 @@ sub parse_options { } if(defined $::opt_E) { $Global::end_of_file_string = $::opt_E; } if(defined $::opt_n) { $Global::max_number_of_args = $::opt_n; } - if(defined $::opt_N) { $Global::max_number_of_args = $::opt_N; } if(defined $::opt_tmpdir) { $ENV{'TMPDIR'} = $::opt_tmpdir; } if(defined $::opt_help) { die_usage(); } if(defined $::opt_colsep) { $Global::trim = 'lr'; } @@ -590,10 +589,21 @@ sub parse_options { $Global::max_number_of_args ||= $Global::max_lines; } + # Read more than one arg at a time (-L, -N) if(defined $::opt_L) { $Global::max_lines = $::opt_L; $Global::max_number_of_args ||= $Global::max_lines; } + if(defined $::opt_N) { + $Global::max_number_of_args = $::opt_N; + $Global::ContextReplace = 1; + } + if((defined $::opt_L or defined $::opt_N) + and + not ($::opt_xargs or $::opt_m)) { + $Global::ContextReplace = 1; + } + %Global::replace_rev = reverse %Global::replace; if(grep /^$Global::arg_sep$|^$Global::arg_file_sep$/o, @ARGV) { @@ -634,8 +644,7 @@ sub parse_options { parse_sshlogin(); - if(remote_hosts() and ($Global::xargs or $Global::Xargs) - and not $::opt_N) { + if(remote_hosts() and ($::opt_X or $::opt_m or $::opt_xargs)) { # As we do not know the max line length on the remote machine # long commands generated by xargs may fail # If opt_N is set, it is probably safe @@ -1646,22 +1655,23 @@ sub usage { "Usage:", "$Global::progname [options] [command [arguments]] < list_of_arguments", "$Global::progname [options] [command [arguments]] (::: arguments|:::: argfile(s))...", + "cat ... | $Global::progname --pipe [options] [command [arguments]]", "", - "-j n Run n jobs in parallel", - "-k Keep same order", - "-X Multiple arguments with context replace", - "--colsep regexp Split input on regexp for positional replacements", - "{} {.} {/} {/.} Replacement strings", + "-j n Run n jobs in parallel", + "-k Keep same order", + "-X Multiple arguments with context replace", + "--colsep regexp Split input on regexp for positional replacements", + "{} {.} {/} {/.} {#} Replacement strings", "{3} {3.} {3/} {3/.} Positional replacement strings", "", - "-S sshlogin - e.g. foo\@server.example.com", - "--trc {}.bar Shorthand for --transfer --return {}.bar --cleanup", - "--onall Run the given command with argument on all sshlogins", - "--nonall Run the given command with no arguments on all sshlogins", + "-S sshlogin Example: foo\@server.example.com", + "--trc {}.bar Shorthand for --transfer --return {}.bar --cleanup", + "--onall Run the given command with argument on all sshlogins", + "--nonall Run the given command with no arguments on all sshlogins", "", - "--pipe Split stdin (standard input) to multiple jobs.", - "--recend Record end separator for --pipe.", - "--recstart Record start separator for --pipe.", + "--pipe Split stdin (standard input) to multiple jobs.", + "--recend str Record end separator for --pipe.", + "--recstart str Record start separator for --pipe.", "", "See 'man $Global::progname' for details", ""); @@ -4010,7 +4020,8 @@ sub get { my $ret = $self->{'arg_sub_queue'}->get(); if(defined $Global::max_number_of_args and $Global::max_number_of_args == 0) { - return []; + ::debug("Read 1 but return 0 args\n"); + return [Arg->new("")]; } else { return $ret; } @@ -4379,7 +4390,7 @@ sub trim_of { for $arg (@strings) { $arg =~ s/^\s+//; $arg =~ s/\s+$//; } } else { print STDERR "$Global::progname: --trim must be one of: r l ". - "rl lr\n"; + "rl lr\n"; ::wait_and_exit(255); } return wantarray ? @strings : "@strings"; diff --git a/src/parallel.pod b/src/parallel.pod index f25b825d..fcca00ff 100644 --- a/src/parallel.pod +++ b/src/parallel.pod @@ -73,7 +73,9 @@ http://www.perlmonks.org/index.pl?node_id=484296). =item B<{}> (still alpha testing) Input line. This replacement string will be replaced by a full line -read from the input source. +read from the input source. The input source is normally stdin +(standard input), but can also be given with B<-a>, B<:::>, or +B<::::>. The replacement string B<{}> can be changed with B<-I>. @@ -194,9 +196,9 @@ To understand positional replacement strings see B<{>IB<}>. =item B<:::> I (alpha testing) -Use arguments from the command line as input source instead of from -stdin (standard input). Unlike other options for GNU B -B<:::> is placed after the I and before the arguments. +Use arguments from the command line as input source instead of stdin +(standard input). Unlike other options for GNU B B<:::> is +placed after the I and before the arguments. The following are equivalent: @@ -234,7 +236,7 @@ Another way to write B<-a> I B<-a> I ... B<:::> and B<::::> can be mixed. -See B<-a>. +See B<-a> and B<:::>. =item B<--null> @@ -261,7 +263,7 @@ contains B. B<-a foo> B<-a bar> will result in the combinations (1,a) (1,b) (1,c) (2,a) (2,b) (2,c). This is useful for replacing nested for-loops. -See also B<--xapply> +See also B<--xapply> and B<{>IB<}>. =item B<--arg-file-sep> I @@ -306,6 +308,8 @@ basename of input line. =item B<--basenameextensionreplace> I +=item B<--bner> I + Use the replacement string I instead of B<{/.}> for basename of input line without extension. @@ -603,7 +607,7 @@ next input line. B<-L 0> means read one line, but insert 0 arguments on the command line. -Implies B<-X> unless B<-m> is set. +Implies B<-X> unless B<-m> or B<--xargs> is set. =item B<--max-lines>[=I] diff --git a/testsuite/Start.sh b/testsuite/Start.sh index 3412e1ad..5eff968a 100644 --- a/testsuite/Start.sh +++ b/testsuite/Start.sh @@ -8,8 +8,15 @@ ls -t tests-to-run/*.sh \ >$SHFILE mkdir -p actual-results -sh -x $SHFILE +stdout sh -x $SHFILE | tee testsuite.log rm $SHFILE - - +# If testsuite.log contains @@ then there is a diff +if grep -q '@@' testsuite.log ; then + false +else + # No @@'s: So everything worked: Copy the source + rm -rf src-passing-testsuite + cp -a ../src src-passing-testsuite +fi +rm testsuite.log diff --git a/testsuite/tests-to-run/test42.sh b/testsuite/tests-to-run/test42.sh index ae22b980..dea31f12 100644 --- a/testsuite/tests-to-run/test42.sh +++ b/testsuite/tests-to-run/test42.sh @@ -4,10 +4,18 @@ echo "### Test --basenamereplace" parallel -j1 -k -X --basenamereplace FOO echo FOO ::: /a/b.c a/b.c b.c /a/b a/b b parallel -k --basenamereplace FOO echo FOO ::: /a/b.c a/b.c b.c /a/b a/b b +echo "### Test --bnr" +parallel -j1 -k -X --bnr FOO echo FOO ::: /a/b.c a/b.c b.c /a/b a/b b +parallel -k --bnr FOO echo FOO ::: /a/b.c a/b.c b.c /a/b a/b b + echo "### Test --basenameextensionreplace" parallel -j1 -k -X --basenameextensionreplace FOO echo FOO ::: /a/b.c a/b.c b.c /a/b a/b b parallel -k --basenameextensionreplace FOO echo FOO ::: /a/b.c a/b.c b.c /a/b a/b b +echo "### Test --bner" +parallel -j1 -k -X --bner FOO echo FOO ::: /a/b.c a/b.c b.c /a/b a/b b +parallel -k --bner FOO echo FOO ::: /a/b.c a/b.c b.c /a/b a/b b + echo "### Test {/}" parallel -j1 -k -X echo {/} ::: /a/b.c a/b.c b.c /a/b a/b b diff --git a/testsuite/tests-to-run/test48.sh b/testsuite/tests-to-run/test48.sh index 4adba56f..15d1da76 100644 --- a/testsuite/tests-to-run/test48.sh +++ b/testsuite/tests-to-run/test48.sh @@ -15,6 +15,8 @@ seq 9 | parallel -N 10 echo echo '### Test -N is not broken by distribution - two lines' seq 19 | parallel -k -N 10 echo +echo '### Test -N context replace' +seq 19 | parallel -N 10 echo a{}b - - +echo '### Test -L context replace' +seq 19 | parallel -L 10 echo a{}b diff --git a/testsuite/tests-to-run/test60.sh b/testsuite/tests-to-run/test60.sh index 6c2e02f2..d012f9ab 100644 --- a/testsuite/tests-to-run/test60.sh +++ b/testsuite/tests-to-run/test60.sh @@ -3,9 +3,9 @@ SERVER1=parallel-server3 SERVER2=parallel-server2 -cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | parallel -k +cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | parallel -j0 -k echo '### Test --onall' -parallel --onall -S parallel@$SERVER2,$SERVER1 '(echo {3} {2}) | awk \{print\ \$2}' ::: a b c ::: 1 2 3 +parallel --onall -S parallel@$SERVER2,$SERVER1 '(echo {3} {2}) | awk \{print\ \$2}' ::: a b c ::: 1 2 echo '### Test | --onall' seq 3 | parallel --onall -S parallel@$SERVER2,$SERVER1 '(echo {3} {2}) | awk \{print\ \$2}' ::: a b c :::: - echo '### Test --onall -u' @@ -13,7 +13,7 @@ parallel --onall -S parallel@$SERVER2,$SERVER1 -u '(echo {3} {2}) | awk \{print\ echo '### Test --nonall' parallel --nonall -k -S parallel@$SERVER2,$SERVER1 'hostname' | sort echo '### Test --nonall -u' -parallel --nonall -S parallel@$SERVER2,$SERVER1 -vu 'sleep {#};hostname;sleep 3;hostname' +parallel --nonall -S parallel@$SERVER2,$SERVER1 -u 'hostname|grep -q nlv.pi.dk && sleep 2; hostname;sleep 4;hostname;' echo '### Test read sshloginfile from STDIN' echo nlv.pi.dk | parallel -S - --nonall hostname echo nlv.pi.dk | parallel --sshloginfile - --nonall hostname diff --git a/testsuite/wanted-results/test15 b/testsuite/wanted-results/test15 index 222be092..43dff7fb 100644 --- a/testsuite/wanted-results/test15 +++ b/testsuite/wanted-results/test15 @@ -82,14 +82,14 @@ a_b c d e ### test too long args -Command line too long (1000005 >= 131071) at number 1: zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz... +parallel: Command line too long (1000005 >= 131071) at number 1: zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz... xargs: argument line too long 1 2 3 4 5 6 7 8 9 10 -Command line too long (1000007 >= 10) at number 5: zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz... +parallel: Command line too long (1000007 >= 10) at number 5: zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz... 1 2 3 4 5 6 @@ -101,20 +101,20 @@ xargs: argument line too long 5 6 7 8 9 10 -Command line too long (1000007 >= 10) at number 5: zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz... +parallel: Command line too long (1000007 >= 10) at number 5: zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz... ### Test -x 1 2 3 4 5 6 7 8 9 10 -Command line too long (10 >= 10) at number 5: 12345... +parallel: Command line too long (10 >= 10) at number 5: 12345... 1 2 3 4 5 6 7 8 9 10 -Command line too long (10 >= 10) at number 5: 12345... +parallel: Command line too long (10 >= 10) at number 5: 12345... 1 2 3 4 5 6 @@ -287,8 +287,8 @@ line 2 ### Test --no-run-if-empty and -r: This should give no output ### Test --help and -h: Help output (just check we get the same amount of lines) Output from -h and --help -6 -6 +22 +22 ### Test --version: Version output (just check we get the same amount of lines) 12 ### Test --verbose and -t diff --git a/testsuite/wanted-results/test27 b/testsuite/wanted-results/test27 index 9c89d234..0055e266 100644 --- a/testsuite/wanted-results/test27 +++ b/testsuite/wanted-results/test27 @@ -71,7 +71,7 @@ y ### -i -s26 -0 echo from \{\} to x{}y < items-0.xi xargs: argument list too long from one to xoney -Command line too long (42 >= 26) at number 2: +parallel: Command line too long (42 >= 26) at number 2: ... ### -l -0 echo < ldata-0.xi 1 22 333 4444 @@ -808,7 +808,7 @@ with 'single quotes' as well. /src/gnu/autoconf-1.11/texinfo.tex ### -s6 echo < files.xi xargs: argument line too long -Command line too long (27 >= 6) at number 1: /src/gnu/autoconf-1.11... +parallel: Command line too long (27 >= 6) at number 1: /src/gnu/autoconf-1.11... ### -iARG -s86 echo ARG is xARGx < files.xi /src/gnu/autoconf-1.11 is x/src/gnu/autoconf-1.11x /src/gnu/autoconf-1.11/README is x/src/gnu/autoconf-1.11/READMEx @@ -974,7 +974,7 @@ from: can't read /var/mail/{} ### -i -s26 echo from \{\} to x{}y < items.xi xargs: argument list too long from dumb to xdumby -Command line too long (36 >= 26) at number 1: ... +parallel: Command line too long (36 >= 26) at number 1: ... ### -i__ echo FIRST __ IS OK < quotes.xi FIRST this is IS OK FIRST quoted stuff IS OK diff --git a/testsuite/wanted-results/test33 b/testsuite/wanted-results/test33 index 4c9c8e37..b2165a09 100644 --- a/testsuite/wanted-results/test33 +++ b/testsuite/wanted-results/test33 @@ -1,10 +1,26 @@ ### Test of -j filename Usage: parallel [options] [command [arguments]] < list_of_arguments -parallel [options] [command [arguments]] ::: arguments -parallel [options] [command [arguments]] :::: argfile(s) +parallel [options] [command [arguments]] (::: arguments|:::: argfile(s))... +cat ... | parallel --pipe [options] [command [arguments]] -See 'man parallel' for the options +-j n Run n jobs in parallel +-k Keep same order +-X Multiple arguments with context replace +--colsep regexp Split input on regexp for positional replacements +{} {.} {/} {/.} {#} Replacement strings +{3} {3.} {3/} {3/.} Positional replacement strings + +-S sshlogin - e.g. foo@server.example.com +--trc {}.bar Shorthand for --transfer --return {}.bar --cleanup +--onall Run the given command with argument on all sshlogins +--nonall Run the given command with no arguments on all sshlogins + +--pipe Split stdin (standard input) to multiple jobs. +--recend Record end separator for --pipe. +--recstart Record start separator for --pipe. + +See 'man parallel' for details Parsing of --jobs/-j/--max-procs/-P failed ### Test of -j filename sleep 0.7 diff --git a/testsuite/wanted-results/test42 b/testsuite/wanted-results/test42 index a9c66fba..d7585530 100644 --- a/testsuite/wanted-results/test42 +++ b/testsuite/wanted-results/test42 @@ -6,6 +6,14 @@ b.c b b b +### Test --bnr +b.c b.c b.c b b b +b.c +b.c +b.c +b +b +b ### Test --basenameextensionreplace b b b b b b b @@ -14,6 +22,14 @@ b b b b +### Test --bner +b b b b b b +b +b +b +b +b +b ### Test {/} b.c b.c b.c b b b ### Test {/.} diff --git a/testsuite/wanted-results/test48 b/testsuite/wanted-results/test48 index 29b9f73e..859ce375 100644 --- a/testsuite/wanted-results/test48 +++ b/testsuite/wanted-results/test48 @@ -46,3 +46,9 @@ ### Test -N is not broken by distribution - two lines 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 +### Test -N context replace +a1b a2b a3b a4b a5b a6b a7b a8b a9b a10b +a11b a12b a13b a14b a15b a16b a17b a18b a19b +### Test -L context replace +a1b a2b a3b a4b a5b a6b a7b a8b a9b a10b +a11b a12b a13b a14b a15b a16b a17b a18b a19b diff --git a/testsuite/wanted-results/test56 b/testsuite/wanted-results/test56 index bf4a06ee..de8d98b3 100644 --- a/testsuite/wanted-results/test56 +++ b/testsuite/wanted-results/test56 @@ -18,4 +18,4 @@ 1 1 130001 1 1 130001 1 1 130001 -Command line too long (260009 >= 131071) at number 9: 10... +parallel: Command line too long (260009 >= 131071) at number 9: 10... diff --git a/testsuite/wanted-results/test60 b/testsuite/wanted-results/test60 index 8d4b8461..b4361745 100644 --- a/testsuite/wanted-results/test60 +++ b/testsuite/wanted-results/test60 @@ -1,22 +1,16 @@ ### Test --onall 1 2 -3 1 2 -3 1 2 -3 1 2 -3 1 2 -3 1 2 -3 ### Test | --onall 1 2 @@ -59,8 +53,6 @@ nlv.pi.dk vh2.pi.dk ### Test --nonall -u -ssh parallel-server3 PARALLEL_SEQ=$PARALLEL_SEQ\;export PARALLEL_SEQ\;PARALLEL_PID=$PARALLEL_PID\;export PARALLEL_PID\; sleep\ 1\;hostname\;sleep\ 3\;hostname; -ssh parallel@parallel-server2 PARALLEL_SEQ=$PARALLEL_SEQ\;export PARALLEL_SEQ\;PARALLEL_PID=$PARALLEL_PID\;export PARALLEL_PID\; sleep\ 1\;hostname\;sleep\ 3\;hostname; vh2.pi.dk nlv.pi.dk vh2.pi.dk