From 9fd660be715a3f41df595486d08c6a67d6223cb3 Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Sat, 26 Jun 2010 01:56:14 +0200 Subject: [PATCH] Bugfix: The length for -X is not close to max (131072) --- doc/FUTURE_IDEAS | 25 -------- doc/release_new_version | 30 ++++++++-- src/parallel | 13 ++-- unittest/Makefile | 3 +- unittest/Start.sh | 1 + unittest/tests-to-run/test15.sh | 102 ++++++++++++++++++-------------- unittest/tests-to-run/test24.sh | 11 ++++ unittest/wanted-results/test15 | 69 +++++++++++++++++---- unittest/wanted-results/test24 | 11 ++++ 9 files changed, 177 insertions(+), 88 deletions(-) mode change 100644 => 100755 unittest/tests-to-run/test24.sh diff --git a/doc/FUTURE_IDEAS b/doc/FUTURE_IDEAS index 25da7e63..017e91f8 100644 --- a/doc/FUTURE_IDEAS +++ b/doc/FUTURE_IDEAS @@ -1,5 +1,3 @@ -# BUG: bash -c 'parallel -a <(seq 1 3) -p echo' - # Hvordan udregnes system limits på remote systems hvis jeg ikke ved, hvormange # argumenter, der er? Lav system limits lokalt og lad det være max @@ -16,29 +14,6 @@ # Clustering Tools | Command Line Tools | Utilities | System Administration # Bash parallel -* 100% options complete with xargs. All options for xargs can now - be used in GNU Parallel - even the more exotic. - -* --basefile for transfering basedata. When running jobs on remote - computers --basefile will transfer files before the first jobs is - run. It can be used to transfer data that remains the same for each - job such as scripts or lookup tables. - -* --progress shows progress. To see how many jobs is running on each - server use --progress. It can be turned on even after GNU Parallel - is started. - -* --eta shows estimated time left in seconds. - -* --halt-on-error stops if an error occurs. GNU Parallel will default - to run all jobs - even if some of them fail. With --halt-on-error - GNU Parallel can ignore errors, wait for the currently running jobs - to finish, or stop immediately when an error occurs. - -* New video showing the new options. - - - =head1 YouTube video GNU Parallel is a tool with lots of uses in shell. Every time you use diff --git a/doc/release_new_version b/doc/release_new_version index 6e1bde72..26100d9b 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -67,19 +67,41 @@ http://freshmeat.net/projects/parallel/releases/new == Send announce == +http://groups.google.com/group/comp.unix.shell/post Newsgroups: comp.unix.shell,comp.unix.admin <<<<< to:parallel@gnu.org, bug-parallel@gnu.org, info-gnu@gnu.org, bug-directory@gnu.org -Subject: GNU Parallel 20100601 released +Subject: GNU Parallel 20100620 released -GNU Parallel 20100601 has been released today. It is available for +GNU Parallel 20100620 has been released. It is available for download at: http://ftp.gnu.org/gnu/parallel/ New in this release: -* GNU Parallel now has progress indicator +* New video showing the new options. + http://www.youtube.com/watch?v=OpaiGYxkSuQ + +* 100% options complete with xargs. All options for xargs can now + be used in GNU Parallel - even the more exotic. + +* --basefile for transferring basedata. When running jobs on remote + computers --basefile will transfer files before the first jobs is + run. It can be used to transfer data that remains the same for each + job such as scripts or lookup tables. + +* --progress shows progress. To see how many jobs is running on each + server use --progress. It can be turned on even after GNU Parallel + is started. + +* --eta shows estimated time left in seconds. + +* --halt-on-error stops if an error occurs. GNU Parallel will default + to run all jobs - even if some of them fail. With --halt-on-error + GNU Parallel can ignore errors, wait for the currently running jobs + to finish, or stop immediately when an error occurs. + = About GNU Parallel = @@ -103,6 +125,6 @@ possible to use output from GNU Parallel as input for other programs. You can find more about GNU Parallel at: http://www.gnu.org/software/parallel/ -Watch the intro video on http://www.youtube.com/watch?v=LlXDtd_pRaY +Watch the intro video on http://www.youtube.com/watch?v=OpaiGYxkSuQ >>>>> diff --git a/src/parallel b/src/parallel index c129d3c2..18f5f0a6 100755 --- a/src/parallel +++ b/src/parallel @@ -1908,6 +1908,10 @@ sub xargs_computations { $spaces,$length_of_command_no_args,$length_of_context) = (1,0,0,0,0); if($command) { + if($command !~ /\s\S*\Q$Global::replacestring\E\S*|\s\S*\Q$Global::replace_no_ext\E\S*/o) { + # No replacement strings: add {} + $command .= " ".$Global::replacestring; + } # Count number of {}'s on the command line my $no_of_replace = ($command =~ s/\Q$Global::replacestring\E/$Global::replacestring/go); @@ -1919,13 +1923,15 @@ sub xargs_computations { # Count my $c = $command; if($Global::Xargs) { - $c =~ s/\S*\Q$Global::replacestring\E\S*//go; - $c =~ s/\S*\Q$Global::replace_no_ext\E\S*//go; - $length_of_command_no_args = length($c) - 1; + $c =~ s/\s\S*\Q$Global::replacestring\E\S*|\s\S*\Q$Global::replace_no_ext\E\S*//go; + $length_of_command_no_args = length($c); $length_of_context = length($command) - $length_of_command_no_args - $no_of_replace * length($Global::replacestring) - $no_of_no_ext * length($Global::replace_no_ext); $spaces = 0; + debug("length_of_command_no_args $length_of_command_no_args\n"); + debug("length_of_context $length_of_context\n"); + debug("no_of_replace $no_of_replace no_of_no_ext $no_of_no_ext\n"); } else { # remove all {}s $c =~ s/\Q$Global::replacestring\E|\Q$Global::replace_no_ext\E//og; @@ -3321,4 +3327,3 @@ $main::opt_version = $main::opt_L = $main::opt_l = $main::opt_show_limits = $main::opt_n = $main::opt_e = $main::opt_verbose = $main::opt_E = $main::opt_r = $Global::xargs = $Global::keeporder = $Global::control_path = 0; - diff --git a/unittest/Makefile b/unittest/Makefile index c9030d33..6dbeef3f 100644 --- a/unittest/Makefile +++ b/unittest/Makefile @@ -6,7 +6,8 @@ unittest: ../src/parallel tests-to-run/* wanted-results/* ren 2>&1 | mop || (echo ren is required for unittest; /bin/false) echo | buffer | mop || (echo buffer is required for unittest; /bin/false) echo 1+2 | bc | mop || (echo bc is required for unittest; /bin/false) - stdout gawk | mop || (echo gawk is required for parallel; /bin/false) + stdout gawk | mop || (echo gawk is required for unittest; /bin/false) + expect -c 'spawn cat; puts "expect is installed"' || (echo expect is required for unittest; /bin/false) sh Start.sh clean: diff --git a/unittest/Start.sh b/unittest/Start.sh index 9e4c77fd..68ed4bff 100644 --- a/unittest/Start.sh +++ b/unittest/Start.sh @@ -7,6 +7,7 @@ ls -t tests-to-run/test*.sh \ | perl -pe 's:(.*/(.*)).sh:sh $1.sh > actual-results/$2; diff -Naur wanted-results/$2 actual-results/$2:' \ >$SHFILE +mkdir -p actual-results sh -x $SHFILE rm $SHFILE diff --git a/unittest/tests-to-run/test15.sh b/unittest/tests-to-run/test15.sh index b3314969..ec8931d5 100755 --- a/unittest/tests-to-run/test15.sh +++ b/unittest/tests-to-run/test15.sh @@ -2,8 +2,6 @@ # Test xargs compatibility -PAR=parallel - echo '### Test -p --interactive' cat >/tmp/parallel-script-for-expect <<_EOF #!/bin/bash @@ -36,22 +34,31 @@ _EOF echo '### Test -L -l and --max-lines' (echo a_b;echo c) | parallel -km -L2 echo +(echo a_b;echo c) | parallel -k -L2 echo (echo a_b;echo c) | xargs -L2 echo (echo a_b;echo c) | parallel -km -L1 echo +(echo a_b;echo c) | parallel -k -L1 echo (echo a_b;echo c) | xargs -L1 echo (echo a_b' ';echo c;echo d) | parallel -km -L1 echo +(echo a_b' ';echo c;echo d) | parallel -k -L1 echo (echo a_b' ';echo c;echo d) | xargs -L1 echo (echo a_b' ';echo c;echo d;echo e) | parallel -km -L2 echo +(echo a_b' ';echo c;echo d;echo e) | parallel -k -L2 echo (echo a_b' ';echo c;echo d;echo e) | xargs -L2 echo (echo a_b' ';echo c;echo d;echo e) | parallel -km -l echo +(echo a_b' ';echo c;echo d;echo e) | parallel -k -l echo (echo a_b' ';echo c;echo d;echo e) | xargs -l echo (echo a_b' ';echo c;echo d;echo e) | parallel -km -l2 echo +(echo a_b' ';echo c;echo d;echo e) | parallel -k -l2 echo (echo a_b' ';echo c;echo d;echo e) | xargs -l2 echo (echo a_b' ';echo c;echo d;echo e) | parallel -km -l1 echo +(echo a_b' ';echo c;echo d;echo e) | parallel -k -l1 echo (echo a_b' ';echo c;echo d;echo e) | xargs -l1 echo (echo a_b' ';echo c;echo d;echo e) | parallel -km --max-lines 2 echo +(echo a_b' ';echo c;echo d;echo e) | parallel -k --max-lines 2 echo (echo a_b' ';echo c;echo d;echo e) | xargs --max-lines=2 echo (echo a_b' ';echo c;echo d;echo e) | parallel -km --max-lines echo +(echo a_b' ';echo c;echo d;echo e) | parallel -k --max-lines echo (echo a_b' ';echo c;echo d;echo e) | xargs --max-lines echo echo '### test too long args' @@ -59,10 +66,13 @@ perl -e 'print "z"x1000000' | parallel echo 2>&1 perl -e 'print "z"x1000000' | xargs echo 2>&1 (seq 1 10; perl -e 'print "z"x1000000'; seq 12 15) | stdout parallel -j1 -km -s 10 echo (seq 1 10; perl -e 'print "z"x1000000'; seq 12 15) | stdout xargs -s 10 echo +(seq 1 10; perl -e 'print "z"x1000000'; seq 12 15) | stdout parallel -j1 -k -s 10 echo echo '### Test -x' (seq 1 10; echo 12345; seq 12 15) | stdout parallel -j1 -km -s 10 -x echo +(seq 1 10; echo 12345; seq 12 15) | stdout parallel -j1 -k -s 10 -x echo (seq 1 10; echo 12345; seq 12 15) | stdout xargs -s 10 -x echo (seq 1 10; echo 1234; seq 12 15) | stdout parallel -j1 -km -s 10 -x echo +(seq 1 10; echo 1234; seq 12 15) | stdout parallel -j1 -k -s 10 -x echo (seq 1 10; echo 1234; seq 12 15) | stdout xargs -s 10 -x echo echo '### Test bugfix if no command given' (echo echo; seq 1 5; perl -e 'print "z"x1000000'; seq 12 15) | stdout parallel -j1 -km -s 10 @@ -71,8 +81,8 @@ echo '### Test bugfix if no command given' echo '### Test -a and --arg-file: Read input from file instead of stdin' seq 1 10 >/tmp/$$ -$PAR -k -a /tmp/$$ echo -$PAR -k --arg-file /tmp/$$ echo +parallel -k -a /tmp/$$ echo +parallel -k --arg-file /tmp/$$ echo cd input-files/test15 @@ -86,67 +96,71 @@ echo 'parallel Expect: 1 3 2' echo 3 | parallel -k -I {} -P 1 -n 1 -a files cat {} - echo '### Test -i and --replace: Replace with argument' -(echo a; echo END; echo b) | $PAR -k -i -eEND echo repl{}ce -(echo a; echo END; echo b) | $PAR -k --replace -eEND echo repl{}ce -(echo a; echo END; echo b) | $PAR -k -i+ -eEND echo repl+ce -(echo e; echo END; echo b) | $PAR -k -i'*' -eEND echo r'*'plac'*' -(echo a; echo END; echo b) | $PAR -k --replace + -eEND echo repl+ce -(echo a; echo END; echo b) | $PAR -k --replace== -eEND echo repl=ce -(echo a; echo END; echo b) | $PAR -k --replace = -eEND echo repl=ce -(echo a; echo END; echo b) | $PAR -k --replace=^ -eEND echo repl^ce -(echo a; echo END; echo b) | $PAR -k -I^ -eEND echo repl^ce +(echo a; echo END; echo b) | parallel -k -i -eEND echo repl{}ce +(echo a; echo END; echo b) | parallel -k --replace -eEND echo repl{}ce +(echo a; echo END; echo b) | parallel -k -i+ -eEND echo repl+ce +(echo e; echo END; echo b) | parallel -k -i'*' -eEND echo r'*'plac'*' +(echo a; echo END; echo b) | parallel -k --replace + -eEND echo repl+ce +(echo a; echo END; echo b) | parallel -k --replace== -eEND echo repl=ce +(echo a; echo END; echo b) | parallel -k --replace = -eEND echo repl=ce +(echo a; echo END; echo b) | parallel -k --replace=^ -eEND echo repl^ce +(echo a; echo END; echo b) | parallel -k -I^ -eEND echo repl^ce echo '### Test -E: Artificial end-of-file' -(echo include this; echo END; echo not this) | $PAR -k -E END echo -(echo include this; echo END; echo not this) | $PAR -k -EEND echo +(echo include this; echo END; echo not this) | parallel -k -E END echo +(echo include this; echo END; echo not this) | parallel -k -EEND echo echo '### Test -e and --eof: Artificial end-of-file' -(echo include this; echo END; echo not this) | $PAR -k -e END echo -(echo include this; echo END; echo not this) | $PAR -k -eEND echo -(echo include this; echo END; echo not this) | $PAR -k --eof=END echo -(echo include this; echo END; echo not this) | $PAR -k --eof END echo +(echo include this; echo END; echo not this) | parallel -k -e END echo +(echo include this; echo END; echo not this) | parallel -k -eEND echo +(echo include this; echo END; echo not this) | parallel -k --eof=END echo +(echo include this; echo END; echo not this) | parallel -k --eof END echo echo '### Test -n and --max-args: Max number of args per line (only with -X and -m)' -(echo line 1;echo line 2;echo line 3) | $PAR -k -n1 -m echo -(echo line 1;echo line 1;echo line 2) | $PAR -k -n2 -m echo -(echo line 1;echo line 2;echo line 3) | $PAR -k -n1 -X echo -(echo line 1;echo line 1;echo line 2) | $PAR -k -n2 -X echo -(echo line 1;echo line 2;echo line 3) | $PAR -k --max-args=1 -X echo -(echo line 1;echo line 2;echo line 3) | $PAR -k --max-args 1 -X echo -(echo line 1;echo line 1;echo line 2) | $PAR -k --max-args=2 -X echo -(echo line 1;echo line 1;echo line 2) | $PAR -k --max-args 2 -X echo +(echo line 1;echo line 2;echo line 3) | parallel -k -n1 -m echo +(echo line 1;echo line 1;echo line 2) | parallel -k -n2 -m echo +(echo line 1;echo line 2;echo line 3) | parallel -k -n1 -X echo +(echo line 1;echo line 1;echo line 2) | parallel -k -n2 -X echo +(echo line 1;echo line 2;echo line 3) | parallel -k -n1 echo +(echo line 1;echo line 1;echo line 2) | parallel -k -n2 echo +(echo line 1;echo line 2;echo line 3) | parallel -k --max-args=1 -X echo +(echo line 1;echo line 2;echo line 3) | parallel -k --max-args 1 -X echo +(echo line 1;echo line 1;echo line 2) | parallel -k --max-args=2 -X echo +(echo line 1;echo line 1;echo line 2) | parallel -k --max-args 2 -X echo +(echo line 1;echo line 2;echo line 3) | parallel -k --max-args 1 echo +(echo line 1;echo line 1;echo line 2) | parallel -k --max-args 2 echo echo '### Test --max-procs and -P: Number of processes' -seq 1 10 | $PAR -k --max-procs +0 echo max proc -seq 1 10 | $PAR -k -P 200% echo 200% proc +seq 1 10 | parallel -k --max-procs +0 echo max proc +seq 1 10 | parallel -k -P 200% echo 200% proc echo '### Test --delimiter and -d: Delimiter instead of newline' echo '# Yes there is supposed to be an extra newline for -d N' -echo line 1Nline 2Nline 3 | $PAR -k -d N echo This is -echo line 1Nline 2Nline 3 | $PAR -k --delimiter N echo This is -printf "delimiter NUL line 1\0line 2\0line 3" | $PAR -k -d '\0' echo -printf "delimiter TAB line 1\tline 2\tline 3" | $PAR -k --delimiter '\t' echo +echo line 1Nline 2Nline 3 | parallel -k -d N echo This is +echo line 1Nline 2Nline 3 | parallel -k --delimiter N echo This is +printf "delimiter NUL line 1\0line 2\0line 3" | parallel -k -d '\0' echo +printf "delimiter TAB line 1\tline 2\tline 3" | parallel -k --delimiter '\t' echo echo '### Test --max-chars and -s: Max number of chars in a line' -(echo line 1;echo line 1;echo line 2) | $PAR -k --max-chars 25 -X echo -(echo line 1;echo line 1;echo line 2) | $PAR -k -s 25 -X echo +(echo line 1;echo line 1;echo line 2) | parallel -k --max-chars 25 -X echo +(echo line 1;echo line 1;echo line 2) | parallel -k -s 25 -X echo echo '### Test --no-run-if-empty and -r: This should give no output' -echo " " | $PAR -r echo -echo " " | $PAR --no-run-if-empty echo +echo " " | parallel -r echo +echo " " | parallel --no-run-if-empty echo echo '### Test --help and -h: Help output (just check we get the same amount of lines)' echo Output from -h and --help -$PAR -h | wc -l -$PAR --help | wc -l +parallel -h | wc -l +parallel --help | wc -l echo '### Test --version: Version output (just check we get the same amount of lines)' -$PAR --version | wc -l +parallel --version | wc -l echo '### Test --verbose and -t' -(echo b; echo c; echo f) | $PAR -k -t echo {}ar 2>&1 >/dev/null -(echo b; echo c; echo f) | $PAR -k --verbose echo {}ar 2>&1 >/dev/null +(echo b; echo c; echo f) | parallel -k -t echo {}ar 2>&1 >/dev/null +(echo b; echo c; echo f) | parallel -k --verbose echo {}ar 2>&1 >/dev/null echo '### Test --show-limits' -(echo b; echo c; echo f) | $PAR -k --show-limits echo {}ar -(echo b; echo c; echo f) | $PAR -k --show-limits -s 100 echo {}ar +(echo b; echo c; echo f) | parallel -k --show-limits echo {}ar +(echo b; echo c; echo f) | parallel -k --show-limits -s 100 echo {}ar diff --git a/unittest/tests-to-run/test24.sh b/unittest/tests-to-run/test24.sh old mode 100644 new mode 100755 index 5d3717da..75481b6b --- a/unittest/tests-to-run/test24.sh +++ b/unittest/tests-to-run/test24.sh @@ -2,3 +2,14 @@ echo '### 64-bit wierdness - this did not complete on a 64-bit machine' seq 1 2 | parallel -j1 'seq 1 1 | parallel true' + +echo "### BUG-fix: bash -c 'parallel -a <(seq 1 3) echo'" +stdout bash -c 'parallel -k -a <(seq 1 3) echo' + +echo "### BUG: The length for -X is not close to max (131072)" +seq 1 60000 | parallel -X echo {.} aa {}{.} {}{}d{} {}dd{}d{.} |head -n 1 |wc +seq 1 60000 | parallel -X echo a{}b{}c |head -n 1 |wc +seq 1 60000 | parallel -X echo |head -n 1 |wc +seq 1 60000 | parallel -X echo a{}b{}c {} |head -n 1 |wc +seq 1 60000 | parallel -X echo {}aa{} |head -n 1 |wc +seq 1 60000 | parallel -X echo {} aa {} |head -n 1 |wc diff --git a/unittest/wanted-results/test15 b/unittest/wanted-results/test15 index 114fc295..83135340 100644 --- a/unittest/wanted-results/test15 +++ b/unittest/wanted-results/test15 @@ -13,6 +13,9 @@ opt--interactive 3 ### Test -L -l and --max-lines a_b c a_b c +a_b c +a_b +c a_b c a_b @@ -21,15 +24,9 @@ a_b c d a_b c d -a_b c d -e -a_b c d -e -a_b c -d -e a_b c d +a_b c d e a_b c d e @@ -41,10 +38,33 @@ e a_b c d e +a_b c +d +e a_b c d e a_b c d e +a_b c d +e +a_b c +d +e +a_b c +d +e +a_b c +d +e +a_b c d +e +a_b c d +e +a_b c d +e +a_b c +d +e a_b c d e @@ -66,6 +86,12 @@ Command line too long (1000007 >= 10) at number 1: zzzzzzzzzzzzzzzzzzzzzzzzzzzzz 7 8 xargs: argument line too long 9 10 +1 2 +3 4 +5 6 +7 8 +Command line too long (1000007 >= 10) at number 1: zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz... +9 10 ### Test -x 1 2 3 4 @@ -75,6 +101,11 @@ Command line too long (15 >= 10) at number 3: 12345... 1 2 3 4 5 6 +Command line too long (15 >= 10) at number 3: 12345... +7 8 +1 2 +3 4 +5 6 xargs: argument line too long 7 8 1 2 @@ -97,6 +128,16 @@ xargs: argument line too long 13 14 15 +1 2 +3 4 +5 6 +7 8 +9 10 +1234 +12 +13 +14 +15 ### Test bugfix if no command given Command line too long (1000002 >= 10) at number 1: zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz... 1 2 3 4 5 @@ -169,6 +210,11 @@ line 2 line 1 line 2 line 3 +line 1 line 1 +line 2 +line 1 +line 2 +line 3 line 1 line 2 line 3 @@ -176,6 +222,11 @@ line 1 line 1 line 2 line 1 line 1 line 2 +line 1 +line 2 +line 3 +line 1 line 1 +line 2 ### Test --max-procs and -P: Number of processes max proc 1 max proc 2 @@ -248,6 +299,4 @@ Maximal used size of command: 100 Execution of will continue now, and it will try to read its input and run commands; if this is not what you wanted to happen, please press CTRL-D or CTRL-C -bar -car -far +bar car far diff --git a/unittest/wanted-results/test24 b/unittest/wanted-results/test24 index 8434e4ad..740f1f87 100644 --- a/unittest/wanted-results/test24 +++ b/unittest/wanted-results/test24 @@ -1 +1,12 @@ ### 64-bit wierdness - this did not complete on a 64-bit machine +### BUG-fix: bash -c 'parallel -a <(seq 1 3) echo' +1 +2 +3 +### BUG: The length for -X is not close to max (131072) + 1 12821 131060 + 1 10948 131060 + 1 23695 131064 + 1 15810 131064 + 1 11790 131058 + 1 25545 131055