Bugfix: The length for -X is not close to max (131072)

doc/FUTURE_IDEAS

@@ -1,5 +1,3 @@
-# BUG: bash -c 'parallel -a <(seq 1 3) -p  echo'
-
 # Hvordan udregnes system limits på remote systems hvis jeg ikke ved, hvormange
 # argumenter, der er? Lav system limits lokalt og lad det være max
 
@@ -16,29 +14,6 @@
 # Clustering Tools | Command Line Tools | Utilities | System Administration
 # Bash parallel
 
-* 100% options complete with xargs. All options for xargs can now
-  be used in GNU Parallel - even the more exotic.
-
-* --basefile for transfering basedata. When running jobs on remote
-  computers --basefile will transfer files before the first jobs is
-  run. It can be used to transfer data that remains the same for each
-  job such as scripts or lookup tables.
-
-* --progress shows progress. To see how many jobs is running on each
-  server use --progress. It can be turned on even after GNU Parallel
-  is started.
-
-* --eta shows estimated time left in seconds.
-
-* --halt-on-error stops if an error occurs. GNU Parallel will default
-  to run all jobs - even if some of them fail. With --halt-on-error
-  GNU Parallel can ignore errors, wait for the currently running jobs
-  to finish, or stop immediately when an error occurs.
-
-* New video showing the new options.
-
-
-
 =head1 YouTube video
 
 GNU Parallel is a tool with lots of uses in shell. Every time you use

doc/release_new_version

@@ -67,19 +67,41 @@ http://freshmeat.net/projects/parallel/releases/new
 
 == Send announce ==
 
+http://groups.google.com/group/comp.unix.shell/post
 Newsgroups: comp.unix.shell,comp.unix.admin
 
 <<<<<
 to:parallel@gnu.org, bug-parallel@gnu.org, info-gnu@gnu.org, bug-directory@gnu.org
 
-Subject: GNU Parallel 20100601 released
+Subject: GNU Parallel 20100620 released
 
-GNU Parallel 20100601 has been released today. It is available for
+GNU Parallel 20100620 has been released. It is available for
 download at: http://ftp.gnu.org/gnu/parallel/
 
 New in this release:
 
-* GNU Parallel now has progress indicator
+* New video showing the new options.
+  http://www.youtube.com/watch?v=OpaiGYxkSuQ
+
+* 100% options complete with xargs. All options for xargs can now
+  be used in GNU Parallel - even the more exotic.
+
+* --basefile for transferring basedata. When running jobs on remote
+  computers --basefile will transfer files before the first jobs is
+  run. It can be used to transfer data that remains the same for each
+  job such as scripts or lookup tables.
+
+* --progress shows progress. To see how many jobs is running on each
+  server use --progress. It can be turned on even after GNU Parallel
+  is started.
+
+* --eta shows estimated time left in seconds.
+
+* --halt-on-error stops if an error occurs. GNU Parallel will default
+  to run all jobs - even if some of them fail. With --halt-on-error
+  GNU Parallel can ignore errors, wait for the currently running jobs
+  to finish, or stop immediately when an error occurs.
+
 
 = About GNU Parallel =
 
@@ -103,6 +125,6 @@ possible to use output from GNU Parallel as input for other programs.
 You can find more about GNU Parallel at:
 http://www.gnu.org/software/parallel/
 
-Watch the intro video on http://www.youtube.com/watch?v=LlXDtd_pRaY
+Watch the intro video on http://www.youtube.com/watch?v=OpaiGYxkSuQ
 
 >>>>>

src/parallel

@@ -1908,6 +1908,10 @@ sub xargs_computations {
 	     $spaces,$length_of_command_no_args,$length_of_context)
 	    = (1,0,0,0,0);
 	if($command) {
+	    if($command !~ /\s\S*\Q$Global::replacestring\E\S*|\s\S*\Q$Global::replace_no_ext\E\S*/o) {
+		# No replacement strings: add {}
+		$command .= " ".$Global::replacestring;
+	    }
 	    # Count number of {}'s on the command line
 	    my $no_of_replace =
 		($command =~ s/\Q$Global::replacestring\E/$Global::replacestring/go);
@@ -1919,13 +1923,15 @@ sub xargs_computations {
 	    # Count
 	    my $c = $command;
 	    if($Global::Xargs) {
-		$c =~ s/\S*\Q$Global::replacestring\E\S*//go;
+		$c =~ s/\s\S*\Q$Global::replacestring\E\S*|\s\S*\Q$Global::replace_no_ext\E\S*//go;
-		$c =~ s/\S*\Q$Global::replace_no_ext\E\S*//go;
+		$length_of_command_no_args = length($c);
-		$length_of_command_no_args = length($c) - 1;
 		$length_of_context = length($command) - $length_of_command_no_args
 		    - $no_of_replace * length($Global::replacestring)
 		    - $no_of_no_ext * length($Global::replace_no_ext);
 		$spaces = 0;
+		debug("length_of_command_no_args $length_of_command_no_args\n");
+		debug("length_of_context $length_of_context\n");
+		debug("no_of_replace $no_of_replace no_of_no_ext $no_of_no_ext\n");
 	    } else {
 		# remove all {}s
 		$c =~ s/\Q$Global::replacestring\E|\Q$Global::replace_no_ext\E//og;
@@ -3321,4 +3327,3 @@ $main::opt_version = $main::opt_L = $main::opt_l =
 $main::opt_show_limits = $main::opt_n = $main::opt_e = $main::opt_verbose =
 $main::opt_E = $main::opt_r = $Global::xargs = $Global::keeporder =
 $Global::control_path = 0;
-

unittest/Makefile

@@ -6,7 +6,8 @@ unittest: ../src/parallel tests-to-run/* wanted-results/*
 	ren 2>&1 | mop || (echo ren is required for unittest; /bin/false)
 	echo | buffer | mop || (echo buffer is required for unittest; /bin/false)
 	echo 1+2 | bc | mop || (echo bc is required for unittest; /bin/false)
-	stdout gawk | mop || (echo gawk is required for parallel; /bin/false)
+	stdout gawk | mop || (echo gawk is required for unittest; /bin/false)
+	expect -c 'spawn cat; puts "expect is installed"' || (echo expect is required for unittest; /bin/false)
 	sh Start.sh
 
 clean:

unittest/Start.sh

@@ -7,6 +7,7 @@ ls -t tests-to-run/test*.sh \
 | perl -pe 's:(.*/(.*)).sh:sh $1.sh > actual-results/$2; diff -Naur wanted-results/$2 actual-results/$2:' \
 >$SHFILE
 
+mkdir -p actual-results
 sh -x $SHFILE
 rm $SHFILE
 

unittest/tests-to-run/test15.sh

@@ -2,8 +2,6 @@
 
 # Test xargs compatibility
 
-PAR=parallel
-
 echo '### Test -p --interactive'
 cat >/tmp/parallel-script-for-expect <<_EOF
 #!/bin/bash
@@ -36,22 +34,31 @@ _EOF
 
 echo '### Test -L -l and --max-lines'
 (echo a_b;echo c) | parallel -km -L2  echo
+(echo a_b;echo c) | parallel -k -L2  echo
 (echo a_b;echo c) | xargs -L2  echo
 (echo a_b;echo c) | parallel -km -L1  echo
+(echo a_b;echo c) | parallel -k -L1  echo
 (echo a_b;echo c) | xargs -L1  echo
 (echo a_b' ';echo c;echo d) | parallel -km -L1  echo
+(echo a_b' ';echo c;echo d) | parallel -k -L1  echo
 (echo a_b' ';echo c;echo d) | xargs -L1  echo
 (echo a_b' ';echo c;echo d;echo e) | parallel -km -L2 echo
+(echo a_b' ';echo c;echo d;echo e) | parallel -k -L2 echo
 (echo a_b' ';echo c;echo d;echo e) | xargs -L2 echo
 (echo a_b' ';echo c;echo d;echo e) | parallel -km -l echo
+(echo a_b' ';echo c;echo d;echo e) | parallel -k -l echo
 (echo a_b' ';echo c;echo d;echo e) | xargs -l echo
 (echo a_b' ';echo c;echo d;echo e) | parallel -km -l2 echo
+(echo a_b' ';echo c;echo d;echo e) | parallel -k -l2 echo
 (echo a_b' ';echo c;echo d;echo e) | xargs -l2 echo
 (echo a_b' ';echo c;echo d;echo e) | parallel -km -l1 echo
+(echo a_b' ';echo c;echo d;echo e) | parallel -k -l1 echo
 (echo a_b' ';echo c;echo d;echo e) | xargs -l1 echo
 (echo a_b' ';echo c;echo d;echo e) | parallel -km --max-lines 2 echo
+(echo a_b' ';echo c;echo d;echo e) | parallel -k --max-lines 2 echo
 (echo a_b' ';echo c;echo d;echo e) | xargs --max-lines=2 echo
 (echo a_b' ';echo c;echo d;echo e) | parallel -km --max-lines echo
+(echo a_b' ';echo c;echo d;echo e) | parallel -k --max-lines echo
 (echo a_b' ';echo c;echo d;echo e) | xargs --max-lines echo
 
 echo '### test too long args'
@@ -59,10 +66,13 @@ perl -e 'print "z"x1000000' | parallel echo 2>&1
 perl -e 'print "z"x1000000' | xargs echo 2>&1
 (seq 1 10; perl -e 'print "z"x1000000'; seq 12 15) | stdout parallel -j1 -km -s 10 echo
 (seq 1 10; perl -e 'print "z"x1000000'; seq 12 15) | stdout xargs -s 10 echo
+(seq 1 10; perl -e 'print "z"x1000000'; seq 12 15) | stdout parallel -j1 -k -s 10 echo
 echo '### Test -x'
 (seq 1 10; echo 12345; seq 12 15) | stdout parallel -j1 -km -s 10 -x echo
+(seq 1 10; echo 12345; seq 12 15) | stdout parallel -j1 -k -s 10 -x echo
 (seq 1 10; echo 12345; seq 12 15) | stdout xargs -s 10 -x echo
 (seq 1 10; echo 1234; seq 12 15) | stdout parallel -j1 -km -s 10 -x echo
+(seq 1 10; echo 1234; seq 12 15) | stdout parallel -j1 -k -s 10 -x echo
 (seq 1 10; echo 1234; seq 12 15) | stdout xargs -s 10 -x echo
 echo '### Test bugfix if no command given'
 (echo echo; seq 1 5; perl -e 'print "z"x1000000'; seq 12 15) | stdout parallel -j1 -km -s 10
@@ -71,8 +81,8 @@ echo '### Test bugfix if no command given'
 
 echo '### Test -a and --arg-file: Read input from file instead of stdin'
 seq 1 10 >/tmp/$$
-$PAR -k -a /tmp/$$ echo
+parallel -k -a /tmp/$$ echo
-$PAR -k --arg-file /tmp/$$ echo
+parallel -k --arg-file /tmp/$$ echo
 
 cd input-files/test15
 
@@ -86,67 +96,71 @@ echo 'parallel Expect: 1 3 2'
 echo 3 | parallel -k -I {} -P 1 -n 1 -a files cat {} -
 
 echo '### Test -i and --replace: Replace with argument'
-(echo a; echo END; echo b) | $PAR -k -i -eEND echo repl{}ce
+(echo a; echo END; echo b) | parallel -k -i -eEND echo repl{}ce
-(echo a; echo END; echo b) | $PAR -k --replace -eEND echo repl{}ce
+(echo a; echo END; echo b) | parallel -k --replace -eEND echo repl{}ce
-(echo a; echo END; echo b) | $PAR -k -i+ -eEND echo repl+ce
+(echo a; echo END; echo b) | parallel -k -i+ -eEND echo repl+ce
-(echo e; echo END; echo b) | $PAR -k -i'*' -eEND echo r'*'plac'*'
+(echo e; echo END; echo b) | parallel -k -i'*' -eEND echo r'*'plac'*'
-(echo a; echo END; echo b) | $PAR -k --replace + -eEND echo repl+ce
+(echo a; echo END; echo b) | parallel -k --replace + -eEND echo repl+ce
-(echo a; echo END; echo b) | $PAR -k --replace== -eEND echo repl=ce
+(echo a; echo END; echo b) | parallel -k --replace== -eEND echo repl=ce
-(echo a; echo END; echo b) | $PAR -k --replace = -eEND echo repl=ce
+(echo a; echo END; echo b) | parallel -k --replace = -eEND echo repl=ce
-(echo a; echo END; echo b) | $PAR -k --replace=^ -eEND echo repl^ce
+(echo a; echo END; echo b) | parallel -k --replace=^ -eEND echo repl^ce
-(echo a; echo END; echo b) | $PAR -k -I^ -eEND echo repl^ce
+(echo a; echo END; echo b) | parallel -k -I^ -eEND echo repl^ce
 
 echo '### Test -E: Artificial end-of-file'
-(echo include this; echo END; echo not this) | $PAR -k -E END echo
+(echo include this; echo END; echo not this) | parallel -k -E END echo
-(echo include this; echo END; echo not this) | $PAR -k -EEND echo
+(echo include this; echo END; echo not this) | parallel -k -EEND echo
 
 echo '### Test -e and --eof: Artificial end-of-file'
-(echo include this; echo END; echo not this) | $PAR -k -e END echo
+(echo include this; echo END; echo not this) | parallel -k -e END echo
-(echo include this; echo END; echo not this) | $PAR -k -eEND echo
+(echo include this; echo END; echo not this) | parallel -k -eEND echo
-(echo include this; echo END; echo not this) | $PAR -k --eof=END echo
+(echo include this; echo END; echo not this) | parallel -k --eof=END echo
-(echo include this; echo END; echo not this) | $PAR -k --eof END echo
+(echo include this; echo END; echo not this) | parallel -k --eof END echo
 
 echo '### Test -n and --max-args: Max number of args per line (only with -X and -m)'
-(echo line 1;echo line 2;echo line 3) | $PAR -k -n1 -m echo
+(echo line 1;echo line 2;echo line 3) | parallel -k -n1 -m echo
-(echo line 1;echo line 1;echo line 2) | $PAR -k -n2 -m echo
+(echo line 1;echo line 1;echo line 2) | parallel -k -n2 -m echo
-(echo line 1;echo line 2;echo line 3) | $PAR -k -n1 -X echo
+(echo line 1;echo line 2;echo line 3) | parallel -k -n1 -X echo
-(echo line 1;echo line 1;echo line 2) | $PAR -k -n2 -X echo
+(echo line 1;echo line 1;echo line 2) | parallel -k -n2 -X echo
-(echo line 1;echo line 2;echo line 3) | $PAR -k --max-args=1 -X echo
+(echo line 1;echo line 2;echo line 3) | parallel -k -n1 echo
-(echo line 1;echo line 2;echo line 3) | $PAR -k --max-args 1 -X echo
+(echo line 1;echo line 1;echo line 2) | parallel -k -n2 echo
-(echo line 1;echo line 1;echo line 2) | $PAR -k --max-args=2 -X echo
+(echo line 1;echo line 2;echo line 3) | parallel -k --max-args=1 -X echo
-(echo line 1;echo line 1;echo line 2) | $PAR -k --max-args 2 -X echo
+(echo line 1;echo line 2;echo line 3) | parallel -k --max-args 1 -X echo
+(echo line 1;echo line 1;echo line 2) | parallel -k --max-args=2 -X echo
+(echo line 1;echo line 1;echo line 2) | parallel -k --max-args 2 -X echo
+(echo line 1;echo line 2;echo line 3) | parallel -k --max-args 1 echo
+(echo line 1;echo line 1;echo line 2) | parallel -k --max-args 2 echo
 
 echo '### Test --max-procs and -P: Number of processes'
-seq 1 10 | $PAR -k --max-procs +0 echo max proc
+seq 1 10 | parallel -k --max-procs +0 echo max proc
-seq 1 10 | $PAR -k -P 200% echo 200% proc
+seq 1 10 | parallel -k -P 200% echo 200% proc
 
 echo '### Test --delimiter and -d: Delimiter instead of newline'
 echo '# Yes there is supposed to be an extra newline for -d N'
-echo line 1Nline 2Nline 3 | $PAR -k -d N echo This is
+echo line 1Nline 2Nline 3 | parallel -k -d N echo This is
-echo line 1Nline 2Nline 3 | $PAR -k --delimiter N echo This is
+echo line 1Nline 2Nline 3 | parallel -k --delimiter N echo This is
-printf "delimiter NUL line 1\0line 2\0line 3" | $PAR -k -d '\0' echo
+printf "delimiter NUL line 1\0line 2\0line 3" | parallel -k -d '\0' echo
-printf "delimiter TAB line 1\tline 2\tline 3" | $PAR -k --delimiter '\t' echo
+printf "delimiter TAB line 1\tline 2\tline 3" | parallel -k --delimiter '\t' echo
 
 echo '### Test --max-chars and -s: Max number of chars in a line'
-(echo line 1;echo line 1;echo line 2) | $PAR -k --max-chars 25 -X echo
+(echo line 1;echo line 1;echo line 2) | parallel -k --max-chars 25 -X echo
-(echo line 1;echo line 1;echo line 2) | $PAR -k -s 25 -X echo
+(echo line 1;echo line 1;echo line 2) | parallel -k -s 25 -X echo
 
 echo '### Test --no-run-if-empty and -r: This should give no output'
-echo "  " | $PAR -r echo
+echo "  " | parallel -r echo
-echo "  " | $PAR --no-run-if-empty echo
+echo "  " | parallel --no-run-if-empty echo
 
 echo '### Test --help and -h: Help output (just check we get the same amount of lines)'
 echo Output from -h and --help
-$PAR -h | wc -l
+parallel -h | wc -l
-$PAR --help | wc -l
+parallel --help | wc -l
 
 echo '### Test --version: Version output (just check we get the same amount of lines)'
-$PAR --version | wc -l
+parallel --version | wc -l
 
 echo '### Test --verbose and -t'
-(echo b; echo c; echo f) | $PAR -k -t echo {}ar 2>&1 >/dev/null
+(echo b; echo c; echo f) | parallel -k -t echo {}ar 2>&1 >/dev/null
-(echo b; echo c; echo f) | $PAR -k --verbose echo {}ar 2>&1 >/dev/null
+(echo b; echo c; echo f) | parallel -k --verbose echo {}ar 2>&1 >/dev/null
 
 echo '### Test --show-limits'
-(echo b; echo c; echo f) | $PAR -k --show-limits echo {}ar
+(echo b; echo c; echo f) | parallel -k --show-limits echo {}ar
-(echo b; echo c; echo f) | $PAR -k --show-limits -s 100 echo {}ar
+(echo b; echo c; echo f) | parallel -k --show-limits -s 100 echo {}ar

unittest/tests-to-run/test24.sh

@@ -2,3 +2,14 @@
 
 echo '### 64-bit wierdness - this did not complete on a 64-bit machine'
 seq 1 2 | parallel -j1 'seq 1 1 | parallel true'
+
+echo "### BUG-fix: bash -c 'parallel -a <(seq 1 3) echo'"
+stdout bash -c 'parallel -k -a <(seq 1 3)  echo'
+
+echo "### BUG: The length for -X is not close to max (131072)"
+seq 1 60000 | parallel -X echo {.} aa {}{.} {}{}d{} {}dd{}d{.} |head -n 1 |wc
+seq 1 60000 | parallel -X echo a{}b{}c |head -n 1 |wc
+seq 1 60000 | parallel -X echo |head -n 1 |wc
+seq 1 60000 | parallel -X echo a{}b{}c {} |head -n 1 |wc
+seq 1 60000 | parallel -X echo {}aa{} |head -n 1 |wc
+seq 1 60000 | parallel -X echo {} aa {} |head -n 1 |wc

unittest/wanted-results/test15

@@ -13,6 +13,9 @@ opt--interactive 3
 ### Test -L -l and --max-lines
 a_b c
 a_b c
+a_b c
+a_b
+c
 a_b
 c
 a_b
@@ -21,15 +24,9 @@ a_b c
 d
 a_b c
 d
-a_b c d
-e
-a_b c d
-e
-a_b c
-d
-e
 a_b c
 d
+a_b c d
 e
 a_b c d
 e
@@ -41,10 +38,33 @@ e
 a_b c
 d
 e
+a_b c
+d
+e
 a_b c d
 e
 a_b c d
 e
+a_b c d
+e
+a_b c
+d
+e
+a_b c
+d
+e
+a_b c
+d
+e
+a_b c d
+e
+a_b c d
+e
+a_b c d
+e
+a_b c
+d
+e
 a_b c
 d
 e
@@ -66,6 +86,12 @@ Command line too long (1000007 >= 10) at number 1: zzzzzzzzzzzzzzzzzzzzzzzzzzzzz
 7 8
 xargs: argument line too long
 9 10
+1 2
+3 4
+5 6
+7 8
+Command line too long (1000007 >= 10) at number 1: zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz...
+9 10
 ### Test -x
 1 2
 3 4
@@ -75,6 +101,11 @@ Command line too long (15 >= 10) at number 3: 12345...
 1 2
 3 4
 5 6
+Command line too long (15 >= 10) at number 3: 12345...
+7 8
+1 2
+3 4
+5 6
 xargs: argument line too long
 7 8
 1 2
@@ -97,6 +128,16 @@ xargs: argument line too long
 13
 14
 15
+1 2
+3 4
+5 6
+7 8
+9 10
+1234
+12
+13
+14
+15
 ### Test bugfix if no command given
 Command line too long (1000002 >= 10) at number 1: zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz...
 1 2 3 4 5
@@ -169,6 +210,11 @@ line 2
 line 1
 line 2
 line 3
+line 1 line 1
+line 2
+line 1
+line 2
+line 3
 line 1
 line 2
 line 3
@@ -176,6 +222,11 @@ line 1 line 1
 line 2
 line 1 line 1
 line 2
+line 1
+line 2
+line 3
+line 1 line 1
+line 2
 ### Test --max-procs and -P: Number of processes
 max proc 1
 max proc 2
@@ -248,6 +299,4 @@ Maximal used size of command: 100
 Execution of  will continue now, and it will try to read its input
 and run commands; if this is not what you wanted to happen, please
 press CTRL-D or CTRL-C
-bar
+bar car far
-car
-far

unittest/wanted-results/test24

@@ -1 +1,12 @@
 ### 64-bit wierdness - this did not complete on a 64-bit machine
+### BUG-fix: bash -c 'parallel -a <(seq 1 3) echo'
+1
+2
+3
+### BUG: The length for -X is not close to max (131072)
+      1   12821  131060
+      1   10948  131060
+      1   23695  131064
+      1   15810  131064
+      1   11790  131058
+      1   25545  131055