diff --git a/src/parallel b/src/parallel index f369c5c7..02d5f588 100755 --- a/src/parallel +++ b/src/parallel @@ -930,7 +930,7 @@ sub __QUOTING_ARGUMENTS_FOR_SHELL__ {} sub shell_quote { my @strings = (@_); for my $a (@strings) { - $a =~ s/([\002-\011\013-\032\\\#\?\`\(\)\{\}\*\>\<\~\|\; \"\!\$\&\'])/\\$1/g; + $a =~ s/([\002-\011\013-\032\\\#\?\`\(\)\{\}\[\]\*\>\<\~\|\; \"\!\$\&\'])/\\$1/g; $a =~ s/[\n]/'\n'/g; # filenames with '\n' is quoted using \' } return wantarray ? @strings : "@strings"; @@ -941,7 +941,7 @@ sub shell_quote_scalar { # Returns: # string quoted with \ as needed by the shell my $a = shift; - $a =~ s/([\002-\011\013-\032\\\#\?\`\(\)\{\}\*\>\<\~\|\; \"\!\$\&\'])/\\$1/g; + $a =~ s/([\002-\011\013-\032\\\#\?\`\(\)\{\}\[\]\*\>\<\~\|\; \"\!\$\&\'])/\\$1/g; $a =~ s/[\n]/'\n'/g; # filenames with '\n' is quoted using \' return $a; } diff --git a/testsuite/tests-to-run/test24.sh b/testsuite/tests-to-run/test24.sh index 75481b6b..05a7907c 100755 --- a/testsuite/tests-to-run/test24.sh +++ b/testsuite/tests-to-run/test24.sh @@ -1,15 +1,2 @@ #!/bin/bash -echo '### 64-bit wierdness - this did not complete on a 64-bit machine' -seq 1 2 | parallel -j1 'seq 1 1 | parallel true' - -echo "### BUG-fix: bash -c 'parallel -a <(seq 1 3) echo'" -stdout bash -c 'parallel -k -a <(seq 1 3) echo' - -echo "### BUG: The length for -X is not close to max (131072)" -seq 1 60000 | parallel -X echo {.} aa {}{.} {}{}d{} {}dd{}d{.} |head -n 1 |wc -seq 1 60000 | parallel -X echo a{}b{}c |head -n 1 |wc -seq 1 60000 | parallel -X echo |head -n 1 |wc -seq 1 60000 | parallel -X echo a{}b{}c {} |head -n 1 |wc -seq 1 60000 | parallel -X echo {}aa{} |head -n 1 |wc -seq 1 60000 | parallel -X echo {} aa {} |head -n 1 |wc diff --git a/testsuite/tests-to-run/test25.sh b/testsuite/tests-to-run/test25.sh index 467526e2..51e1b33c 100644 --- a/testsuite/tests-to-run/test25.sh +++ b/testsuite/tests-to-run/test25.sh @@ -1,15 +1,5 @@ #!/bin/bash -echo '### Test basic --arg-sep' -parallel -k echo ::: a b -echo '### Run commands using --arg-sep' -parallel -kv ::: 'echo a' 'echo b' -echo '### Change --arg-sep' -parallel --arg-sep ::: -kv ::: 'echo a' 'echo b' -parallel --arg-sep .--- -kv .--- 'echo a' 'echo b' -parallel --argsep ::: -kv ::: 'echo a' 'echo b' -parallel --argsep .--- -kv .--- 'echo a' 'echo b' - echo '### Test stdin goes to first command only ("-" as argument)' cat >/tmp/parallel-script-for-script <' | parallel -v echo -parallel -v echo ::: '>' -(echo '>'; echo 2) | parallel -j1 -vX echo -parallel -X -j1 echo ::: '>' 2 -echo '### Must not quote' -echo 'echo | wc -l' | parallel -v -parallel -v ::: 'echo | wc -l' -echo 'echo a b c | wc -w' | parallel -v -parallel -kv ::: 'echo a b c | wc -w' 'echo a b | wc -w' diff --git a/testsuite/tests-to-run/test28.sh b/testsuite/tests-to-run/test28.sh index dd28c359..947a80fd 100644 --- a/testsuite/tests-to-run/test28.sh +++ b/testsuite/tests-to-run/test28.sh @@ -1,9 +1,2 @@ #!/bin/bash -x -rsync -Ha --delete input-files/segfault/ tmp/ -cd tmp - -echo '### Test of segfaulting issue' -echo 'This gave /home/tange/bin/stdout: line 3: 20374 Segmentation fault "$@" 2>&1' -echo 'before adding wait() before exit' -seq 1 300 | stdout parallel ./trysegfault \ No newline at end of file diff --git a/testsuite/tests-to-run/test30.sh b/testsuite/tests-to-run/test30.sh index 712b1f89..a5fef034 100644 --- a/testsuite/tests-to-run/test30.sh +++ b/testsuite/tests-to-run/test30.sh @@ -18,7 +18,7 @@ echo '### bug #34422: parallel -X --eta crashes with div by zero' seq 2 | stdout parallel -X --eta echo echo '### --timeout on remote machines' -parallel -j0 --timeout 3 --onall -S .. 'sleep {}; echo slept {}' ::: 1 8 9 ; echo jobs failed: $? +parallel -j0 --timeout 3 --onall -S localhost,parallel@parallel-server1 'sleep {}; echo slept {}' ::: 1 8 9 ; echo jobs failed: $? echo '### --pipe without command' seq -w 10 | stdout parallel --pipe diff --git a/testsuite/tests-to-run/test45.sh b/testsuite/tests-to-run/test45.sh index 8e424e0b..0020836a 100644 --- a/testsuite/tests-to-run/test45.sh +++ b/testsuite/tests-to-run/test45.sh @@ -1,5 +1,8 @@ #!/bin/bash +rsync -Ha --delete input-files/segfault/ tmp/ +cd tmp + SERVER1=parallel-server3 SERVER2=parallel-server2 @@ -29,4 +32,94 @@ echo '### Test --load read from a file - less than 10s'; echo '### Bug in --load'; parallel -k --load 30 sleep 0.1\;echo ::: 1 2 3 +echo '### Test --timeout'; + parallel -j0 -k --timeout 1 echo {}\; sleep {}\; echo {} ::: 1.1 3.3 4.4 5.5 + +echo '### Test retired'; + stdout parallel -B; + stdout parallel -g; + stdout parallel -H; + stdout parallel -T; + stdout parallel -U; + stdout parallel -W; + stdout parallel -Y; + +echo '### Test --joblog followed by --resume --joblog'; + rm -f /tmp/joblog; + timeout -k 1 1 parallel -j2 --joblog /tmp/joblog sleep {} ::: 1.1 2.2 3.3 4.4 2>/dev/null; + parallel -j2 --resume --joblog /tmp/joblog sleep {} ::: 1.1 2.2 3.3 4.4; + cat /tmp/joblog | wc; + rm -f /tmp/joblog; + +echo '### Test --resume --joblog followed by --resume --joblog'; + rm -f /tmp/joblog2; + timeout -k 1 1 parallel -j2 --resume --joblog /tmp/joblog2 sleep {} ::: 1.1 2.2 3.3 4.4 2>/dev/null; + parallel -j2 --resume --joblog /tmp/joblog2 sleep {} ::: 1.1 2.2 3.3 4.4; + cat /tmp/joblog2 | wc; + rm -f /tmp/joblog2; + +echo '### Test --header'; + printf "a\tb\n1.2\t3/4.5" | parallel --header echo {b} {a} {b.} {b/} {b//} {b/.}; + +echo '### 64-bit wierdness - this did not complete on a 64-bit machine'; + seq 1 2 | parallel -j1 'seq 1 1 | parallel true' + +echo "### BUG-fix: bash -c 'parallel -a <(seq 1 3) echo'"; + stdout bash -c 'parallel -k -a <(seq 1 3) echo' + +echo "### BUG: The length for -X is not close to max (131072)"; + seq 1 60000 | parallel -X echo {.} aa {}{.} {}{}d{} {}dd{}d{.} |head -n 1 |wc; + seq 1 60000 | parallel -X echo a{}b{}c |head -n 1 |wc; + seq 1 60000 | parallel -X echo |head -n 1 |wc; + seq 1 60000 | parallel -X echo a{}b{}c {} |head -n 1 |wc; + seq 1 60000 | parallel -X echo {}aa{} |head -n 1 |wc; + seq 1 60000 | parallel -X echo {} aa {} |head -n 1 |wc; + +echo "### bug #35268: shell_quote doesn't treats [] brackets correctly"; + touch /tmp/foo1; + stdout parallel echo ::: '/tmp/foo[123]' + +echo '### Test make .deb package'; + cd ~/privat/parallel/packager/debian; + stdout make | grep 'To install the GNU Parallel Debian package, run:' + +echo '### Test of segfaulting issue'; + echo 'This gave /home/tange/bin/stdout: line 3: 20374 Segmentation fault "$@" 2>&1'; + echo 'before adding wait() before exit'; + seq 1 300 | stdout parallel ./trysegfault + +echo '### Test basic --arg-sep'; + parallel -k echo ::: a b + +echo '### Run commands using --arg-sep'; + parallel -kv ::: 'echo a' 'echo b' + +echo '### Change --arg-sep'; + parallel --arg-sep ::: -kv ::: 'echo a' 'echo b'; + parallel --arg-sep .--- -kv .--- 'echo a' 'echo b'; + parallel --argsep ::: -kv ::: 'echo a' 'echo b'; + parallel --argsep .--- -kv .--- 'echo a' 'echo b' + +echo '### Test stdin goes to first command only' +echo via cat |parallel --arg-sep .--- -kv .--- 'cat' 'echo b' +echo via cat |parallel -kv ::: 'cat' 'echo b' + +echo '### Bug made 4 5 go before 1 2 3'; + parallel -k ::: "sleep 1; echo 1" "echo 2" "echo 3" "echo 4" "echo 5" + +echo '### Bug made 3 go before 1 2'; + parallel -kj 1 ::: "sleep 1; echo 1" "echo 2" "echo 3" + +echo '### Bug did not quote'; + echo '>' | parallel -v echo; + parallel -v echo ::: '>'; + (echo '>'; echo 2) | parallel -j1 -vX echo; + parallel -X -j1 echo ::: '>' 2 + +echo '### Must not quote'; + echo 'echo | wc -l' | parallel -v; + parallel -v ::: 'echo | wc -l'; + echo 'echo a b c | wc -w' | parallel -v; + parallel -kv ::: 'echo a b c | wc -w' 'echo a b | wc -w' + EOF diff --git a/testsuite/tests-to-run/test65.sh b/testsuite/tests-to-run/test65.sh index b5886a60..2fc66000 100644 --- a/testsuite/tests-to-run/test65.sh +++ b/testsuite/tests-to-run/test65.sh @@ -2,37 +2,9 @@ # -L1 will join lines ending in ' ' cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | parallel -j10 -k -L1 -echo '### Test --timeout'; - parallel -j0 -k --timeout 1 echo {}\; sleep {}\; echo {} ::: 1.1 2.2 3.3 4.4 -echo '### Test retired'; - stdout parallel -B; - stdout parallel -g; - stdout parallel -H; - stdout parallel -T; - stdout parallel -U; - stdout parallel -W; - stdout parallel -Y; -echo '### Test --joblog followed by --resume --joblog'; - rm -f /tmp/joblog; - timeout -k 1 1 parallel -j2 --joblog /tmp/joblog sleep {} ::: 1.1 2.2 3.3 4.4 2>/dev/null; - parallel -j2 --resume --joblog /tmp/joblog sleep {} ::: 1.1 2.2 3.3 4.4; - cat /tmp/joblog | wc; - rm -f /tmp/joblog; -echo '### Test --resume --joblog followed by --resume --joblog'; - rm -f /tmp/joblog2; - timeout -k 1 1 parallel -j2 --resume --joblog /tmp/joblog2 sleep {} ::: 1.1 2.2 3.3 4.4 2>/dev/null; - parallel -j2 --resume --joblog /tmp/joblog2 sleep {} ::: 1.1 2.2 3.3 4.4; - cat /tmp/joblog2 | wc; - rm -f /tmp/joblog2; -echo '### Test --header'; - printf "a\tb\n1.2\t3/4.5" | parallel --header echo {b} {a} {b.} {b/} {b//} {b/.}; EOF echo '### Test --shellquote' cat <<'_EOF' | parallel --shellquote awk -v FS="\",\"" '{print $1, $3, $4, $5, $9, $14}' | grep -v "#" | sed -e '1d' -e 's/\"//g' -e 's/\/\/\//\t/g' | cut -f1-6,11 | sed -e 's/\/\//\t/g' -e 's/ /\t/g _EOF - -echo '### Test make .deb package' -cd ~/privat/parallel/packager/debian -stdout make | grep 'To install the GNU Parallel Debian package, run:' diff --git a/testsuite/wanted-results/test04 b/testsuite/wanted-results/test04 index c2199357..dd94659c 100644 Binary files a/testsuite/wanted-results/test04 and b/testsuite/wanted-results/test04 differ diff --git a/testsuite/wanted-results/test05 b/testsuite/wanted-results/test05 index 39ac577e..4bd50982 100644 --- a/testsuite/wanted-results/test05 +++ b/testsuite/wanted-results/test05 @@ -1,8 +1,8 @@ -497efe2559ec1b70289bcdeb1bf9d049 - +dcf2c2d2262ebbd3a97bb5e1d759cd38 - There are 6246 dirs with 6246 files Removing files -0b265d2499d13ed6adc8cd20813c963a - +b001b66cebd62b43bf78f1ab282495c9 - There are 6246 dirs with files Removing dirs -05b6f40267e8a40977c1bf1a17cac3b6 - +b88970f52ad2ae3d922bc44c3ed99657 - There are 1 dirs with files diff --git a/testsuite/wanted-results/test09 b/testsuite/wanted-results/test09 index f4169955..35864de4 100644 --- a/testsuite/wanted-results/test09 +++ b/testsuite/wanted-results/test09 @@ -50,27 +50,27 @@ sleep 2;false 0 9 parallel: Starting no more jobs. Waiting for 9 jobs to finish. This job failed: -perl -e sleep\ \$ARGV[0]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 1 +perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 1 parallel: Starting no more jobs. Waiting for 8 jobs to finish. This job failed: -perl -e sleep\ \$ARGV[0]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 2 +perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 2 parallel: Starting no more jobs. Waiting for 7 jobs to finish. This job failed: -perl -e sleep\ \$ARGV[0]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 3 +perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 3 parallel: Starting no more jobs. Waiting for 6 jobs to finish. This job failed: -perl -e sleep\ \$ARGV[0]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 4 +perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 4 parallel: Starting no more jobs. Waiting for 5 jobs to finish. This job failed: -perl -e sleep\ \$ARGV[0]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 5 +perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 5 parallel: Starting no more jobs. Waiting for 4 jobs to finish. This job failed: -perl -e sleep\ \$ARGV[0]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 6 +perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 6 parallel: Starting no more jobs. Waiting for 3 jobs to finish. This job failed: -perl -e sleep\ \$ARGV[0]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 7 +perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 7 parallel: Starting no more jobs. Waiting for 2 jobs to finish. This job failed: -perl -e sleep\ \$ARGV[0]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 8 +perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 8 parallel: Starting no more jobs. Waiting for 1 jobs to finish. This job failed: -perl -e sleep\ \$ARGV[0]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 9 +perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 9 0 1 parallel: This job failed: -perl -e sleep\ \$ARGV[0]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 1 +perl -e sleep\ \$ARGV\[0\]\;print\ STDERR\ @ARGV,\"\\n\"\;\ exit\ shift 1 ### Test last dying print --halt-on-error 9 1 diff --git a/testsuite/wanted-results/test24 b/testsuite/wanted-results/test24 index 740f1f87..e69de29b 100644 --- a/testsuite/wanted-results/test24 +++ b/testsuite/wanted-results/test24 @@ -1,12 +0,0 @@ -### 64-bit wierdness - this did not complete on a 64-bit machine -### BUG-fix: bash -c 'parallel -a <(seq 1 3) echo' -1 -2 -3 -### BUG: The length for -X is not close to max (131072) - 1 12821 131060 - 1 10948 131060 - 1 23695 131064 - 1 15810 131064 - 1 11790 131058 - 1 25545 131055 diff --git a/testsuite/wanted-results/test25 b/testsuite/wanted-results/test25 index 8740e781..63286bb3 100644 --- a/testsuite/wanted-results/test25 +++ b/testsuite/wanted-results/test25 @@ -1,28 +1,3 @@ -### Test basic --arg-sep -a -b -### Run commands using --arg-sep -echo a -a -echo b -b -### Change --arg-sep -echo a -a -echo b -b -echo a -a -echo b -b -echo a -a -echo b -b -echo a -a -echo b -b ### Test stdin goes to first command only ("-" as argument) cat - via first cat @@ -33,41 +8,3 @@ echo a a cat via pseudotty -### Test stdin goes to first command only -cat -via cat -echo b -b -cat -via cat -echo b -b -### Bug made 4 5 go before 1 2 3 -1 -2 -3 -4 -5 -### Bug made 3 go before 1 2 -1 -2 -3 -### Bug did not quote -echo \> -> -echo \> -> -echo \> 2 -> 2 -> 2 -### Must not quote -echo | wc -l -1 -echo | wc -l -1 -echo a b c | wc -w -3 -echo a b c | wc -w -3 -echo a b | wc -w -2 diff --git a/testsuite/wanted-results/test28 b/testsuite/wanted-results/test28 index 3eb86d27..e69de29b 100644 --- a/testsuite/wanted-results/test28 +++ b/testsuite/wanted-results/test28 @@ -1,3 +0,0 @@ -### Test of segfaulting issue -This gave /home/tange/bin/stdout: line 3: 20374 Segmentation fault "$@" 2>&1 -before adding wait() before exit diff --git a/testsuite/wanted-results/test45 b/testsuite/wanted-results/test45 index 1ee77480..a2a89676 100644 --- a/testsuite/wanted-results/test45 +++ b/testsuite/wanted-results/test45 @@ -13,3 +13,120 @@ OK 1 2 3 +### Test --timeout +1.1 +1.1 +3.3 +3.3 +4.4 +5.5 +### Test retired +parallel: -g has been retired. Use --group. +parallel: -B has been retired. Use --bf. +parallel: -T has been retired. Use --tty. +parallel: -U has been retired. Use --er. +parallel: -W has been retired. Use --wd. +parallel: -Y has been retired. Use --shebang. +parallel: -H has been retired. Use --halt. +parallel: -g has been retired. Use --group. +parallel: -B has been retired. Use --bf. +parallel: -T has been retired. Use --tty. +parallel: -U has been retired. Use --er. +parallel: -W has been retired. Use --wd. +parallel: -Y has been retired. Use --shebang. +parallel: -H has been retired. Use --halt. +parallel: -g has been retired. Use --group. +parallel: -B has been retired. Use --bf. +parallel: -T has been retired. Use --tty. +parallel: -U has been retired. Use --er. +parallel: -W has been retired. Use --wd. +parallel: -Y has been retired. Use --shebang. +parallel: -H has been retired. Use --halt. +### Test --joblog followed by --resume --joblog + 5 49 203 +### Test --resume --joblog followed by --resume --joblog + 5 49 203 +### Test --header +3/4.5 1.2 3/4 4.5 3 4 +### 64-bit wierdness - this did not complete on a 64-bit machine +### BUG-fix: bash -c 'parallel -a <(seq 1 3) echo' +1 +2 +3 +### BUG: The length for -X is not close to max (131072) + 1 12821 131060 + 1 10948 131060 + 1 23695 131064 + 1 15810 131064 + 1 11790 131058 + 1 25545 131055 +### bug #35268: shell_quote doesn't treats [] brackets correctly +/tmp/foo[123] +### Test make .deb package +To install the GNU Parallel Debian package, run: +### Test of segfaulting issue +This gave /home/tange/bin/stdout: line 3: 20374 Segmentation fault "$@" 2>&1 +before adding wait() before exit +### Test basic --arg-sep +a +b +### Run commands using --arg-sep +echo a +a +echo b +b +### Change --arg-sep +echo a +a +echo b +b +echo a +a +echo b +b +echo a +a +echo b +b +echo a +a +echo b +b +### Test stdin goes to first command only +cat +via cat +echo b +b +cat +via cat +echo b +b +### Bug made 4 5 go before 1 2 3 +1 +2 +3 +4 +5 +### Bug made 3 go before 1 2 +1 +2 +3 +### Bug did not quote +echo \> +> +echo \> +> +echo \> 2 +> 2 +> 2 +### Must not quote +echo | wc -l +1 +echo | wc -l +1 +echo a b c | wc -w +3 +echo a b c | wc -w +3 +echo a b | wc -w +2 diff --git a/testsuite/wanted-results/test65 b/testsuite/wanted-results/test65 index be42eef5..39bd8c4e 100644 --- a/testsuite/wanted-results/test65 +++ b/testsuite/wanted-results/test65 @@ -1,38 +1,2 @@ -### Test --timeout -1.1 -1.1 -2.2 -3.3 -4.4 -### Test retired -parallel: -g has been retired. Use --group. -parallel: -B has been retired. Use --bf. -parallel: -T has been retired. Use --tty. -parallel: -U has been retired. Use --er. -parallel: -W has been retired. Use --wd. -parallel: -Y has been retired. Use --shebang. -parallel: -H has been retired. Use --halt. -parallel: -g has been retired. Use --group. -parallel: -B has been retired. Use --bf. -parallel: -T has been retired. Use --tty. -parallel: -U has been retired. Use --er. -parallel: -W has been retired. Use --wd. -parallel: -Y has been retired. Use --shebang. -parallel: -H has been retired. Use --halt. -parallel: -g has been retired. Use --group. -parallel: -B has been retired. Use --bf. -parallel: -T has been retired. Use --tty. -parallel: -U has been retired. Use --er. -parallel: -W has been retired. Use --wd. -parallel: -Y has been retired. Use --shebang. -parallel: -H has been retired. Use --halt. -### Test --joblog followed by --resume --joblog - 5 49 205 -### Test --resume --joblog followed by --resume --joblog - 5 49 205 -### Test --header -3/4.5 1.2 3/4 4.5 3 4 ### Test --shellquote awk\ -v\ FS=\"\\\",\\\"\"\ \'\{print\ \$1,\ \$3,\ \$4,\ \$5,\ \$9,\ \$14\}\'\ \|\ grep\ -v\ \"\#\"\ \|\ sed\ -e\ \'1d\'\ -e\ \'s/\\\"//g\'\ -e\ \'s/\\/\\/\\//\\t/g\'\ \|\ cut\ -f1-6,11\ \|\ sed\ -e\ \'s/\\/\\//\\t/g\'\ -e\ \'s/\ /\\t/g -### Test make .deb package -To install the GNU Parallel Debian package, run: