echo 'bug #41613: --compress --line-buffer no newline'; bug #41613: --compress --line-buffer no newline perl -e 'print "It worked"'| $NICEPAR --pipe --compress --line-buffer cat; echo It worked echo 'bug #41613: --compress --line-buffer no --tagstring'; bug #41613: --compress --line-buffer no --tagstring diff <(nice perl -e 'for("x011".."x110"){print "$_\t", ("\n", map { rand } (1..100000)) }'| $NICEPAR -N10 -L1 --pipe -j6 --block 20M --compress pv -qL 1000000 | perl -pe 's/(....).*/$1/') <(nice perl -e 'for("x011".."x110"){print "$_\t", ("\n", map { rand } (1..100000)) }'| $NICEPAR -N10 -L1 --pipe -j6 --block 20M --compress --line-buffer pv -qL 1000000 | perl -pe 's/(....).*/$1/') >/dev/null || (echo 'Good: --line-buffer matters'; false) && echo 'Bad: --line-buffer not working' Good: --line-buffer matters echo 'bug #41613: --compress --line-buffer with --tagstring'; bug #41613: --compress --line-buffer with --tagstring diff <(nice perl -e 'for("x011".."x110"){print "$_\t", ("\n", map { rand } (1..100000)) }'| $NICEPAR -N10 -L1 --pipe -j6 --block 20M --compress --tagstring {#} pv -qL 1000000 | perl -pe 's/(....).*/$1/') <(nice perl -e 'for("x011".."x110"){print "$_\t", ("\n", map { rand } (1..100000)) }'| $NICEPAR -N10 -L1 --pipe -j6 --block 20M --compress --tagstring {#} --line-buffer pv -qL 1000000 | perl -pe 's/(....).*/$1/') >/dev/null || (echo 'Good: --line-buffer matters'; false) && echo 'Bad: --line-buffer not working' Good: --line-buffer matters echo 'bug #41613: --compress --line-buffer - no newline'; bug #41613: --compress --line-buffer - no newline echo 'pipe compress tagstring' pipe compress tagstring perl -e 'print "O"'| $NICEPAR --compress --tagstring {#} --pipe --line-buffer cat; echo "K" 1 OK echo 'pipe compress notagstring' pipe compress notagstring perl -e 'print "O"'| $NICEPAR --compress --pipe --line-buffer cat; echo "K" OK echo 'pipe nocompress tagstring' pipe nocompress tagstring perl -e 'print "O"'| $NICEPAR --tagstring {#} --pipe --line-buffer cat; echo "K" 1 OK echo 'pipe nocompress notagstring' pipe nocompress notagstring perl -e 'print "O"'| $NICEPAR --pipe --line-buffer cat; echo "K" OK echo 'nopipe compress tagstring' nopipe compress tagstring $NICEPAR --compress --tagstring {#} --line-buffer echo {} O ::: -n; echo "K" 1 OK echo 'nopipe compress notagstring' nopipe compress notagstring $NICEPAR --compress --line-buffer echo {} O ::: -n; echo "K" OK echo 'nopipe nocompress tagstring' nopipe nocompress tagstring $NICEPAR --tagstring {#} --line-buffer echo {} O ::: -n; echo "K" 1 OK echo 'nopipe nocompress notagstring' nopipe nocompress notagstring $NICEPAR --line-buffer echo {} O ::: -n; echo "K" OK echo 'bug #41412: --timeout + --delay causes deadlock'; bug #41412: --timeout + --delay causes deadlock seq 10 | parallel -j10 --timeout 1 --delay .3 echo; 1 2 3 4 5 6 7 8 9 10 parallel -j3 --timeout 1 --delay 2 echo ::: 1 2 3; 1 2 3 parallel -j10 --timeout 2.2 --delay 3 "sleep {}; echo {}" ::: 1 2 7 8 9 1 2 echo '### Test --spreadstdin - more procs than args'; rm -f /tmp/parallel.ss.*; seq 1 5 | stdout $NICEPAR -j 10 --spreadstdin 'cat >/tmp/parallel.ss.$PARALLEL_SEQ' >/dev/null; cat /tmp/parallel.ss.*; ### Test --spreadstdin - more procs than args 1 2 3 4 5 echo '### Test --spreadstdin - more args than procs'; rm -f /tmp/parallel.ss2.*; seq 1 10 | stdout $NICEPAR -j 5 --spreadstdin 'cat >/tmp/parallel.ss2.$PARALLEL_SEQ' >/dev/null; cat /tmp/parallel.ss2.* ### Test --spreadstdin - more args than procs 1 2 3 4 5 6 7 8 9 10 nice nice seq 1 1000 | $NICEPAR -j1 --spreadstdin cat "|cat "|wc -c 3893 nice nice seq 1 10000 | $NICEPAR -j10 --spreadstdin cat "|cat "|wc -c 48894 nice nice seq 1 100000 | $NICEPAR -j1 --spreadstdin cat "|cat "|wc -c 588895 nice nice seq 1 1000000 | $NICEPAR -j10 --spreadstdin cat "|cat "|wc -c 6888896 seq 1 10 | $NICEPAR --recend "\n" -j1 --spreadstdin gzip -9 >/tmp/foo.gz echo '### Test --spreadstdin - similar to the failing below'; nice seq 1 100000 | $NICEPAR --recend "\n" -j10 --spreadstdin gzip -9 >/tmp/foo2.gz; diff <(nice seq 1 100000) <(zcat /tmp/foo2.gz |sort -n); diff <(nice seq 1 100000|wc -c) <(zcat /tmp/foo2.gz |wc -c) ### Test --spreadstdin - similar to the failing below echo '### Test --spreadstdin - this failed during devel'; nice seq 1 1000000 | md5sum; nice seq 1 1000000 | $NICEPAR --recend "\n" -j10 --spreadstdin gzip -9 | zcat | sort -n | md5sum ### Test --spreadstdin - this failed during devel 8a7095c1c23bfadc311fe6b16d950582 - 8a7095c1c23bfadc311fe6b16d950582 - echo '### Test --spreadstdin -k'; nice seq 1 1000000 | $NICEPAR -k --recend "\n" -j10 --spreadstdin gzip -9 | zcat | md5sum ### Test --spreadstdin -k 8a7095c1c23bfadc311fe6b16d950582 - echo '### Test --spreadstdin --files'; nice seq 1 1000000 | shuf | $NICEPAR --files --recend "\n" -j10 --spreadstdin sort -n | parallel -Xj1 sort -nm {} ";"rm {} | md5sum ### Test --spreadstdin --files 8a7095c1c23bfadc311fe6b16d950582 - echo '### Test --number-of-cpus'; stdout $NICEPAR --number-of-cpus ### Test --number-of-cpus 1 echo '### Test --number-of-cores'; stdout $NICEPAR --number-of-cores ### Test --number-of-cores 8 echo '### Test --use-cpus-instead-of-cores'; (seq 1 8 | stdout parallel --use-cpus-instead-of-cores -j100% sleep) && echo CPUs done & (seq 1 8 | stdout parallel -j100% sleep) && echo cores done & echo 'Cores should complete first on machines with less than 8 physical CPUs'; wait ### Test --use-cpus-instead-of-cores Cores should complete first on machines with less than 8 physical CPUs cores done CPUs done echo '### Test --tag ::: a ::: b'; stdout $NICEPAR -k --tag -j1 echo stderr-{.} ">&2;" echo stdout-{} ::: a ::: b ### Test --tag ::: a ::: b a b stdout-a b a b stderr-a b echo '### Test --tag ::: a b'; stdout $NICEPAR -k --tag -j1 echo stderr-{.} ">&2;" echo stdout-{} ::: a b ### Test --tag ::: a b a stdout-a a stderr-a b stdout-b b stderr-b echo '### Test --tag -X ::: a b'; stdout $NICEPAR -k --tag -X -j1 echo stderr-{.} ">&2;" echo stdout-{} ::: a b ### Test --tag -X ::: a b a b stdout-a stdout-b a b stderr-a stderr-b echo '### Test bash redirection <()'; ### Test bash redirection <() $NICEPAR 'cat <(echo {}); echo b' ::: a a b echo '### Test bug https://savannah.gnu.org/bugs/index.php?33352' ### Test bug https://savannah.gnu.org/bugs/index.php?33352 # produce input slowly to parallel so that it will reap a process # while blocking in read() # Having found the solution it is suddenly very easy to reproduce the # problem - even on other hardware: # # perl -e '@x=1 .. 17000; for(1..30) { print "@x\n"}' | pv -qL 200000 # |parallel -j2 --pipe --keeporder --block 150000 cat | md5sum # # This gives different md5sums for each run. # # The problem is that read(STDIN) is being interrupted by a dead # child. The chance of this happening is very small if there are few # children dying or read(STDIN) never has to wait for data. # # The test above forces data to arrive slowly (using pv) which causes # read(STDIN) to take a long time - thus being interrupted by a dead # child. echo "# md5sum - directly" # md5sum - directly nice perl -e '@x=1 .. 17000; for(1..100) { print "@x\n"}' | md5sum 350eda13a37912d755c9d733d149bdaf - echo "# parallel | md5sum" # parallel | md5sum nice nice perl -e '@x=1 .. 17000; for(1..100) { print "@x\n"}' | pv -qL 1000000 | $PAR cat | md5sum 350eda13a37912d755c9d733d149bdaf - echo "# --recend ''" # --recend '' nice nice perl -e '@x=1 .. 17000; for(1..100) { print "@x\n"}' | pv -qL 1000000 | $PAR --recend '' cat | md5sum 350eda13a37912d755c9d733d149bdaf - echo "# --recend '' --files" # --recend '' --files nice nice perl -e '@x=1 .. 17000; for(1..100) { print "@x\n"}' | pv -qL 1000000 | $PAR --recend '' --files cat | parallel -Xj1 cat {} ';' rm {} | md5sum 350eda13a37912d755c9d733d149bdaf - echo "# --recend '' --files --tmpdir" # --recend '' --files --tmpdir nice nice perl -e '@x=1 .. 17000; for(1..100) { print "@x\n"}' | pv -qL 1000000 | $PAR --recend '' --files --tmpdir /dev/shm cat | parallel -Xj1 cat {} ';' rm {} | md5sum 350eda13a37912d755c9d733d149bdaf - echo "# --recend '' --files --halt-on-error" # --recend '' --files --halt-on-error nice nice perl -e '@x=1 .. 17000; for(1..100) { print "@x\n"}' | pv -qL 1000000 | $PAR --recend '' --files --halt-on-error 2 cat | parallel -Xj1 cat {} ';' rm {} | md5sum 350eda13a37912d755c9d733d149bdaf - echo '### Test of -j filename - non-existent file'; nice stdout parallel -j no_such_file echo ::: 1 ### Test of -j filename - non-existent file Usage: parallel [options] [command [arguments]] < list_of_arguments parallel [options] [command [arguments]] (::: arguments|:::: argfile(s))... cat ... | parallel --pipe [options] [command [arguments]] -j n Run n jobs in parallel -k Keep same order -X Multiple arguments with context replace --colsep regexp Split input on regexp for positional replacements {} {.} {/} {/.} {#} {%} {= perl code =} Replacement strings {3} {3.} {3/} {3/.} {=3 perl code =} Positional replacement strings With --plus: {} = {+/}/{/} = {.}.{+.} = {+/}/{/.}.{+.} = {..}.{+..} = {+/}/{/..}.{+..} = {...}.{+...} = {+/}/{/...}.{+...} -S sshlogin Example: foo@server.example.com --slf .. Use ~/.parallel/sshloginfile as the list of sshlogins --trc {}.bar Shorthand for --transfer --return {}.bar --cleanup --onall Run the given command with argument on all sshlogins --nonall Run the given command with no arguments on all sshlogins --pipe Split stdin (standard input) to multiple jobs. --recend str Record end separator for --pipe. --recstart str Record start separator for --pipe. See 'man parallel' for details When using programs that use GNU Parallel to process data for publication please cite: O. Tange (2011): GNU Parallel - The Command-Line Power Tool, ;login: The USENIX Magazine, February 2011:42-47. Or you can get GNU Parallel without this requirement by paying 10000 EUR. parallel: Error: Parsing of --jobs/-j/--max-procs/-P failed. echo '### Test of -j filename'; echo 3 >/tmp/jobs_to_run1; parallel -j /tmp/jobs_to_run1 -v sleep {} ::: 10 8 6 5 4; # Should give 6 8 10 5 4 ### Test of -j filename sleep 6 sleep 8 sleep 10 sleep 5 sleep 4 echo '### Test ::::' ### Test :::: echo '### Change --arg-file-sep' ### Change --arg-file-sep $XAP --arg-file-sep :::: -k echo {1} {2} :::: <(seq 1 10) <(seq 5 15) 1 5 2 6 3 7 4 8 5 9 6 10 7 11 8 12 9 13 10 14 1 15 $XAP --arg-file-sep .--- -k echo {1} {2} .--- <(seq 1 10) <(seq 5 15) 1 5 2 6 3 7 4 8 5 9 6 10 7 11 8 12 9 13 10 14 1 15 $XAP --argfilesep :::: -k echo {1} {2} :::: <(seq 1 10) <(seq 5 15) 1 5 2 6 3 7 4 8 5 9 6 10 7 11 8 12 9 13 10 14 1 15 $XAP --argfilesep .--- -k echo {1} {2} .--- <(seq 1 10) <(seq 5 15) 1 5 2 6 3 7 4 8 5 9 6 10 7 11 8 12 9 13 10 14 1 15 echo '### Test xapply --max-replace-args' ### Test xapply --max-replace-args seq 0 7 | $XAP -k --max-replace-args=3 echo {3} {2} {1} 2 1 0 5 4 3 7 6 echo '### Test -N' ### Test -N seq 1 5 | $XAP -kN3 echo {1} {2} {3} 1 2 3 4 5 echo '### Test -N with 0' ### Test -N with 0 seq 0 7 | $XAP -kN3 echo {1} {2} {3} 0 1 2 3 4 5 6 7 echo '### Test :::: on nonexistent' ### Test :::: on nonexistent stdout $XAP -k echo {1} {2} {3} :::: nonexistent parallel: Error: Cannot open input file `nonexistent': No such file or directory. echo '### Test :::: two files' ### Test :::: two files $XAP -k echo {1} {2} :::: <(seq 1 10) <(seq 5 15) 1 5 2 6 3 7 4 8 5 9 6 10 7 11 8 12 9 13 10 14 1 15 echo '### Test -d, ::::' ### Test -d, :::: $XAP -kd, 'echo a{1} {2}b' :::: <(echo 1,2,3,) <(echo 5,6,7,8) a1 5b a2 6b a3 7b a 8 b echo '### Test -d, :::: one file too much' ### Test -d, :::: one file too much $XAP -kd, echo 'a{1}' '{2}b' :::: <(echo 1,2,3,) <(echo 5,6,7,8) <(echo 9,0) a1 5b a2 6b a3 7b a 8 b echo '### Bug: did not quote' ### Bug: did not quote $XAP echo {1} {2} :::: <(echo '>') <(echo b) > b echo '### Quote test triplet 1' ### Quote test triplet 1 $XAP -kv :::: <(echo 'echo a'; echo 'echo b') echo a a echo b b $XAP -kv -a <(echo 'echo a'; echo 'echo b') echo a a echo b b (echo 'echo a'; echo 'echo b') | $XAP -kv echo a a echo b b echo '### Quote test triplet 2' ### Quote test triplet 2 $XAP -kv echo :::: <(echo 'echo a'; echo 'echo b') echo echo\ a echo a echo echo\ b echo b $XAP -kv -a <(echo 'echo a'; echo 'echo b') echo echo echo\ a echo a echo echo\ b echo b (echo 'echo a'; echo 'echo b') | $XAP -kv echo echo echo\ a echo a echo echo\ b echo b echo '### Quoting if there is a command and 2 arg files' ### Quoting if there is a command and 2 arg files $XAP -kv echo :::: <(echo 'echo a') <(echo 'echo b') echo echo\ a echo\ b echo a echo b echo '### Quoting if there is a command and 2 arg files of uneven length' ### Quoting if there is a command and 2 arg files of uneven length $XAP -kv echo :::: <(echo 'echo a';echo a1) <(echo 'echo b') echo echo\ a echo\ b echo a echo b echo a1 echo\ b a1 echo b echo '### Quoting if there is no command and 2 arg files' ### Quoting if there is no command and 2 arg files $XAP -kv :::: <(echo 'echo a') <(echo 'echo b') echo a echo b a echo b echo '### Quoting if there is no command and 2 arg files of uneven length' ### Quoting if there is no command and 2 arg files of uneven length $XAP -kv :::: <(echo 'echo a';echo echo a1) <(echo 'echo b') echo a echo b a echo b echo a1 echo b a1 echo b echo '### Test multiple -a' ### Test multiple -a $XAP -kv -a <(echo a) -a <(echo b) echo {2} {1} echo b a b a $XAP -kv echo {2} {1} :::: <(echo a) <(echo b) echo b a b a echo '### Multiple -a: An unused file' ### Multiple -a: An unused file $XAP -kv -a <(echo a) -a <(echo b) -a <(echo c) echo {2} {1} echo b a b a $XAP -kv echo {2} {1} :::: <(echo a) <(echo b) <(echo c) echo b a b a echo '### Multiple -a: nonexistent' ### Multiple -a: nonexistent stdout $XAP -kv echo {2} {1} :::: nonexist nonexist2 parallel: Error: Cannot open input file `nonexist': No such file or directory. stdout $XAP -kv -a nonexist -a nonexist2 echo {2} {1} parallel: Error: Cannot open input file `nonexist': No such file or directory. echo '### Test {#.}' ### Test {#.} $XAP -kv -a <(echo a-noext) -a <(echo b-withext.extension) -a <(echo c-ext.gif) echo {3.} {2.} {1.} echo c-ext b-withext a-noext c-ext b-withext a-noext echo "### Tests that failed for OO-rewrite" ### Tests that failed for OO-rewrite parallel -u --semaphore seq 1 10 '|' pv -qL 20; sem --wait; echo done 1 2 3 4 5 6 7 8 9 10 done echo a | parallel echo {1} a echo "echo a" | parallel a nice parallel -j1 -I :: -X echo 'a::b::^c::[.}c' ::: 1 a1b1^c1[.}c echo "### BUG: The length for -X is not close to max (131072)" ### BUG: The length for -X is not close to max (131072) seq 1 4000 | nice parallel -k -X echo {.} aa {}{.} {}{}d{} {}dd{}d{.} |head -n 1 |wc 1 12821 131060 echo "### BUG: empty lines with --show-limit" ### BUG: empty lines with --show-limit echo | $NICEPAR --show-limits Maximal size of command: 131071 Maximal used size of command: 131071 Execution of will continue now, and it will try to read its input and run commands; if this is not what you wanted to happen, please press CTRL-D or CTRL-C echo '### Test -N' ### Test -N seq 1 5 | $NICEPAR -kN3 echo {1} {2} {3} 1 2 3 4 5 echo '### Test --arg-file-sep with files of different lengths' ### Test --arg-file-sep with files of different lengths $XAP --arg-file-sep :::: -k echo {1} {2} :::: <(seq 1 1) <(seq 3 4) 1 3 1 4 echo '### Test respect -s' ### Test respect -s $NICEPAR -kvm -IARG -s15 echo ARG ::: 1 22 333 4444 55555 666666 7777777 88888888 999999999 echo 1 22 333 1 22 333 echo 4444 4444 echo 55555 55555 echo 666666 666666 echo 7777777 7777777 echo 88888888 88888888 echo 999999999 999999999 echo '### Test eof string after :::' ### Test eof string after ::: $NICEPAR -k -E ole echo ::: foo ole bar foo echo '### Test -C and --trim rl' ### Test -C and --trim rl $NICEPAR -k -C %+ echo '"{1}_{3}_{2}_{4}"' ::: 'a% c %%b' 'a%c% b %d' a_b_c_ a_b_c_d echo '### Test empty input' ### Test empty input