echo 'bug #44250: pxz complains File format not recognized but decompresses anyway' bug #44250: pxz complains File format not recognized but decompresses anyway # The first line dumps core if run from make file. Why?! stdout parallel --compress --compress-program pxz ls /{} ::: OK-if-missing-file Segmentation fault (core dumped) parallel: Error: pxz -dc failed stdout parallel --compress --compress-program pixz --decompress-program 'pixz -d' ls /{} ::: OK-if-missing-file ls: cannot access /OK-if-missing-file: No such file or directory stdout parallel --compress --compress-program pixz --decompress-program 'pixz -d' true ::: OK-if-no-output stdout parallel --compress --compress-program pxz true ::: OK-if-no-output echo 'bug #41613: --compress --line-buffer no newline'; bug #41613: --compress --line-buffer no newline perl -e 'print "It worked"'| $NICEPAR --pipe --compress --line-buffer cat; echo It worked echo 'bug #41613: --compress --line-buffer no --tagstring'; bug #41613: --compress --line-buffer no --tagstring diff <(nice perl -e 'for("x011".."x110"){print "$_\t", ("\n", map { rand } (1..100000)) }'| $NICEPAR -N10 -L1 --pipe -j6 --block 20M --compress pv -qL 1000000 | perl -pe 's/(....).*/$1/') <(nice perl -e 'for("x011".."x110"){print "$_\t", ("\n", map { rand } (1..100000)) }'| $NICEPAR -N10 -L1 --pipe -j6 --block 20M --compress --line-buffer pv -qL 1000000 | perl -pe 's/(....).*/$1/') >/dev/null || (echo 'Good: --line-buffer matters'; false) && echo 'Bad: --line-buffer not working' Good: --line-buffer matters echo 'bug #41613: --compress --line-buffer with --tagstring'; bug #41613: --compress --line-buffer with --tagstring diff <(nice perl -e 'for("x011".."x110"){print "$_\t", ("\n", map { rand } (1..100000)) }'| $NICEPAR -N10 -L1 --pipe -j6 --block 20M --compress --tagstring {#} pv -qL 1000000 | perl -pe 's/(....).*/$1/') <(nice perl -e 'for("x011".."x110"){print "$_\t", ("\n", map { rand } (1..100000)) }'| $NICEPAR -N10 -L1 --pipe -j6 --block 20M --compress --tagstring {#} --line-buffer pv -qL 1000000 | perl -pe 's/(....).*/$1/') >/dev/null || (echo 'Good: --line-buffer matters'; false) && echo 'Bad: --line-buffer not working' Good: --line-buffer matters echo 'bug #41613: --compress --line-buffer - no newline'; bug #41613: --compress --line-buffer - no newline echo 'pipe compress tagstring' pipe compress tagstring perl -e 'print "O"'| $NICEPAR --compress --tagstring {#} --pipe --line-buffer cat; echo "K" 1 OK echo 'pipe compress notagstring' pipe compress notagstring perl -e 'print "O"'| $NICEPAR --compress --pipe --line-buffer cat; echo "K" OK echo 'pipe nocompress tagstring' pipe nocompress tagstring perl -e 'print "O"'| $NICEPAR --tagstring {#} --pipe --line-buffer cat; echo "K" 1 OK echo 'pipe nocompress notagstring' pipe nocompress notagstring perl -e 'print "O"'| $NICEPAR --pipe --line-buffer cat; echo "K" OK echo 'nopipe compress tagstring' nopipe compress tagstring $NICEPAR --compress --tagstring {#} --line-buffer echo {} O ::: -n; echo "K" 1 OK echo 'nopipe compress notagstring' nopipe compress notagstring $NICEPAR --compress --line-buffer echo {} O ::: -n; echo "K" OK echo 'nopipe nocompress tagstring' nopipe nocompress tagstring $NICEPAR --tagstring {#} --line-buffer echo {} O ::: -n; echo "K" 1 OK echo 'nopipe nocompress notagstring' nopipe nocompress notagstring $NICEPAR --line-buffer echo {} O ::: -n; echo "K" OK echo 'bug #41412: --timeout + --delay causes deadlock'; bug #41412: --timeout + --delay causes deadlock seq 10 | parallel -j10 --timeout 1 --delay .3 echo; 1 2 3 4 5 6 7 8 9 10 parallel -j3 --timeout 1 --delay 2 echo ::: 1 2 3; 1 2 3 parallel -j10 --timeout 2.2 --delay 3 "sleep {}; echo {}" ::: 1 2 7 8 9 1 2 echo '### Test --spreadstdin - more procs than args'; rm -f /tmp/parallel.ss.*; seq 1 5 | stdout $NICEPAR -j 10 --spreadstdin 'cat >/tmp/parallel.ss.$PARALLEL_SEQ' >/dev/null; cat /tmp/parallel.ss.*; rm -f /tmp/parallel.ss.* ### Test --spreadstdin - more procs than args 1 2 3 4 5 echo '### Test --spreadstdin - more args than procs'; rm -f /tmp/parallel.ss2.*; seq 1 10 | stdout $NICEPAR -j 5 --spreadstdin 'cat >/tmp/parallel.ss2.$PARALLEL_SEQ' >/dev/null; cat /tmp/parallel.ss2.*; rm -f /tmp/parallel.ss2.* ### Test --spreadstdin - more args than procs 1 2 3 4 5 6 7 8 9 10 nice nice seq 1 1000 | $NICEPAR -j1 --spreadstdin cat "|cat "|wc -c 3893 nice nice seq 1 10000 | $NICEPAR -j10 --spreadstdin cat "|cat "|wc -c 48894 nice nice seq 1 100000 | $NICEPAR -j1 --spreadstdin cat "|cat "|wc -c 588895 nice nice seq 1 1000000 | $NICEPAR -j10 --spreadstdin cat "|cat "|wc -c 6888896 seq 1 10 | $NICEPAR --recend "\n" -j1 --spreadstdin gzip -9 >/tmp/foo.gz; rm /tmp/foo.gz echo '### Test --spreadstdin - similar to the failing below'; nice seq 1 100000 | $NICEPAR --recend "\n" -j10 --spreadstdin gzip -9 >/tmp/foo2.gz; diff <(nice seq 1 100000) <(zcat /tmp/foo2.gz |sort -n); diff <(nice seq 1 100000|wc -c) <(zcat /tmp/foo2.gz |wc -c); rm /tmp/foo2.gz ### Test --spreadstdin - similar to the failing below echo '### Test --spreadstdin - this failed during devel'; nice seq 1 1000000 | md5sum; nice seq 1 1000000 | $NICEPAR --recend "\n" -j10 --spreadstdin gzip -9 | zcat | sort -n | md5sum ### Test --spreadstdin - this failed during devel 8a7095c1c23bfadc311fe6b16d950582 - 8a7095c1c23bfadc311fe6b16d950582 - echo '### Test --spreadstdin -k'; nice seq 1 1000000 | $NICEPAR -k --recend "\n" -j10 --spreadstdin gzip -9 | zcat | md5sum ### Test --spreadstdin -k 8a7095c1c23bfadc311fe6b16d950582 - echo '### Test --spreadstdin --files'; nice seq 1 1000000 | shuf | $NICEPAR --files --recend "\n" -j10 --spreadstdin sort -n | parallel -Xj1 sort -nm {} ";"rm {} | md5sum ### Test --spreadstdin --files 8a7095c1c23bfadc311fe6b16d950582 - echo '### Test --number-of-cpus'; stdout $NICEPAR --number-of-cpus ### Test --number-of-cpus 1 echo '### Test --number-of-cores'; stdout $NICEPAR --number-of-cores ### Test --number-of-cores 8 echo '### Test --use-cpus-instead-of-cores'; (seq 1 8 | stdout parallel --use-cpus-instead-of-cores -j100% sleep) && echo CPUs done & (seq 1 8 | stdout parallel -j100% sleep) && echo cores done & echo 'Cores should complete first on machines with less than 8 physical CPUs'; wait ### Test --use-cpus-instead-of-cores Cores should complete first on machines with less than 8 physical CPUs cores done CPUs done echo '### Test --tag ::: a ::: b'; stdout $NICEPAR -k --tag -j1 echo stderr-{.} ">&2;" echo stdout-{} ::: a ::: b ### Test --tag ::: a ::: b a b stdout-a b a b stderr-a b echo '### Test --tag ::: a b'; stdout $NICEPAR -k --tag -j1 echo stderr-{.} ">&2;" echo stdout-{} ::: a b ### Test --tag ::: a b a stdout-a a stderr-a b stdout-b b stderr-b echo '### Test --tag -X ::: a b'; stdout $NICEPAR -k --tag -X -j1 echo stderr-{.} ">&2;" echo stdout-{} ::: a b ### Test --tag -X ::: a b a b stdout-a stdout-b a b stderr-a stderr-b echo '### Test bash redirection <()'; ### Test bash redirection <() $NICEPAR 'cat <(echo {}); echo b' ::: a a b echo '### Test bug https://savannah.gnu.org/bugs/index.php?33352' ### Test bug https://savannah.gnu.org/bugs/index.php?33352 # produce input slowly to parallel so that it will reap a process # while blocking in read() # Having found the solution it is suddenly very easy to reproduce the # problem - even on other hardware: # # perl -e '@x=1 .. 17000; for(1..30) { print "@x\n"}' | pv -qL 200000 # |parallel -j2 --pipe --keeporder --block 150000 cat | md5sum # # This gives different md5sums for each run. # # The problem is that read(STDIN) is being interrupted by a dead # child. The chance of this happening is very small if there are few # children dying or read(STDIN) never has to wait for data. # # The test above forces data to arrive slowly (using pv) which causes # read(STDIN) to take a long time - thus being interrupted by a dead # child. echo "# md5sum - directly" # md5sum - directly nice perl -e '@x=1 .. 17000; for(1..100) { print "@x\n"}' | md5sum 350eda13a37912d755c9d733d149bdaf - echo "# parallel | md5sum" # parallel | md5sum nice nice perl -e '@x=1 .. 17000; for(1..100) { print "@x\n"}' | pv -qL 1000000 | $PAR cat | md5sum 350eda13a37912d755c9d733d149bdaf - echo "# --recend ''" # --recend '' nice nice perl -e '@x=1 .. 17000; for(1..100) { print "@x\n"}' | pv -qL 1000000 | $PAR --recend '' cat | md5sum 350eda13a37912d755c9d733d149bdaf - echo "# --recend '' --files" # --recend '' --files nice nice perl -e '@x=1 .. 17000; for(1..100) { print "@x\n"}' | pv -qL 1000000 | $PAR --recend '' --files cat | parallel -Xj1 cat {} ';' rm {} | md5sum 350eda13a37912d755c9d733d149bdaf - echo "# --recend '' --files --tmpdir" # --recend '' --files --tmpdir nice nice perl -e '@x=1 .. 17000; for(1..100) { print "@x\n"}' | pv -qL 1000000 | $PAR --recend '' --files --tmpdir /dev/shm cat | parallel -Xj1 cat {} ';' rm {} | md5sum 350eda13a37912d755c9d733d149bdaf - echo "# --recend '' --files --halt-on-error" # --recend '' --files --halt-on-error nice nice perl -e '@x=1 .. 17000; for(1..100) { print "@x\n"}' | pv -qL 1000000 | $PAR --recend '' --files --halt-on-error 2 cat | parallel -Xj1 cat {} ';' rm {} | md5sum 350eda13a37912d755c9d733d149bdaf - echo '### Test of -j filename - non-existent file'; nice stdout parallel -j no_such_file echo ::: 1 ### Test of -j filename - non-existent file Usage: parallel [options] [command [arguments]] < list_of_arguments parallel [options] [command [arguments]] (::: arguments|:::: argfile(s))... cat ... | parallel --pipe [options] [command [arguments]] -j n Run n jobs in parallel -k Keep same order -X Multiple arguments with context replace --colsep regexp Split input on regexp for positional replacements {} {.} {/} {/.} {#} {%} {= perl code =} Replacement strings {3} {3.} {3/} {3/.} {=3 perl code =} Positional replacement strings With --plus: {} = {+/}/{/} = {.}.{+.} = {+/}/{/.}.{+.} = {..}.{+..} = {+/}/{/..}.{+..} = {...}.{+...} = {+/}/{/...}.{+...} -S sshlogin Example: foo@server.example.com --slf .. Use ~/.parallel/sshloginfile as the list of sshlogins --trc {}.bar Shorthand for --transfer --return {}.bar --cleanup --onall Run the given command with argument on all sshlogins --nonall Run the given command with no arguments on all sshlogins --pipe Split stdin (standard input) to multiple jobs. --recend str Record end separator for --pipe. --recstart str Record start separator for --pipe. See 'man parallel' for details Academic tradition requires you to cite works you base your article on. When using programs that use GNU Parallel to process data for publication please cite: O. Tange (2011): GNU Parallel - The Command-Line Power Tool, ;login: The USENIX Magazine, February 2011:42-47. This helps funding further development; and it won't cost you a cent. If you pay 10000 EUR you should feel free to use GNU Parallel without citing. parallel: Error: Parsing of --jobs/-j/--max-procs/-P failed. echo '### Test of -j filename'; echo 3 >/tmp/jobs_to_run1; parallel -j /tmp/jobs_to_run1 -v sleep {} ::: 10 8 6 5 4; # Should give 6 8 10 5 4 ### Test of -j filename sleep 6 sleep 8 sleep 10 sleep 5 sleep 4 echo '### Test ::::' ### Test :::: echo '### Change --arg-file-sep' ### Change --arg-file-sep $XAP --arg-file-sep :::: -k echo {1} {2} :::: <(seq 1 10) <(seq 5 15) 1 5 2 6 3 7 4 8 5 9 6 10 7 11 8 12 9 13 10 14 1 15 $XAP --arg-file-sep .--- -k echo {1} {2} .--- <(seq 1 10) <(seq 5 15) 1 5 2 6 3 7 4 8 5 9 6 10 7 11 8 12 9 13 10 14 1 15 $XAP --argfilesep :::: -k echo {1} {2} :::: <(seq 1 10) <(seq 5 15) 1 5 2 6 3 7 4 8 5 9 6 10 7 11 8 12 9 13 10 14 1 15 $XAP --argfilesep .--- -k echo {1} {2} .--- <(seq 1 10) <(seq 5 15) 1 5 2 6 3 7 4 8 5 9 6 10 7 11 8 12 9 13 10 14 1 15 echo '### Test xapply --max-replace-args' ### Test xapply --max-replace-args seq 0 7 | $XAP -k --max-replace-args=3 echo {3} {2} {1} 2 1 0 5 4 3 7 6 echo '### Test -N' ### Test -N seq 1 5 | $XAP -kN3 echo {1} {2} {3} 1 2 3 4 5 echo '### Test -N with 0' ### Test -N with 0 seq 0 7 | $XAP -kN3 echo {1} {2} {3} 0 1 2 3 4 5 6 7 echo '### Test :::: on nonexistent' ### Test :::: on nonexistent stdout $XAP -k echo {1} {2} {3} :::: nonexistent parallel: Error: Cannot open input file `nonexistent': No such file or directory. echo '### Test :::: two files' ### Test :::: two files $XAP -k echo {1} {2} :::: <(seq 1 10) <(seq 5 15) 1 5 2 6 3 7 4 8 5 9 6 10 7 11 8 12 9 13 10 14 1 15 echo '### Test -d, ::::' ### Test -d, :::: $XAP -kd, 'echo a{1} {2}b' :::: <(echo 1,2,3,) <(echo 5,6,7,8) a1 5b a2 6b a3 7b a 8 b echo '### Test -d, :::: one file too much' ### Test -d, :::: one file too much $XAP -kd, echo 'a{1}' '{2}b' :::: <(echo 1,2,3,) <(echo 5,6,7,8) <(echo 9,0) a1 5b a2 6b a3 7b a 8 b echo '### Bug: did not quote' ### Bug: did not quote $XAP echo {1} {2} :::: <(echo '>') <(echo b) > b echo '### Quote test triplet 1' ### Quote test triplet 1 $XAP -kv :::: <(echo 'echo a'; echo 'echo b') echo a a echo b b $XAP -kv -a <(echo 'echo a'; echo 'echo b') echo a a echo b b (echo 'echo a'; echo 'echo b') | $XAP -kv echo a a echo b b echo '### Quote test triplet 2' ### Quote test triplet 2 $XAP -kv echo :::: <(echo 'echo a'; echo 'echo b') echo echo\ a echo a echo echo\ b echo b $XAP -kv -a <(echo 'echo a'; echo 'echo b') echo echo echo\ a echo a echo echo\ b echo b (echo 'echo a'; echo 'echo b') | $XAP -kv echo echo echo\ a echo a echo echo\ b echo b echo '### Quoting if there is a command and 2 arg files' ### Quoting if there is a command and 2 arg files $XAP -kv echo :::: <(echo 'echo a') <(echo 'echo b') echo echo\ a echo\ b echo a echo b echo '### Quoting if there is a command and 2 arg files of uneven length' ### Quoting if there is a command and 2 arg files of uneven length $XAP -kv echo :::: <(echo 'echo a';echo a1) <(echo 'echo b') echo echo\ a echo\ b echo a echo b echo a1 echo\ b a1 echo b echo '### Quoting if there is no command and 2 arg files' ### Quoting if there is no command and 2 arg files $XAP -kv :::: <(echo 'echo a') <(echo 'echo b') echo a echo b a echo b echo '### Quoting if there is no command and 2 arg files of uneven length' ### Quoting if there is no command and 2 arg files of uneven length $XAP -kv :::: <(echo 'echo a';echo echo a1) <(echo 'echo b') echo a echo b a echo b echo a1 echo b a1 echo b echo '### Test multiple -a' ### Test multiple -a $XAP -kv -a <(echo a) -a <(echo b) echo {2} {1} echo b a b a $XAP -kv echo {2} {1} :::: <(echo a) <(echo b) echo b a b a echo '### Multiple -a: An unused file' ### Multiple -a: An unused file $XAP -kv -a <(echo a) -a <(echo b) -a <(echo c) echo {2} {1} echo b a b a $XAP -kv echo {2} {1} :::: <(echo a) <(echo b) <(echo c) echo b a b a echo '### Multiple -a: nonexistent' ### Multiple -a: nonexistent stdout $XAP -kv echo {2} {1} :::: nonexist nonexist2 parallel: Error: Cannot open input file `nonexist': No such file or directory. stdout $XAP -kv -a nonexist -a nonexist2 echo {2} {1} parallel: Error: Cannot open input file `nonexist': No such file or directory. echo '### Test {#.}' ### Test {#.} $XAP -kv -a <(echo a-noext) -a <(echo b-withext.extension) -a <(echo c-ext.gif) echo {3.} {2.} {1.} echo c-ext b-withext a-noext c-ext b-withext a-noext echo "### Tests that failed for OO-rewrite" ### Tests that failed for OO-rewrite parallel -u --semaphore seq 1 10 '|' pv -qL 20; sem --wait; echo done 1 2 3 4 5 6 7 8 9 10 done echo a | parallel echo {1} a echo "echo a" | parallel a nice parallel -j1 -I :: -X echo 'a::b::^c::[.}c' ::: 1 a1b1^c1[.}c echo "### BUG: The length for -X is not close to max (131072)" ### BUG: The length for -X is not close to max (131072) seq 1 4000 | nice parallel -k -X echo {.} aa {}{.} {}{}d{} {}dd{}d{.} |head -n 1 |wc 1 12821 131060 echo "### BUG: empty lines with --show-limit" ### BUG: empty lines with --show-limit echo | $NICEPAR --show-limits Maximal size of command: 131071 Maximal used size of command: 131071 Execution of will continue now, and it will try to read its input and run commands; if this is not what you wanted to happen, please press CTRL-D or CTRL-C echo '### Test -N' ### Test -N seq 1 5 | $NICEPAR -kN3 echo {1} {2} {3} 1 2 3 4 5 echo '### Test --arg-file-sep with files of different lengths' ### Test --arg-file-sep with files of different lengths $XAP --arg-file-sep :::: -k echo {1} {2} :::: <(seq 1 1) <(seq 3 4) 1 3 1 4 echo '### Test respect -s' ### Test respect -s $NICEPAR -kvm -IARG -s15 echo ARG ::: 1 22 333 4444 55555 666666 7777777 88888888 999999999 echo 1 22 333 1 22 333 echo 4444 4444 echo 55555 55555 echo 666666 666666 echo 7777777 7777777 echo 88888888 88888888 echo 999999999 999999999 echo '### Test eof string after :::' ### Test eof string after ::: $NICEPAR -k -E ole echo ::: foo ole bar foo echo '### Test -C and --trim rl' ### Test -C and --trim rl $NICEPAR -k -C %+ echo '"{1}_{3}_{2}_{4}"' ::: 'a% c %%b' 'a%c% b %d' a_b_c_ a_b_c_d echo '### Test empty input' ### Test empty input