echo 'bug #41412: --timeout + --delay causes deadlock'; bug #41412: --timeout + --delay causes deadlock seq 10 | parallel -j10 --timeout 1 --delay 0.3 echo; 1 2 3 4 5 6 7 8 9 10 parallel -j3 --timeout 1 --delay 2 echo ::: 1 2 3; 1 2 3 parallel -j10 --timeout 2.2 --delay 3 "sleep {}; echo {}" ::: 1 2 7 8 9 1 2 parallel: Warning: This job was killed because it timed out: parallel: Warning: sleep 7; echo 7 parallel: Warning: This job was killed because it timed out: parallel: Warning: sleep 8; echo 8 parallel: Warning: This job was killed because it timed out: parallel: Warning: sleep 9; echo 9 echo '### Test --spreadstdin - more procs than args'; rm -f /tmp/parallel.ss.*; seq 1 5 | stdout $NICEPAR -j 10 --spreadstdin 'cat >/tmp/parallel.ss.$PARALLEL_SEQ' >/dev/null; cat /tmp/parallel.ss.*; rm -f /tmp/parallel.ss.* ### Test --spreadstdin - more procs than args 1 2 3 4 5 echo '### Test --spreadstdin - more args than procs'; rm -f /tmp/parallel.ss2.*; seq 1 10 | stdout $NICEPAR -j 5 --spreadstdin 'cat >/tmp/parallel.ss2.$PARALLEL_SEQ' >/dev/null; cat /tmp/parallel.ss2.*; rm -f /tmp/parallel.ss2.* ### Test --spreadstdin - more args than procs 1 2 3 4 5 6 7 8 9 10 nice nice seq 1 1000 | $NICEPAR -j1 --spreadstdin cat "|cat "|wc -c 3893 nice nice seq 1 10000 | $NICEPAR -j10 --spreadstdin cat "|cat "|wc -c 48894 nice nice seq 1 100000 | $NICEPAR -j1 --spreadstdin cat "|cat "|wc -c 588895 nice nice seq 1 1000000 | $NICEPAR -j10 --spreadstdin cat "|cat "|wc -c 6888896 seq 1 10 | $NICEPAR --recend "\n" -j1 --spreadstdin gzip -9 >/tmp/foo.gz; rm /tmp/foo.gz echo '### Test --spreadstdin - similar to the failing below'; nice seq 1 100000 | $NICEPAR --recend "\n" -j10 --spreadstdin gzip -9 >/tmp/foo2.gz; diff <(nice seq 1 100000) <(zcat /tmp/foo2.gz |sort -n); diff <(nice seq 1 100000|wc -c) <(zcat /tmp/foo2.gz |wc -c); rm /tmp/foo2.gz ### Test --spreadstdin - similar to the failing below echo '### Test --spreadstdin - this failed during devel'; nice seq 1 1000000 | md5sum; nice seq 1 1000000 | $NICEPAR --recend "\n" -j10 --spreadstdin gzip -9 | zcat | sort -n | md5sum ### Test --spreadstdin - this failed during devel 8a7095c1c23bfadc311fe6b16d950582 - 8a7095c1c23bfadc311fe6b16d950582 - echo '### Test --spreadstdin -k'; nice seq 1 1000000 | $NICEPAR -k --recend "\n" -j10 --spreadstdin gzip -9 | zcat | md5sum ### Test --spreadstdin -k 8a7095c1c23bfadc311fe6b16d950582 - echo '### Test --spreadstdin --files'; nice seq 1 1000000 | shuf | $NICEPAR --files --recend "\n" -j10 --spreadstdin sort -n | parallel -Xj1 sort -nm {} ";"rm {} | md5sum ### Test --spreadstdin --files 8a7095c1c23bfadc311fe6b16d950582 - echo '### Test --tag ::: a ::: b'; stdout $NICEPAR -k --tag -j1 echo stderr-{.} ">&2;" echo stdout-{} ::: a ::: b ### Test --tag ::: a ::: b a b stdout-a b a b stderr-a b echo '### Test --tag ::: a b'; stdout $NICEPAR -k --tag -j1 echo stderr-{.} ">&2;" echo stdout-{} ::: a b ### Test --tag ::: a b a stdout-a a stderr-a b stdout-b b stderr-b echo '### Test --tag -X ::: a b'; stdout $NICEPAR -k --tag -X -j1 echo stderr-{.} ">&2;" echo stdout-{} ::: a b ### Test --tag -X ::: a b a b stdout-a stdout-b a b stderr-a stderr-b echo '### Test bash redirection <()'; ### Test bash redirection <() $NICEPAR 'cat <(echo {}); echo b' ::: a a b echo '### Test bug https://savannah.gnu.org/bugs/index.php?33352' ### Test bug https://savannah.gnu.org/bugs/index.php?33352 # produce input slowly to parallel so that it will reap a process # while blocking in read() # Having found the solution it is suddenly very easy to reproduce the # problem - even on other hardware: # # perl -e '@x=1 .. 17000; for(1..30) { print "@x\n"}' | pv -qL 200000 # |parallel -j2 --pipe --keeporder --block 150000 cat | md5sum # # This gives different md5sums for each run. # # The problem is that read(STDIN) is being interrupted by a dead # child. The chance of this happening is very small if there are few # children dying or read(STDIN) never has to wait for data. # # The test above forces data to arrive slowly (using pv) which causes # read(STDIN) to take a long time - thus being interrupted by a dead # child. echo "# md5sum - directly" # md5sum - directly nice perl -e '@x=1 .. 17000; for(1..100) { print "@x\n"}' | md5sum 350eda13a37912d755c9d733d149bdaf - echo "# parallel | md5sum" # parallel | md5sum nice nice perl -e '@x=1 .. 17000; for(1..100) { print "@x\n"}' | pv -qL 1000000 | $PAR cat | md5sum 350eda13a37912d755c9d733d149bdaf - echo "# --recend ''" # --recend '' nice nice perl -e '@x=1 .. 17000; for(1..100) { print "@x\n"}' | pv -qL 1000000 | $PAR --recend '' cat | md5sum 350eda13a37912d755c9d733d149bdaf - echo "# --recend '' --files" # --recend '' --files nice nice perl -e '@x=1 .. 17000; for(1..100) { print "@x\n"}' | pv -qL 1000000 | $PAR --recend '' --files cat | parallel -Xj1 cat {} ';' rm {} | md5sum 350eda13a37912d755c9d733d149bdaf - echo "# --recend '' --files --tmpdir" # --recend '' --files --tmpdir nice nice perl -e '@x=1 .. 17000; for(1..100) { print "@x\n"}' | pv -qL 1000000 | $PAR --recend '' --files --tmpdir /dev/shm cat | parallel -Xj1 cat {} ';' rm {} | md5sum 350eda13a37912d755c9d733d149bdaf - echo "# --recend '' --files --halt-on-error" # --recend '' --files --halt-on-error nice nice perl -e '@x=1 .. 17000; for(1..100) { print "@x\n"}' | pv -qL 1000000 | $PAR --recend '' --files --halt-on-error 2 cat | parallel -Xj1 cat {} ';' rm {} | md5sum 350eda13a37912d755c9d733d149bdaf - echo '### Test ::::' ### Test :::: echo '### Change --arg-file-sep' ### Change --arg-file-sep $XAP --arg-file-sep :::: -k echo {1} {2} :::: <(seq 1 10) <(seq 5 15) 1 5 2 6 3 7 4 8 5 9 6 10 7 11 8 12 9 13 10 14 1 15 $XAP --arg-file-sep .--- -k echo {1} {2} .--- <(seq 1 10) <(seq 5 15) 1 5 2 6 3 7 4 8 5 9 6 10 7 11 8 12 9 13 10 14 1 15 $XAP --argfilesep :::: -k echo {1} {2} :::: <(seq 1 10) <(seq 5 15) 1 5 2 6 3 7 4 8 5 9 6 10 7 11 8 12 9 13 10 14 1 15 $XAP --argfilesep .--- -k echo {1} {2} .--- <(seq 1 10) <(seq 5 15) 1 5 2 6 3 7 4 8 5 9 6 10 7 11 8 12 9 13 10 14 1 15 echo '### Test xapply --max-replace-args' ### Test xapply --max-replace-args seq 0 7 | $XAP -k --max-replace-args=3 echo {3} {2} {1} 2 1 0 5 4 3 7 6 echo '### Test -N' ### Test -N seq 1 5 | $XAP -kN3 echo {1} {2} {3} 1 2 3 4 5 echo '### Test -N with 0' ### Test -N with 0 seq 0 7 | $XAP -kN3 echo {1} {2} {3} 0 1 2 3 4 5 6 7 echo '### Test :::: on nonexistent' ### Test :::: on nonexistent stdout $XAP -k echo {1} {2} {3} :::: nonexistent parallel: Error: Cannot open input file `nonexistent': No such file or directory. echo '### Test :::: two files' ### Test :::: two files $XAP -k echo {1} {2} :::: <(seq 1 10) <(seq 5 15) 1 5 2 6 3 7 4 8 5 9 6 10 7 11 8 12 9 13 10 14 1 15 echo '### Test -d, ::::' ### Test -d, :::: $XAP -kd, 'echo a{1} {2}b' :::: <(echo 1,2,3,) <(echo 5,6,7,8) a1 5b a2 6b a3 7b a 8 b echo '### Test -d, :::: one file too much' ### Test -d, :::: one file too much $XAP -kd, echo 'a{1}' '{2}b' :::: <(echo 1,2,3,) <(echo 5,6,7,8) <(echo 9,0) a1 5b a2 6b a3 7b a 8 b echo '### Bug: did not quote' ### Bug: did not quote $XAP echo {1} {2} :::: <(echo '>') <(echo b) > b echo '### Quote test triplet 1' ### Quote test triplet 1 $XAP -kv :::: <(echo 'echo a'; echo 'echo b') echo a a echo b b $XAP -kv -a <(echo 'echo a'; echo 'echo b') echo a a echo b b (echo 'echo a'; echo 'echo b') | $XAP -kv echo a a echo b b echo '### Quote test triplet 2' ### Quote test triplet 2 $XAP -kv echo :::: <(echo 'echo a'; echo 'echo b') echo 'echo a' echo a echo 'echo b' echo b $XAP -kv -a <(echo 'echo a'; echo 'echo b') echo echo 'echo a' echo a echo 'echo b' echo b (echo 'echo a'; echo 'echo b') | $XAP -kv echo echo 'echo a' echo a echo 'echo b' echo b echo '### Quoting if there is a command and 2 arg files' ### Quoting if there is a command and 2 arg files $XAP -kv echo :::: <(echo 'echo a') <(echo 'echo b') echo 'echo a' 'echo b' echo a echo b echo '### Quoting if there is a command and 2 arg files of uneven length' ### Quoting if there is a command and 2 arg files of uneven length $XAP -kv echo :::: <(echo 'echo a';echo a1) <(echo 'echo b') echo 'echo a' 'echo b' echo a echo b echo a1 'echo b' a1 echo b echo '### Quoting if there is no command and 2 arg files' ### Quoting if there is no command and 2 arg files $XAP -kv :::: <(echo 'echo a') <(echo 'echo b') echo a echo b a echo b echo '### Quoting if there is no command and 2 arg files of uneven length' ### Quoting if there is no command and 2 arg files of uneven length $XAP -kv :::: <(echo 'echo a';echo echo a1) <(echo 'echo b') echo a echo b a echo b echo a1 echo b a1 echo b echo '### Test multiple -a' ### Test multiple -a $XAP -kv -a <(echo a) -a <(echo b) echo {2} {1} echo b a b a $XAP -kv echo {2} {1} :::: <(echo a) <(echo b) echo b a b a echo '### Multiple -a: An unused file' ### Multiple -a: An unused file $XAP -kv -a <(echo a) -a <(echo b) -a <(echo c) echo {2} {1} echo b a b a $XAP -kv echo {2} {1} :::: <(echo a) <(echo b) <(echo c) echo b a b a echo '### Multiple -a: nonexistent' ### Multiple -a: nonexistent stdout $XAP -kv echo {2} {1} :::: nonexist nonexist2 parallel: Error: Cannot open input file `nonexist': No such file or directory. stdout $XAP -kv -a nonexist -a nonexist2 echo {2} {1} parallel: Error: Cannot open input file `nonexist': No such file or directory. echo '### Test {#.}' ### Test {#.} $XAP -kv -a <(echo a-noext) -a <(echo b-withext.extension) -a <(echo c-ext.gif) echo {3.} {2.} {1.} echo c-ext b-withext a-noext c-ext b-withext a-noext echo "### Tests that failed for OO-rewrite" ### Tests that failed for OO-rewrite parallel -u --semaphore --id local9 seq 1 10 '|' pv -qL 20; sem --id local9 --wait; echo done 1 2 3 4 5 6 7 8 9 10 done echo a | parallel echo {1} a echo "echo a" | parallel a nice parallel -j1 -I :: -X echo 'a::b::^c::[.}c' ::: 1 a1b1^c1[.}c echo "### BUG: The length for -X is not close to max (131072)" ### BUG: The length for -X is not close to max (131072) seq 1 4000 | nice parallel -k -X echo {.} aa {}{.} {}{}d{} {}dd{}d{.} |head -n 1 |wc 1 6725 64004 echo '### Test -N' ### Test -N seq 1 5 | $NICEPAR -kN3 echo {1} {2} {3} 1 2 3 4 5 echo '### Test --arg-file-sep with files of different lengths' ### Test --arg-file-sep with files of different lengths $XAP --arg-file-sep :::: -k echo {1} {2} :::: <(seq 1 1) <(seq 3 4) 1 3 1 4 echo '### Test respect -s' ### Test respect -s $NICEPAR -kvm -IARG -s15 echo ARG ::: 1 22 333 4444 55555 666666 7777777 88888888 999999999 echo 1 22 333 1 22 333 echo 4444 4444 echo 55555 55555 echo 666666 666666 echo 7777777 7777777 echo 88888888 88888888 echo 999999999 999999999 echo '### Test eof string after :::' ### Test eof string after ::: $NICEPAR -k -E ole echo ::: foo ole bar foo echo '### Test -C and --trim rl' ### Test -C and --trim rl $NICEPAR -k -C %+ echo '"{1}_{3}_{2}_{4}"' ::: 'a% c %%b' 'a%c% b %d' a_b_c_ a_b_c_d echo '### Test empty input' ### Test empty input