2011-01-24 19:06:30 +00:00
|
|
|
### Test --pipe
|
2014-07-15 00:40:38 +00:00
|
|
|
echo '### Test 200M records with too small block'; ( echo start; seq 1 44 | parallel -uj1 cat /tmp/blocktest\;true; echo end; echo start; seq 1 44 | parallel -uj1 cat /tmp/blocktest\;true; echo end; echo start; seq 1 44 | parallel -uj1 cat /tmp/blocktest\;true; echo end; ) | stdout parallel -k --block 200m -j2 --pipe --recend 'end\n' wc -c | egrep -v '^0$'
|
2011-08-20 19:13:25 +00:00
|
|
|
### Test 200M records with too small block
|
2014-03-22 11:38:45 +00:00
|
|
|
parallel: Warning: A record was longer than 200000000. Increasing to --blocksize 260000001
|
2011-08-20 19:13:25 +00:00
|
|
|
303111434
|
|
|
|
303111434
|
|
|
|
303111434
|
2014-07-15 00:40:38 +00:00
|
|
|
echo '### Test -N with multiple jobslots and multiple args'
|
2011-01-27 21:05:34 +00:00
|
|
|
### Test -N with multiple jobslots and multiple args
|
2014-07-15 00:40:38 +00:00
|
|
|
seq 1 1 | parallel -j2 -k -N 3 --pipe 'cat;echo a'
|
2011-01-27 21:05:34 +00:00
|
|
|
1
|
|
|
|
a
|
2014-07-15 00:40:38 +00:00
|
|
|
seq 1 2 | parallel -j2 -k -N 3 --pipe 'cat;echo bb'
|
2011-01-27 21:05:34 +00:00
|
|
|
1
|
|
|
|
2
|
|
|
|
bb
|
2014-07-15 00:40:38 +00:00
|
|
|
seq 1 3 | parallel -j2 -k -N 3 --pipe 'cat;echo ccc'
|
2011-01-27 21:05:34 +00:00
|
|
|
1
|
|
|
|
2
|
|
|
|
3
|
|
|
|
ccc
|
2014-07-15 00:40:38 +00:00
|
|
|
seq 1 4 | parallel -j2 -k -N 3 --pipe 'cat;echo dddd'
|
2011-01-27 21:05:34 +00:00
|
|
|
1
|
|
|
|
2
|
|
|
|
3
|
|
|
|
dddd
|
|
|
|
4
|
|
|
|
dddd
|
2014-07-15 00:40:38 +00:00
|
|
|
seq 1 5 | parallel -j2 -k -N 3 --pipe 'cat;echo eeeee'
|
2011-01-27 21:05:34 +00:00
|
|
|
1
|
|
|
|
2
|
|
|
|
3
|
|
|
|
eeeee
|
|
|
|
4
|
|
|
|
5
|
|
|
|
eeeee
|
2014-07-15 00:40:38 +00:00
|
|
|
seq 1 6 | parallel -j2 -k -N 3 --pipe 'cat;echo ffffff'
|
2011-01-27 21:05:34 +00:00
|
|
|
1
|
|
|
|
2
|
|
|
|
3
|
|
|
|
ffffff
|
|
|
|
4
|
|
|
|
5
|
|
|
|
6
|
|
|
|
ffffff
|
2014-07-15 00:40:38 +00:00
|
|
|
seq 1 7 | parallel -j2 -k -N 3 --pipe 'cat;echo ggggggg'
|
2011-01-27 21:05:34 +00:00
|
|
|
1
|
|
|
|
2
|
|
|
|
3
|
|
|
|
ggggggg
|
|
|
|
4
|
|
|
|
5
|
|
|
|
6
|
|
|
|
ggggggg
|
|
|
|
7
|
|
|
|
ggggggg
|
2014-07-15 00:40:38 +00:00
|
|
|
seq 1 8 | parallel -j2 -k -N 3 --pipe 'cat;echo hhhhhhhh'
|
2011-01-27 21:05:34 +00:00
|
|
|
1
|
|
|
|
2
|
|
|
|
3
|
|
|
|
hhhhhhhh
|
|
|
|
4
|
|
|
|
5
|
|
|
|
6
|
|
|
|
hhhhhhhh
|
|
|
|
7
|
|
|
|
8
|
|
|
|
hhhhhhhh
|
2014-07-15 00:40:38 +00:00
|
|
|
seq 1 9 | parallel -j2 -k -N 3 --pipe 'cat;echo iiiiiiiii'
|
2011-01-27 21:05:34 +00:00
|
|
|
1
|
|
|
|
2
|
|
|
|
3
|
|
|
|
iiiiiiiii
|
|
|
|
4
|
|
|
|
5
|
|
|
|
6
|
|
|
|
iiiiiiiii
|
|
|
|
7
|
|
|
|
8
|
|
|
|
9
|
|
|
|
iiiiiiiii
|
2014-07-15 00:40:38 +00:00
|
|
|
seq 1 10 | parallel -j2 -k -N 3 --pipe 'cat;echo jjjjjjjjjj'
|
2011-01-27 21:05:34 +00:00
|
|
|
1
|
|
|
|
2
|
|
|
|
3
|
|
|
|
jjjjjjjjjj
|
|
|
|
4
|
|
|
|
5
|
|
|
|
6
|
|
|
|
jjjjjjjjjj
|
|
|
|
7
|
|
|
|
8
|
|
|
|
9
|
|
|
|
jjjjjjjjjj
|
|
|
|
10
|
|
|
|
jjjjjjjjjj
|
2014-07-15 00:40:38 +00:00
|
|
|
echo '### Test -l -N -L and -n with multiple jobslots and multiple args'
|
2011-02-21 20:44:19 +00:00
|
|
|
### Test -l -N -L and -n with multiple jobslots and multiple args
|
2014-07-15 00:40:38 +00:00
|
|
|
seq 1 12 | parallel -kj20 -l 2 --block 8 --pipe "cat; echo a"
|
2011-02-21 20:44:19 +00:00
|
|
|
1
|
|
|
|
2
|
|
|
|
3
|
|
|
|
4
|
|
|
|
a
|
|
|
|
5
|
2013-02-01 19:42:45 +00:00
|
|
|
6
|
|
|
|
7
|
|
|
|
8
|
|
|
|
a
|
|
|
|
9
|
|
|
|
10
|
|
|
|
a
|
|
|
|
11
|
|
|
|
12
|
2011-02-21 20:44:19 +00:00
|
|
|
a
|
2014-07-15 00:40:38 +00:00
|
|
|
seq 1 5 | parallel -kj2 -N 2 --pipe "cat; echo b"
|
2011-02-21 20:44:19 +00:00
|
|
|
1
|
|
|
|
2
|
|
|
|
b
|
|
|
|
3
|
|
|
|
4
|
|
|
|
b
|
|
|
|
5
|
|
|
|
b
|
2014-07-15 00:40:38 +00:00
|
|
|
seq 1 5 | parallel -kj2 -n 2 --pipe "cat; echo d"
|
2011-02-21 20:44:19 +00:00
|
|
|
1
|
|
|
|
2
|
2012-11-25 23:19:58 +00:00
|
|
|
d
|
2011-02-21 20:44:19 +00:00
|
|
|
3
|
|
|
|
4
|
2012-11-25 23:19:58 +00:00
|
|
|
d
|
2011-02-21 20:44:19 +00:00
|
|
|
5
|
2012-11-25 23:19:58 +00:00
|
|
|
d
|
2014-07-15 00:40:38 +00:00
|
|
|
echo '### Test -L --pipe'
|
2012-11-25 23:19:58 +00:00
|
|
|
### Test -L --pipe
|
2014-07-15 00:40:38 +00:00
|
|
|
seq 1 5 | parallel -kj2 -L 2 --pipe "cat; echo c"
|
2011-02-21 20:44:19 +00:00
|
|
|
1
|
|
|
|
2
|
|
|
|
3
|
|
|
|
4
|
2013-02-02 21:20:36 +00:00
|
|
|
c
|
2011-02-21 20:44:19 +00:00
|
|
|
5
|
2012-11-25 23:19:58 +00:00
|
|
|
c
|
2014-07-15 00:40:38 +00:00
|
|
|
echo '### Test output is the same for different block size'
|
2011-01-27 21:05:34 +00:00
|
|
|
### Test output is the same for different block size
|
2014-07-15 00:40:38 +00:00
|
|
|
echo -n 01a02a0a0a12a34a45a6a | parallel -k -j1 --blocksize 100 --pipe --recend a -N 3 'echo -n "$PARALLEL_SEQ>"; cat; echo; sleep 0.1'
|
2011-01-27 21:05:34 +00:00
|
|
|
1>01a02a0a
|
|
|
|
2>0a12a34a
|
|
|
|
3>45a6a
|
2014-07-15 00:40:38 +00:00
|
|
|
echo -n 01a02a0a0a12a34a45a6a | stdout parallel -k -j1 --blocksize 1 --pipe --recend a -N 3 'echo -n "$PARALLEL_SEQ>"; cat; echo; sleep 0.1'
|
2014-03-22 11:38:45 +00:00
|
|
|
parallel: Warning: A record was longer than 1. Increasing to --blocksize 3
|
|
|
|
parallel: Warning: A record was longer than 3. Increasing to --blocksize 5
|
2011-01-27 21:05:34 +00:00
|
|
|
1>01a02a0a
|
2014-03-22 11:38:45 +00:00
|
|
|
parallel: Warning: A record was longer than 5. Increasing to --blocksize 8
|
2011-01-27 21:05:34 +00:00
|
|
|
2>0a12a34a
|
|
|
|
3>45a6a
|
2014-07-15 00:40:38 +00:00
|
|
|
echo '### Test 10M records with too big block'; ( echo start; seq 1 1 | parallel -uj1 cat /tmp/blocktest\;true; echo end; echo start; seq 1 1 | parallel -uj1 cat /tmp/blocktest\;true; echo end; echo start; seq 1 1 | parallel -uj1 cat /tmp/blocktest\;true; echo end; ) | stdout parallel -k --block 10M -j2 --pipe --recstart 'start\n' wc -c | egrep -v '^0$'
|
2011-08-20 19:13:25 +00:00
|
|
|
### Test 10M records with too big block
|
2011-01-27 21:05:34 +00:00
|
|
|
6888906
|
|
|
|
6888906
|
|
|
|
6888906
|
2014-07-15 00:40:38 +00:00
|
|
|
echo '### Test --rrs -N1 --recend single'; echo 12a34a45a6 | parallel -k --pipe --recend a -N1 --rrs 'echo -n "$PARALLEL_SEQ>"; cat; echo; sleep 0.1'
|
2011-01-25 23:34:08 +00:00
|
|
|
### Test --rrs -N1 --recend single
|
|
|
|
1>12
|
|
|
|
2>34
|
|
|
|
3>45
|
|
|
|
4>6
|
|
|
|
|
2014-07-15 00:40:38 +00:00
|
|
|
echo '### Test --rrs -N1 --regexp --recend alternate'; echo 12a34b45a6 | parallel -k --pipe --regexp --recend 'a|b' -N1 --rrs 'echo -n "$PARALLEL_SEQ>"; cat; echo; sleep 0.1'
|
2011-03-01 22:04:15 +00:00
|
|
|
### Test --rrs -N1 --regexp --recend alternate
|
|
|
|
1>12
|
|
|
|
2>34
|
|
|
|
3>45
|
|
|
|
4>6
|
|
|
|
|
2014-07-15 00:40:38 +00:00
|
|
|
echo '### Test --rrs -N1 --recend single'; echo 12a34b45a6 | parallel -k --pipe --recend 'b' -N1 --rrs 'echo -n "$PARALLEL_SEQ>"; cat; echo; sleep 0.1'
|
2011-01-25 23:34:08 +00:00
|
|
|
### Test --rrs -N1 --recend single
|
|
|
|
1>12a34
|
|
|
|
2>45a6
|
|
|
|
|
2014-07-15 00:40:38 +00:00
|
|
|
echo '### Test --rrs --recend single'; echo 12a34a45a6 | parallel -k --pipe --recend a --rrs 'echo -n "$PARALLEL_SEQ>"; cat; echo; sleep 0.1'
|
2011-01-25 23:34:08 +00:00
|
|
|
### Test --rrs --recend single
|
|
|
|
1>123445
|
|
|
|
2>6
|
2011-01-26 23:29:28 +00:00
|
|
|
|
2014-07-15 00:40:38 +00:00
|
|
|
echo '### Test --rrs --regexp --recend alternate'; echo 12a34b45a6 | parallel -k --pipe --regexp --recend 'a|b' --rrs 'echo -n "$PARALLEL_SEQ>"; cat; echo; sleep 0.1'
|
2011-03-01 22:04:15 +00:00
|
|
|
### Test --rrs --regexp --recend alternate
|
|
|
|
1>123445
|
|
|
|
2>6
|
|
|
|
|
2014-07-15 00:40:38 +00:00
|
|
|
echo '### Test --rrs --recend single'; echo 12a34b45a6 | parallel -k --pipe --recend 'b' --rrs 'echo -n "$PARALLEL_SEQ>"; cat; echo; sleep 0.1'
|
2011-03-01 22:04:15 +00:00
|
|
|
### Test --rrs --recend single
|
2011-01-25 23:34:08 +00:00
|
|
|
1>12a34
|
|
|
|
2>45a6
|
2011-01-26 23:29:28 +00:00
|
|
|
|
2014-07-15 00:40:38 +00:00
|
|
|
echo '### Test -N even'; seq 1 10 | parallel -j2 -k -N 2 --pipe cat";echo ole;sleep 0.\$PARALLEL_SEQ"
|
2011-01-24 19:06:30 +00:00
|
|
|
### Test -N even
|
|
|
|
1
|
|
|
|
2
|
|
|
|
ole
|
|
|
|
3
|
|
|
|
4
|
|
|
|
ole
|
|
|
|
5
|
|
|
|
6
|
|
|
|
ole
|
|
|
|
7
|
|
|
|
8
|
|
|
|
ole
|
|
|
|
9
|
|
|
|
10
|
|
|
|
ole
|
2014-07-15 00:40:38 +00:00
|
|
|
echo '### Test -N odd'; seq 1 11 | parallel -j2 -k -N 2 --pipe cat";echo ole;sleep 0.\$PARALLEL_SEQ"
|
2011-01-24 19:06:30 +00:00
|
|
|
### Test -N odd
|
|
|
|
1
|
|
|
|
2
|
|
|
|
ole
|
|
|
|
3
|
|
|
|
4
|
|
|
|
ole
|
|
|
|
5
|
|
|
|
6
|
|
|
|
ole
|
|
|
|
7
|
|
|
|
8
|
|
|
|
ole
|
|
|
|
9
|
|
|
|
10
|
|
|
|
ole
|
|
|
|
11
|
|
|
|
ole
|
2014-07-15 00:40:38 +00:00
|
|
|
echo '### Test -N even+2'; seq 1 12 | parallel -j2 -k -N 2 --pipe cat";echo ole;sleep 1.\$PARALLEL_SEQ"
|
2011-01-24 19:06:30 +00:00
|
|
|
### Test -N even+2
|
|
|
|
1
|
|
|
|
2
|
|
|
|
ole
|
|
|
|
3
|
|
|
|
4
|
|
|
|
ole
|
|
|
|
5
|
|
|
|
6
|
|
|
|
ole
|
|
|
|
7
|
|
|
|
8
|
|
|
|
ole
|
|
|
|
9
|
|
|
|
10
|
|
|
|
ole
|
|
|
|
11
|
|
|
|
12
|
|
|
|
ole
|
2014-07-15 00:40:38 +00:00
|
|
|
echo '### Test --recstart + --recend'; cat /tmp/blocktest | parallel --block 1M -k --recstart 44 --recend "44" -j10 --pipe sort -n |md5sum
|
2011-01-24 19:06:30 +00:00
|
|
|
### Test --recstart + --recend
|
2011-01-26 23:29:28 +00:00
|
|
|
3c20e43c58152da30261c5827a1f9084 -
|
2014-07-15 00:40:38 +00:00
|
|
|
echo '### Race condition bug - 1 - would block'; seq 1 80 | nice parallel -j0 'seq 1 10| parallel --block 1 --recend "" --pipe cat;true' >/dev/null
|
2011-01-24 19:06:30 +00:00
|
|
|
### Race condition bug - 1 - would block
|
2014-07-15 00:40:38 +00:00
|
|
|
echo '### Race condition bug - 2 - would block'; seq 1 100 | nice parallel -j100 --block 1 --recend "" --pipe cat >/dev/null
|
2011-01-24 19:06:30 +00:00
|
|
|
### Race condition bug - 2 - would block
|
2014-07-15 00:40:38 +00:00
|
|
|
echo '### Test --block size=1'; seq 1 10| parallel --block 1 --files --recend "" --pipe sort -n | parallel -Xj1 sort -nm {} ";"rm {}
|
2011-01-24 19:06:30 +00:00
|
|
|
### Test --block size=1
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
0
|
|
|
|
1
|
|
|
|
1
|
|
|
|
2
|
|
|
|
3
|
|
|
|
4
|
|
|
|
5
|
|
|
|
6
|
|
|
|
7
|
|
|
|
8
|
|
|
|
9
|
2014-07-15 00:40:38 +00:00
|
|
|
echo '### Test --block size=1M -j10 --files - more jobs than data'; sort -n < /tmp/blocktest | md5sum; cat /tmp/blocktest | parallel --files --recend "\n" -j10 --pipe sort -n | parallel -Xj1 sort -nm {} ";"rm {} | md5sum
|
2011-01-24 19:06:30 +00:00
|
|
|
### Test --block size=1M -j10 --files - more jobs than data
|
|
|
|
8a7095c1c23bfadc311fe6b16d950582 -
|
|
|
|
8a7095c1c23bfadc311fe6b16d950582 -
|
2014-07-15 00:40:38 +00:00
|
|
|
echo '### Test --block size=1M -j1 - more data than cpu'; cat /tmp/blocktest | parallel --files --recend "\n" -j1 --pipe sort -n | parallel -Xj1 sort -nm {} ";"rm {} | md5sum
|
2011-01-24 19:06:30 +00:00
|
|
|
### Test --block size=1M -j1 - more data than cpu
|
|
|
|
8a7095c1c23bfadc311fe6b16d950582 -
|
2014-07-15 00:40:38 +00:00
|
|
|
echo '### Test --block size=1M -j1 - more data than cpu'; cat /tmp/blocktest | parallel --files --recend "\n" -j2 --pipe sort -n | parallel -Xj1 sort -nm {} ";"rm {} | md5sum
|
2011-01-24 19:06:30 +00:00
|
|
|
### Test --block size=1M -j1 - more data than cpu
|
|
|
|
8a7095c1c23bfadc311fe6b16d950582 -
|
2014-07-15 00:40:38 +00:00
|
|
|
echo '### Test --pipe default settings'; cat /tmp/blocktest | parallel --pipe sort | sort -n | md5sum
|
2011-01-24 19:06:30 +00:00
|
|
|
### Test --pipe default settings
|
|
|
|
8a7095c1c23bfadc311fe6b16d950582 -
|