mirror of
https://git.savannah.gnu.org/git/parallel.git
synced 2024-11-25 15:37:56 +00:00
5bba5c922d
parallel: $PARALLEL_SSHLOGIN = sshlogin with number of cores removed.
parallel: $PARALLEL_SSHHOST = the host part of an sshlogin.
parallel: {slot} {sshlogin} {host} added to --plus.
1431 lines
46 KiB
Plaintext
1431 lines
46 KiB
Plaintext
### test parallel_tutorial
|
|
fetch -o - http://pi.dk/3 ) > install.sh
|
|
$ sha1sum install.sh
|
|
12345678 3374ec53 bacb199b 245af2dd a86df6c9
|
|
$ md5sum install.sh
|
|
029a9ac0 6e8b5bc6 052eac57 b2c3c9ca
|
|
$ sha512sum install.sh
|
|
40f53af6 9e20dae5 713ba06c f517006d 9897747b ed8a4694 b1acba1b 1464beb4
|
|
60055629 3f2356f3 3e9c4e3c 76e3f3af a9db4b32 bd33322b 975696fc e6b23cfb
|
|
$ bash install.sh
|
|
/bin/bash: -c: line 0: syntax error near unexpected token `)'
|
|
/bin/bash: -c: line 0: ` fetch -o - http://pi.dk/3 ) > install.sh'
|
|
parallel -k echo ::: A B C > abc-file
|
|
parallel -k echo ::: D E F > def-file
|
|
perl -e 'printf "A\0B\0C\0"' > abc0-file
|
|
perl -e 'printf "A_B_C_"' > abc_-file
|
|
perl -e 'printf "f1\tf2\nA\tB\nC\tD\n"' > tsv-file.tsv
|
|
perl -e 'for(1..8){print "$_\n"}' > num8
|
|
perl -e 'for(1..128){print "$_\n"}' > num128
|
|
perl -e 'for(1..30000){print "$_\n"}' > num30000
|
|
perl -e 'for(1..1000000){print "$_\n"}' > num1000000
|
|
(echo %head1; echo %head2; \
|
|
sleep .3
|
|
/bin/bash: -c: line 2: syntax error: unexpected end of file
|
|
sleep .3
|
|
perl -e 'for(1..10){print "$_\n"}') > num_%header
|
|
/bin/bash: -c: line 0: syntax error near unexpected token `)'
|
|
/bin/bash: -c: line 0: ` perl -e 'for(1..10){print "$_\n"}') > num_%header'
|
|
perl -e 'print "HHHHAAABBBCCC"' > fixedlen
|
|
parallel echo ::: A B C
|
|
A
|
|
B
|
|
C
|
|
parallel -a abc-file echo
|
|
A
|
|
B
|
|
C
|
|
cat abc-file | parallel echo
|
|
A
|
|
B
|
|
C
|
|
parallel echo ::: A B C ::: D E F
|
|
A D
|
|
A E
|
|
A F
|
|
B D
|
|
B E
|
|
B F
|
|
C D
|
|
C E
|
|
C F
|
|
parallel -a abc-file -a def-file echo
|
|
A D
|
|
A E
|
|
A F
|
|
B D
|
|
B E
|
|
B F
|
|
C D
|
|
C E
|
|
C F
|
|
cat abc-file | parallel -a - -a def-file echo
|
|
A D
|
|
A E
|
|
A F
|
|
B D
|
|
B E
|
|
B F
|
|
C D
|
|
C E
|
|
C F
|
|
cat abc-file | parallel echo :::: - def-file
|
|
A D
|
|
A E
|
|
A F
|
|
B D
|
|
B E
|
|
B F
|
|
C D
|
|
C E
|
|
C F
|
|
parallel echo ::: A B C :::: def-file
|
|
A D
|
|
A E
|
|
A F
|
|
B D
|
|
B E
|
|
B F
|
|
C D
|
|
C E
|
|
C F
|
|
parallel --link echo ::: A B C ::: D E F
|
|
A D
|
|
B E
|
|
C F
|
|
parallel --link echo ::: A B C D E ::: F G
|
|
A F
|
|
B G
|
|
C F
|
|
D G
|
|
E F
|
|
parallel echo :::: abc-file :::+ G H I :::: def-file
|
|
A G D
|
|
A G E
|
|
A G F
|
|
B H D
|
|
B H E
|
|
B H F
|
|
C I D
|
|
C I E
|
|
C I F
|
|
parallel echo :::: abc-file ::: G H I ::::+ def-file
|
|
A G D
|
|
A H E
|
|
A I F
|
|
B G D
|
|
B H E
|
|
B I F
|
|
C G D
|
|
C H E
|
|
C I F
|
|
parallel echo ::: A B C D E :::+ F G
|
|
A F
|
|
B G
|
|
parallel --arg-sep ,, echo ,, A B C :::: def-file
|
|
A D
|
|
A E
|
|
A F
|
|
B D
|
|
B E
|
|
B F
|
|
C D
|
|
C E
|
|
C F
|
|
parallel --arg-file-sep // echo ::: A B C // def-file
|
|
A D
|
|
A E
|
|
A F
|
|
B D
|
|
B E
|
|
B F
|
|
C D
|
|
C E
|
|
C F
|
|
parallel -d _ echo :::: abc_-file
|
|
A
|
|
B
|
|
C
|
|
parallel -d '\0' echo :::: abc0-file
|
|
A
|
|
B
|
|
C
|
|
parallel -E stop echo ::: A B stop C D
|
|
A
|
|
B
|
|
(echo 1; echo; echo 2) | parallel --no-run-if-empty echo
|
|
9
|
|
parallel ::: ls 'echo foo' pwd
|
|
abc-file
|
|
abc0-file
|
|
abc_-file
|
|
def-file
|
|
fixedlen
|
|
num1000000
|
|
num128
|
|
num30000
|
|
num8
|
|
tsv-file.tsv
|
|
foo
|
|
~/privat/parallel/testsuite/tmp
|
|
my_func() {
|
|
echo in my_func $1
|
|
}
|
|
export -f my_func
|
|
parallel my_func ::: 1 2 3
|
|
in my_func 1
|
|
in my_func 2
|
|
in my_func 3
|
|
parallel echo ::: A/B.C
|
|
A/B.C
|
|
parallel echo {} ::: A/B.C
|
|
A/B.C
|
|
parallel echo {.} ::: A/B.C
|
|
A/B
|
|
parallel echo {/} ::: A/B.C
|
|
B.C
|
|
parallel echo {//} ::: A/B.C
|
|
A
|
|
parallel echo {/.} ::: A/B.C
|
|
B
|
|
parallel echo {#} ::: A B C
|
|
9
|
|
parallel -j 2 echo {%} ::: A B C
|
|
9
|
|
parallel -I ,, echo ,, ::: A/B.C
|
|
A/B.C
|
|
parallel --extensionreplace ,, echo ,, ::: A/B.C
|
|
A/B
|
|
parallel --basenamereplace ,, echo ,, ::: A/B.C
|
|
B.C
|
|
parallel --dirnamereplace ,, echo ,, ::: A/B.C
|
|
A
|
|
parallel --basenameextensionreplace ,, echo ,, ::: A/B.C
|
|
B
|
|
parallel --seqreplace ,, echo ,, ::: A B C
|
|
9
|
|
parallel -j2 --slotreplace ,, echo ,, ::: A B C
|
|
9
|
|
parallel echo '{= s:\.[^.]+$::;s:\.[^.]+$::; =}' ::: foo.tar.gz
|
|
foo
|
|
parallel echo Job {#} of {= '$_=total_jobs()' =} ::: {1..5}
|
|
Job 1 of 5
|
|
Job 2 of 5
|
|
Job 3 of 5
|
|
Job 4 of 5
|
|
Job 5 of 5
|
|
parallel echo {} shell quoted is {= '$_=Q($_)' =} ::: '*/!#$'
|
|
*/!#$ shell quoted is '*/!#$'
|
|
parallel echo {= 'if($_==3) { skip() }' =} ::: {1..5}
|
|
9
|
|
parallel echo {= 'if($arg[1]==$arg[2]) { skip() }' =} \
|
|
::: {1..3} ::: {1..3}
|
|
1 2
|
|
1 3
|
|
2 1
|
|
2 3
|
|
3 1
|
|
3 2
|
|
parallel --parens ,,,, echo ',, s:\.[^.]+$::;s:\.[^.]+$::; ,,' \
|
|
::: foo.tar.gz
|
|
foo
|
|
parallel --rpl '.. s:\.[^.]+$::;s:\.[^.]+$::;' echo '..' \
|
|
::: foo.tar.gz
|
|
foo
|
|
parallel --rpl '{..} s:\.[^.]+$::;s:\.[^.]+$::;' echo '{..}'
|
|
::: foo.tar.gz
|
|
/bin/bash: line 1: :::: command not found
|
|
parallel --rpl '{%(.+?)} s/$$1$//;' echo {%.tar.gz}.zip ::: foo.tar.gz
|
|
foo.zip
|
|
parallel --rpl '{/(.+?)/(.*?)} s/$$1/$$2/;' echo {/tar.gz/zip} \
|
|
::: foo.tar.gz
|
|
foo.zip
|
|
--rpl '{} '
|
|
--rpl '{#} $_=$job->seq()'
|
|
--rpl '{%} $_=$job->slot()'
|
|
--rpl '{/} s:.*/::'
|
|
--rpl '{//} $Global::use{"File::Basename"} ||=
|
|
eval "use File::Basename; 1;"; $_ = dirname($_);'
|
|
--rpl '{/.} s:.*/::; s:\.[^/.]+$::;'
|
|
--rpl '{.} s:\.[^/.]+$::'
|
|
/bin/bash: --rpl: command not found
|
|
/bin/bash: line 1: --rpl: command not found
|
|
/bin/bash: line 2: --rpl: command not found
|
|
/bin/bash: line 3: --rpl: command not found
|
|
/bin/bash: line 5: --rpl: command not found
|
|
/bin/bash: line 6: --rpl: command not found
|
|
/bin/bash: line 7: --rpl: command not found
|
|
parallel echo {1} and {2} ::: A B ::: C D
|
|
A and C
|
|
A and D
|
|
B and C
|
|
B and D
|
|
parallel echo /={1/} //={1//} /.={1/.} .={1.} ::: A/B.C D/E.F
|
|
/=B.C //=A /.=B .=A/B
|
|
/=E.F //=D /.=E .=D/E
|
|
parallel echo 1={1} 2={2} 3={3} -1={-1} -2={-2} -3={-3} \
|
|
::: A B ::: C D ::: E F
|
|
1=A 2=C 3=E -1=E -2=C -3=A
|
|
1=A 2=C 3=F -1=F -2=C -3=A
|
|
1=A 2=D 3=E -1=E -2=D -3=A
|
|
1=A 2=D 3=F -1=F -2=D -3=A
|
|
1=B 2=C 3=E -1=E -2=C -3=B
|
|
1=B 2=C 3=F -1=F -2=C -3=B
|
|
1=B 2=D 3=E -1=E -2=D -3=B
|
|
1=B 2=D 3=F -1=F -2=D -3=B
|
|
parallel echo '{=2 s:\.[^.]+$::;s:\.[^.]+$::; =} {1}' \
|
|
::: bar ::: foo.tar.gz
|
|
foo bar
|
|
parallel --rpl '{..} s:\.[^.]+$::;s:\.[^.]+$::;' echo '{2..} {1}' \
|
|
::: bar ::: foo.tar.gz
|
|
foo bar
|
|
parallel --colsep '\t' echo 1={1} 2={2} :::: tsv-file.tsv
|
|
1=f1 2=f2
|
|
1=A 2=B
|
|
1=C 2=D
|
|
parallel --header : echo f1={f1} f2={f2} ::: f1 A B ::: f2 C D
|
|
f1=A f2=C
|
|
f1=A f2=D
|
|
f1=B f2=C
|
|
f1=B f2=D
|
|
parallel --header : --colsep '\t' echo f1={f1} f2={f2} \
|
|
:::: tsv-file.tsv
|
|
f1=A f2=B
|
|
f1=C f2=D
|
|
parallel --plus echo {} ::: dir/sub/file.ex1.ex2.ex3
|
|
parallel --plus echo {+/}/{/} ::: dir/sub/file.ex1.ex2.ex3
|
|
parallel --plus echo {.}.{+.} ::: dir/sub/file.ex1.ex2.ex3
|
|
parallel --plus echo {+/}/{/.}.{+.} ::: dir/sub/file.ex1.ex2.ex3
|
|
parallel --plus echo {..}.{+..} ::: dir/sub/file.ex1.ex2.ex3
|
|
parallel --plus echo {+/}/{/..}.{+..} ::: dir/sub/file.ex1.ex2.ex3
|
|
parallel --plus echo {...}.{+...} ::: dir/sub/file.ex1.ex2.ex3
|
|
parallel --plus echo {+/}/{/...}.{+...} ::: dir/sub/file.ex1.ex2.ex3
|
|
dir/sub/file.ex1.ex2.ex3
|
|
parallel --plus echo Job {#} of {##} ::: {1..5}
|
|
Job 1 of 5
|
|
Job 2 of 5
|
|
Job 3 of 5
|
|
Job 4 of 5
|
|
Job 5 of 5
|
|
unset myvar
|
|
echo ${myvar:-myval}
|
|
parallel --plus echo {:-myval} ::: "$myvar"
|
|
myval
|
|
myvar=abcAaAdef
|
|
echo ${myvar:2}
|
|
parallel --plus echo {:2} ::: "$myvar"
|
|
cAaAdef
|
|
echo ${myvar:2:3}
|
|
parallel --plus echo {:2:3} ::: "$myvar"
|
|
echo ${myvar#bc}
|
|
parallel --plus echo {#bc} ::: "$myvar"
|
|
echo ${myvar#abc}
|
|
parallel --plus echo {#abc} ::: "$myvar"
|
|
echo ${myvar%de}
|
|
parallel --plus echo {%de} ::: "$myvar"
|
|
echo ${myvar%def}
|
|
parallel --plus echo {%def} ::: "$myvar"
|
|
echo ${myvar/def/ghi}
|
|
parallel --plus echo {/def/ghi} ::: "$myvar"
|
|
echo ${myvar^a}
|
|
parallel --plus echo {^a} ::: "$myvar"
|
|
echo ${myvar^^a}
|
|
parallel --plus echo {^^a} ::: "$myvar"
|
|
myvar=AbcAaAdef
|
|
echo ${myvar,A}
|
|
parallel --plus echo '{,A}' ::: "$myvar"
|
|
echo ${myvar,,A}
|
|
parallel --plus echo '{,,A}' ::: "$myvar"
|
|
abcAaAdef
|
|
abcaaadef
|
|
cat num30000 | parallel --xargs echo | wc -l
|
|
9
|
|
cat num30000 | parallel --xargs -s 10000 echo | wc -l
|
|
99
|
|
cat num30000 | parallel --jobs 4 -m echo | wc -l
|
|
9
|
|
parallel --jobs 4 -m echo ::: 1 2 3 4 5 6 7 8 9 10
|
|
1 2 3
|
|
4 5 6
|
|
7 8 9
|
|
99
|
|
parallel --jobs 4 -X echo pre-{}-post ::: A B C D E F G
|
|
pre-A-post pre-B-post
|
|
pre-C-post pre-D-post
|
|
pre-E-post pre-F-post
|
|
pre-G-post
|
|
parallel -N3 echo ::: A B C D E F G H
|
|
A B C
|
|
D E F
|
|
G H
|
|
parallel -N3 echo 1={1} 2={2} 3={3} ::: A B C D E F G H
|
|
1=A 2=B 3=C
|
|
1=D 2=E 3=F
|
|
1=G 2=H 3=
|
|
parallel -N0 echo foo ::: 1 2 3
|
|
foo
|
|
perl -e 'print "@ARGV\n"' A
|
|
A
|
|
parallel perl -e 'print "@ARGV\n"' ::: This wont work
|
|
Warning: Input is read from the terminal. You either know what you
|
|
Warning: are doing (in which case: YOU ARE AWESOME!) or you forgot
|
|
Warning: ::: or :::: or to pipe data into parallel. If so
|
|
Warning: Press CTRL-D to exit.
|
|
perl -e 'print "@ARGV\n"'
|
|
[CTRL-D]
|
|
/bin/bash: line 1: Warning:: command not found
|
|
/bin/bash: -c: line 2: syntax error near unexpected token `('
|
|
/bin/bash: -c: line 2: ` Warning: are doing (in which case: YOU ARE AWESOME!) or you forgot'
|
|
parallel --trim r echo pre-{}-post ::: ' A '
|
|
pre- A-post
|
|
parallel --trim l echo pre-{}-post ::: ' A '
|
|
pre-A -post
|
|
parallel --trim lr echo pre-{}-post ::: ' A '
|
|
pre-A-post
|
|
parallel echo \={} ::: zsh bash ls
|
|
=zsh
|
|
=bash
|
|
=ls
|
|
parallel 'set a="{}"; if( { test -d "$a" } ) echo "$a is a dir"' ::: *
|
|
/bin/bash: -c: line 0: syntax error near unexpected token `)'
|
|
/bin/bash: -c: line 0: `set a="abc-file"; if( { test -d "$a" } ) echo "$a is a dir"'
|
|
/bin/bash: -c: line 0: syntax error near unexpected token `)'
|
|
/bin/bash: -c: line 0: `set a="abc0-file"; if( { test -d "$a" } ) echo "$a is a dir"'
|
|
/bin/bash: -c: line 0: syntax error near unexpected token `)'
|
|
/bin/bash: -c: line 0: `set a="abc_-file"; if( { test -d "$a" } ) echo "$a is a dir"'
|
|
/bin/bash: -c: line 0: syntax error near unexpected token `)'
|
|
/bin/bash: -c: line 0: `set a="def-file"; if( { test -d "$a" } ) echo "$a is a dir"'
|
|
/bin/bash: -c: line 0: syntax error near unexpected token `)'
|
|
/bin/bash: -c: line 0: `set a="fixedlen"; if( { test -d "$a" } ) echo "$a is a dir"'
|
|
/bin/bash: -c: line 0: syntax error near unexpected token `)'
|
|
/bin/bash: -c: line 0: `set a="num1000000"; if( { test -d "$a" } ) echo "$a is a dir"'
|
|
/bin/bash: -c: line 0: syntax error near unexpected token `)'
|
|
/bin/bash: -c: line 0: `set a="num128"; if( { test -d "$a" } ) echo "$a is a dir"'
|
|
/bin/bash: -c: line 0: syntax error near unexpected token `)'
|
|
/bin/bash: -c: line 0: `set a="num30000"; if( { test -d "$a" } ) echo "$a is a dir"'
|
|
/bin/bash: -c: line 0: syntax error near unexpected token `)'
|
|
/bin/bash: -c: line 0: `set a="num8"; if( { test -d "$a" } ) echo "$a is a dir"'
|
|
/bin/bash: -c: line 0: syntax error near unexpected token `)'
|
|
/bin/bash: -c: line 0: `set a="tsv-file.tsv"; if( { test -d "$a" } ) echo "$a is a dir"'
|
|
parallel --tag echo foo-{} ::: A B C
|
|
A foo-A
|
|
B foo-B
|
|
C foo-C
|
|
parallel --tagstring {}-bar echo foo-{} ::: A B C
|
|
A-bar foo-A
|
|
B-bar foo-B
|
|
C-bar foo-C
|
|
parallel --dryrun echo {} ::: A B C
|
|
echo A
|
|
echo B
|
|
echo C
|
|
parallel --verbose echo {} ::: A B C
|
|
A
|
|
B
|
|
C
|
|
echo A
|
|
echo B
|
|
echo C
|
|
parallel -j2 'printf "%s-start\n%s" {} {};
|
|
sleep {};printf "%s\n" -middle;echo {}-end' ::: 4 2 1
|
|
4-start
|
|
2-start
|
|
2-middle
|
|
2-end
|
|
1-start
|
|
1-middle
|
|
1-end
|
|
parallel -j2 --ungroup 'printf "%s-start\n%s" {} {};
|
|
sleep {};printf "%s\n" -middle;echo {}-end' ::: 4 2 1
|
|
4-start
|
|
42-start
|
|
2-middle
|
|
2-end
|
|
1-start
|
|
1-middle
|
|
1-end
|
|
-middle
|
|
parallel -j2 --linebuffer 'printf "%s-start\n%s" {} {};
|
|
sleep {};printf "%s\n" -middle;echo {}-end' ::: 4 2 1
|
|
4-start
|
|
2-start
|
|
2-middle
|
|
2-end
|
|
1-start
|
|
1-middle
|
|
1-end
|
|
parallel -j2 -k 'printf "%s-start\n%s" {} {};
|
|
sleep {};printf "%s\n" -middle;echo {}-end' ::: 4 2 1
|
|
4-start
|
|
2-start
|
|
2-middle
|
|
2-end
|
|
1-start
|
|
1-middle
|
|
1-end
|
|
parallel --files echo ::: A B C
|
|
/tmp/parallel-tutorial-tmpdir/parXXXXX.par
|
|
parallel --tmpdir /var/tmp --files echo ::: A B C
|
|
/var/tmp/parXXXXX.par
|
|
parallel --results outdir echo ::: A B C
|
|
A
|
|
B
|
|
C
|
|
outdir/1/A/seq
|
|
outdir/1/A/stderr
|
|
outdir/1/A/stdout
|
|
outdir/1/B/seq
|
|
outdir/1/B/stderr
|
|
outdir/1/B/stdout
|
|
outdir/1/C/seq
|
|
outdir/1/C/stderr
|
|
outdir/1/C/stdout
|
|
/bin/bash: outdir/1/A/seq: No such file or directory
|
|
/bin/bash: line 1: outdir/1/A/stderr: No such file or directory
|
|
/bin/bash: line 2: outdir/1/A/stdout: No such file or directory
|
|
/bin/bash: line 3: outdir/1/B/seq: No such file or directory
|
|
/bin/bash: line 4: outdir/1/B/stderr: No such file or directory
|
|
/bin/bash: line 5: outdir/1/B/stdout: No such file or directory
|
|
/bin/bash: line 6: outdir/1/C/seq: No such file or directory
|
|
/bin/bash: line 7: outdir/1/C/stderr: No such file or directory
|
|
/bin/bash: line 8: outdir/1/C/stdout: No such file or directory
|
|
parallel --header : --results outdir echo ::: f1 A B ::: f2 C D
|
|
A C
|
|
A D
|
|
B C
|
|
B D
|
|
outdir/f1/A/f2/C/seq
|
|
outdir/f1/A/f2/C/stderr
|
|
outdir/f1/A/f2/C/stdout
|
|
outdir/f1/A/f2/D/seq
|
|
outdir/f1/A/f2/D/stderr
|
|
outdir/f1/A/f2/D/stdout
|
|
outdir/f1/B/f2/C/seq
|
|
outdir/f1/B/f2/C/stderr
|
|
outdir/f1/B/f2/C/stdout
|
|
outdir/f1/B/f2/D/seq
|
|
outdir/f1/B/f2/D/stderr
|
|
outdir/f1/B/f2/D/stdout
|
|
/bin/bash: outdir/f1/A/f2/C/seq: No such file or directory
|
|
/bin/bash: line 1: outdir/f1/A/f2/C/stderr: No such file or directory
|
|
/bin/bash: line 2: outdir/f1/A/f2/C/stdout: No such file or directory
|
|
/bin/bash: line 3: outdir/f1/A/f2/D/seq: No such file or directory
|
|
/bin/bash: line 4: outdir/f1/A/f2/D/stderr: No such file or directory
|
|
/bin/bash: line 5: outdir/f1/A/f2/D/stdout: No such file or directory
|
|
/bin/bash: line 6: outdir/f1/B/f2/C/seq: No such file or directory
|
|
/bin/bash: line 7: outdir/f1/B/f2/C/stderr: No such file or directory
|
|
/bin/bash: line 8: outdir/f1/B/f2/C/stdout: No such file or directory
|
|
/bin/bash: line 9: outdir/f1/B/f2/D/seq: No such file or directory
|
|
/bin/bash: line 10: outdir/f1/B/f2/D/stderr: No such file or directory
|
|
/bin/bash: line 11: outdir/f1/B/f2/D/stdout: No such file or directory
|
|
/usr/bin/time -f %e parallel -N0 -j64 sleep 1 :::: num128
|
|
9
|
|
/usr/bin/time -f %e parallel -N0 sleep 1 :::: num128
|
|
99
|
|
/usr/bin/time -f %e parallel -N0 --jobs 200% sleep 1 :::: num128
|
|
9
|
|
/usr/bin/time -f %e parallel -N0 --jobs 0 sleep 1 :::: num128
|
|
9
|
|
echo 50% > my_jobs
|
|
/usr/bin/time -f %e parallel -N0 --jobs my_jobs sleep 1 :::: num128 &
|
|
sleep 1
|
|
echo 0 > my_jobs
|
|
wait
|
|
9
|
|
parallel --use-cpus-instead-of-cores -N0 sleep 1 :::: num8
|
|
parallel --shuf echo ::: 1 2 3 ::: a b c ::: A B C
|
|
123 abc ABC
|
|
seq 10 20 | parallel --tmux 'echo start {}; sleep {}; echo done {}'
|
|
See output with: tmux -S /tmp/parallel-tutorial-tmpdir/tmsXXXXX attach
|
|
tmux -S /tmp/tmsXXXXX attach
|
|
no sessions
|
|
parallel --delay 2.5 echo Starting {}\;date ::: 1 2 3
|
|
Starting 1
|
|
DATE OUTPUT
|
|
Starting 2
|
|
DATE OUTPUT
|
|
Starting 3
|
|
DATE OUTPUT
|
|
parallel --timeout 4.1 sleep {}\; echo {} ::: 2 4 6 8
|
|
9
|
|
parallel: Warning: This job was killed because it timed out:
|
|
parallel: Warning: sleep 6; echo 6
|
|
parallel: Warning: This job was killed because it timed out:
|
|
parallel: Warning: sleep 8; echo 8
|
|
parallel --timeout 200% sleep {}\; echo {} ::: 2.1 2.2 3 7 2.3
|
|
9
|
|
parallel: Warning: This job was killed because it timed out:
|
|
parallel: Warning: sleep 7; echo 7
|
|
parallel --eta sleep ::: 1 3 2 2 1 3 3 2 1
|
|
Computers / CPU cores / Max jobs to run
|
|
1:local / 8 / 8
|
|
Computer:jobs running/jobs completed/%of started jobs/Average seconds to complete
|
|
parallel --progress sleep ::: 1 3 2 2 1 3 3 2 1
|
|
Computers / CPU cores / Max jobs to run
|
|
1:local / 8 / 8
|
|
Computer:jobs running/jobs completed/%of started jobs/Average seconds to complete
|
|
seq 1000 | parallel -j10 --bar '(echo -n {};sleep 0.1)' \
|
|
2> >(zenity --timeout=15 --progress --auto-kill --auto-close)
|
|
parallel --joblog /tmp/log exit ::: 1 2 3 0
|
|
cat /tmp/log;
|
|
parallel --joblog /tmp/log exit ::: 1 2 3 0
|
|
cat /tmp/log; parallel --resume --joblog /tmp/log exit ::: 1 2 3 0 0 0
|
|
cat /tmp/log;
|
|
parallel --resume-failed --joblog /tmp/log exit ::: 1 2 3 0 0 0
|
|
cat /tmp/log;
|
|
parallel --retry-failed --joblog /tmp/log
|
|
cat /tmp/log;
|
|
parallel -j2 --halt soon,fail=1 echo {}\; exit {} ::: 0 0 1 2 3
|
|
Seq Host Starttime JobRuntime Send Receive Exitval Signal Command
|
|
1 : TIMESTAMP 9.999 0 0 1 0 exit 1
|
|
2 : TIMESTAMP 9.999 0 0 2 0 exit 2
|
|
3 : TIMESTAMP 9.999 0 0 3 0 exit 3
|
|
4 : TIMESTAMP 9.999 0 0 0 0 exit 0
|
|
Seq Host Starttime JobRuntime Send Receive Exitval Signal Command
|
|
1 : TIMESTAMP 9.999 0 0 1 0 exit 1
|
|
2 : TIMESTAMP 9.999 0 0 2 0 exit 2
|
|
3 : TIMESTAMP 9.999 0 0 3 0 exit 3
|
|
4 : TIMESTAMP 9.999 0 0 0 0 exit 0
|
|
Seq Host Starttime JobRuntime Send Receive Exitval Signal Command
|
|
1 : TIMESTAMP 9.999 0 0 1 0 exit 1
|
|
2 : TIMESTAMP 9.999 0 0 2 0 exit 2
|
|
3 : TIMESTAMP 9.999 0 0 3 0 exit 3
|
|
4 : TIMESTAMP 9.999 0 0 0 0 exit 0
|
|
5 : TIMESTAMP 9.999 0 0 0 0 exit 0
|
|
6 : TIMESTAMP 9.999 0 0 0 0 exit 0
|
|
Seq Host Starttime JobRuntime Send Receive Exitval Signal Command
|
|
1 : TIMESTAMP 9.999 0 0 1 0 exit 1
|
|
2 : TIMESTAMP 9.999 0 0 2 0 exit 2
|
|
3 : TIMESTAMP 9.999 0 0 3 0 exit 3
|
|
4 : TIMESTAMP 9.999 0 0 0 0 exit 0
|
|
5 : TIMESTAMP 9.999 0 0 0 0 exit 0
|
|
6 : TIMESTAMP 9.999 0 0 0 0 exit 0
|
|
1 : TIMESTAMP 9.999 0 0 1 0 exit 1
|
|
2 : TIMESTAMP 9.999 0 0 2 0 exit 2
|
|
3 : TIMESTAMP 9.999 0 0 3 0 exit 3
|
|
Seq Host Starttime JobRuntime Send Receive Exitval Signal Command
|
|
1 : TIMESTAMP 9.999 0 0 1 0 exit 1
|
|
2 : TIMESTAMP 9.999 0 0 2 0 exit 2
|
|
3 : TIMESTAMP 9.999 0 0 3 0 exit 3
|
|
4 : TIMESTAMP 9.999 0 0 0 0 exit 0
|
|
5 : TIMESTAMP 9.999 0 0 0 0 exit 0
|
|
6 : TIMESTAMP 9.999 0 0 0 0 exit 0
|
|
1 : TIMESTAMP 9.999 0 0 1 0 exit 1
|
|
2 : TIMESTAMP 9.999 0 0 2 0 exit 2
|
|
3 : TIMESTAMP 9.999 0 0 3 0 exit 3
|
|
1 : TIMESTAMP 9.999 0 0 1 0 exit 1
|
|
2 : TIMESTAMP 9.999 0 0 2 0 exit 2
|
|
3 : TIMESTAMP 9.999 0 0 3 0 exit 3
|
|
9
|
|
parallel: This job failed:
|
|
echo X; exit X
|
|
parallel: Starting no more jobs. Waiting for 1 jobs to finish.
|
|
parallel: This job failed:
|
|
echo X; exit X
|
|
parallel -j2 --halt now,fail=1 echo {}\; exit {} ::: 0 0 1 2 3
|
|
9
|
|
parallel: This job failed:
|
|
echo X; exit X
|
|
parallel -j2 --halt soon,fail=20% echo {}\; exit {} \
|
|
::: 0 1 2 3 4 5 6 7 8 9
|
|
9
|
|
parallel: This job failed:
|
|
echo X; exit X
|
|
parallel: This job failed:
|
|
echo X; exit X
|
|
parallel: Starting no more jobs. Waiting for 1 jobs to finish.
|
|
parallel: This job failed:
|
|
echo X; exit X
|
|
parallel -j2 --halt now,success=1 echo {}\; exit {} ::: 1 2 3 0 4 5 6
|
|
9
|
|
parallel: This job succeeded:
|
|
echo X; exit X
|
|
parallel -k --retries 3 \
|
|
'echo tried {} >>/tmp/runs; echo completed {}; exit {}' ::: 1 2 0
|
|
cat /tmp/runs
|
|
completed 1
|
|
completed 2
|
|
completed 0
|
|
tried 0
|
|
show_signals() {
|
|
perl -e 'for(keys %SIG) {
|
|
$SIG{$_} = eval "sub { print \"Got $_\\n\"; }";
|
|
}
|
|
while(1){sleep 1}'
|
|
}
|
|
export -f show_signals
|
|
echo | parallel --termseq TERM,200,TERM,100,TERM,50,KILL,25 \
|
|
-u --timeout 1 show_signals
|
|
parallel: Warning: This job was killed because it timed out:
|
|
parallel: Warning: show_signals ''
|
|
echo | parallel --termseq INT,200,TERM,100,KILL,25 \
|
|
-u --timeout 1 show_signals
|
|
/bin/bash: show_signals: command not found
|
|
parallel --load 100% echo load is less than {} job per cpu ::: 1
|
|
load is less than 1 job per cpu
|
|
parallel --noswap echo the system is not swapping ::: now
|
|
the system is not swapping now
|
|
parallel --memfree 1G echo will run if more than 1 GB is ::: free
|
|
will run if more than 1 GB is free
|
|
parallel --nice 17 echo this is being run with nice -n ::: 17
|
|
this is being run with nice -n 17
|
|
parallel -S $SERVER1 echo running on ::: $SERVER1
|
|
running on parallel@lo
|
|
parallel -S $SERVER1 echo running on ::: username@$SERVER1
|
|
running on username@parallel@lo
|
|
parallel -S : echo running on ::: the_local_machine
|
|
running on the_local_machine
|
|
parallel -S '/usr/bin/ssh '$SERVER1 echo custom ::: ssh
|
|
custom ssh
|
|
parallel --ssh /usr/bin/ssh -S $SERVER1 echo custom ::: ssh
|
|
custom ssh
|
|
export PARALLEL_SSH=/usr/bin/ssh
|
|
parallel -S $SERVER1 echo custom ::: ssh
|
|
custom ssh
|
|
parallel -S $SERVER1 -S $SERVER2 echo ::: running on more hosts
|
|
running
|
|
on
|
|
more
|
|
hosts
|
|
echo $SERVER1 > nodefile
|
|
# Force 4 cores, special ssh-command, username
|
|
echo 4//usr/bin/ssh $SERVER2 >> nodefile
|
|
parallel --sshloginfile nodefile echo ::: running on more hosts
|
|
running
|
|
on
|
|
more
|
|
hosts
|
|
parallel -S 4/$SERVER1 echo force {} cpus on server ::: 4
|
|
force 4 cpus on server
|
|
parallel --hostgroup -S @grp1/$SERVER1 -S @grp2/$SERVER2 echo {} \
|
|
::: run_on_grp1@grp1 run_on_grp2@grp2
|
|
run_on_grp1
|
|
run_on_grp2
|
|
echo This is input_file > input_file
|
|
parallel -S $SERVER1 --transferfile {} cat ::: input_file
|
|
This is input_file
|
|
echo This is input_file > input_file
|
|
parallel -S $SERVER1 --transferfile {} --return {}.out \
|
|
cat {} ">"{}.out ::: input_file
|
|
cat input_file.out
|
|
This is input_file
|
|
echo This is input_file > input_file
|
|
parallel -S $SERVER1 --transferfile {} --return {}.out --cleanup \
|
|
cat {} ">"{}.out ::: input_file
|
|
cat input_file.out
|
|
This is input_file
|
|
echo This is input_file > input_file
|
|
parallel -S $SERVER1 --trc {}.out cat {} ">"{}.out ::: input_file
|
|
cat input_file.out
|
|
This is input_file
|
|
echo common data > common_file
|
|
parallel --basefile common_file -S $SERVER1 \
|
|
cat common_file\; echo {} ::: foo
|
|
common data
|
|
foo
|
|
parallel -S $SERVER1 pwd ::: ""
|
|
parallel --workdir . -S $SERVER1 pwd ::: ""
|
|
parallel --workdir ... -S $SERVER1 pwd ::: ""
|
|
/mnt/4tb/home/parallel
|
|
BASE64/mnt/4tb/home/parallel/.TMPWORKDIR
|
|
parallel -S $SERVER1 --sshdelay 0.2 echo ::: 1 2 3
|
|
9
|
|
parallel --controlmaster -S $SERVER1 echo ::: 1 2 3
|
|
9
|
|
parallel --filter-hosts -S 173.194.32.46,$SERVER1 echo ::: bar
|
|
bar
|
|
parallel: Warning: Removed 173.194.32.46.
|
|
parallel --onall -S $SERVER1,$SERVER2 echo ::: foo bar
|
|
foo
|
|
bar
|
|
foo
|
|
bar
|
|
parallel --nonall --tag -S $SERVER1,$SERVER2 echo foo bar
|
|
csh@lo foo bar
|
|
parallel@lo foo bar
|
|
source `which env_parallel.bash`
|
|
alias myecho=echo
|
|
myvar="Joe's var is"
|
|
env_parallel -S $SERVER1 'myecho $myvar' ::: green
|
|
Unknown option: -S
|
|
Unknown option: parallel@lo
|
|
Unknown option: myecho $myvar
|
|
Unknown option: :::
|
|
Unknown option: green
|
|
env_parallel only works if it is a function. Do the below and restart your shell.
|
|
bash: Put this in $HOME/.bashrc: . `which env_parallel.bash`
|
|
E.g. by doing: echo '. `which env_parallel.bash`' >> $HOME/.bashrc
|
|
Supports: aliases, functions, variables, arrays
|
|
zsh: Put this in $HOME/.zshrc: . `which env_parallel.zsh`
|
|
E.g. by doing: echo '. `which env_parallel.zsh`' >> $HOME/.zshenv
|
|
Supports: functions, variables, arrays
|
|
fish: Put this in $HOME/.config/fish/config.fish:
|
|
. (which env_parallel.fish)
|
|
E.g. by doing:
|
|
echo '. (which env_parallel.fish)' >> $HOME/.config/fish/config.fish
|
|
Supports: aliases, functions, variables, arrays
|
|
ksh: Put this in $HOME/.kshrc: source `which env_parallel.ksh`
|
|
E.g. by doing: echo 'source `which env_parallel.ksh`' >> $HOME/.kshrc
|
|
Supports: aliases, functions, variables, arrays
|
|
mksh: Put this in $HOME/.mkshrc: source `which env_parallel.mksh`
|
|
E.g. by doing: echo 'source `which env_parallel.mksh`' >> $HOME/.mkshrc
|
|
Supports: aliases, functions, variables, arrays
|
|
pdksh: Put this in $HOME/.profile: source `which env_parallel.pdksh`
|
|
E.g. by doing: echo '. `which env_parallel.pdksh`' >> $HOME/.profile
|
|
Supports: aliases, functions, variables, arrays
|
|
ash: Put this in $HOME/.profile: . `which env_parallel.ash`
|
|
E.g. by doing: echo '. `which env_parallel.ash`' >> $HOME/.profile
|
|
Supports: aliases, variables
|
|
dash: Put this in $HOME/.profile: . `which env_parallel.dash`
|
|
E.g. by doing: echo '. `which env_parallel.dash`' >> $HOME/.profile
|
|
Supports: aliases, variables
|
|
csh: Put this in $HOME/.cshrc: source `which env_parallel.csh`
|
|
E.g. by doing: echo 'source `which env_parallel.csh`' >> $HOME/.cshrc
|
|
Supports: aliases, variables, arrays with no special chars
|
|
tcsh: Put this in $HOME/.tcshrc: source `which env_parallel.tcsh`
|
|
E.g. by doing: echo 'source `which env_parallel.tcsh`' >> $HOME/.tcshrc
|
|
Supports: aliases, variables, arrays with no special chars
|
|
To install in all shells run:
|
|
env_parallel --install
|
|
For details: see man env_parallel
|
|
MYVAR='foo bar'
|
|
export MYVAR
|
|
parallel --env MYVAR -S $SERVER1 echo '$MYVAR' ::: baz
|
|
foo bar baz
|
|
my_func() {
|
|
echo in my_func $1
|
|
}
|
|
export -f my_func
|
|
parallel --env my_func -S $SERVER1 my_func ::: baz
|
|
in my_func baz
|
|
parallel --record-env
|
|
cat ~/.parallel/ignored_vars|sort
|
|
BASH_FUNC_run_once%%
|
|
BASH_FUNC_run_test%%
|
|
COLORFGBG
|
|
COLORTERM
|
|
DBUS_SESSION_BUS_ADDRESS
|
|
DEBEMAIL
|
|
DEBFULLNAME
|
|
DEFAULTS_PATH
|
|
DESKTOP_SESSION
|
|
DISPLAY
|
|
GOPATH
|
|
GPG_AGENT_INFO
|
|
GS_LIB
|
|
HISTCONTROL
|
|
HOME
|
|
JAVA_HOME
|
|
JDK_HOME
|
|
KDE_FULL_SESSION
|
|
KDE_SESSION_UID
|
|
KDE_SESSION_VERSION
|
|
KONSOLE_DBUS_SERVICE
|
|
KONSOLE_DBUS_SESSION
|
|
KONSOLE_DBUS_WINDOW
|
|
KONSOLE_PROFILE_NAME
|
|
LANG
|
|
LANGUAGE
|
|
LC_ALL
|
|
LC_TIME
|
|
LESS
|
|
LESSCLOSE
|
|
LESSOPEN
|
|
LOGNAME
|
|
LS_COLORS
|
|
MAKEFLAGS
|
|
MAKELEVEL
|
|
MAKE_TERMERR
|
|
MAKE_TERMOUT
|
|
MANDATORY_PATH
|
|
MFLAGS
|
|
OLDPWD
|
|
ORACLE_HOME
|
|
ORACLE_SID
|
|
PARALLEL
|
|
PARALLEL_HOME
|
|
PARALLEL_JOBSLOT
|
|
PARALLEL_PID
|
|
PARALLEL_RSYNC_OPTS
|
|
PARALLEL_SEQ
|
|
PARALLEL_SSHHOST
|
|
PARALLEL_SSHLOGIN
|
|
PARALLEL_TMP
|
|
PATH
|
|
PERL_MB_OPT
|
|
PERL_MM_OPT
|
|
PROFILEHOME
|
|
PWD
|
|
QT_ACCESSIBILITY
|
|
QT_AUTO_SCREEN_SCALE_FACTOR
|
|
SERVER1
|
|
SERVER2
|
|
SESSION_MANAGER
|
|
SHELL
|
|
SHELL_SESSION_ID
|
|
SHLVL
|
|
SSH_AGENT_PID
|
|
SSH_AUTH_SOCK
|
|
S_COLORS
|
|
TERM
|
|
TIMEOUT
|
|
TMPDIR
|
|
TRIES
|
|
USER
|
|
VISUAL
|
|
WINDOWID
|
|
XAUTHORITY
|
|
XCURSOR_SIZE
|
|
XCURSOR_THEME
|
|
XDG_CONFIG_DIRS
|
|
XDG_CURRENT_DESKTOP
|
|
XDG_DATA_DIRS
|
|
XDG_RUNTIME_DIR
|
|
XDG_SEAT
|
|
XDG_SEAT_PATH
|
|
XDG_SESSION_CLASS
|
|
XDG_SESSION_DESKTOP
|
|
XDG_SESSION_ID
|
|
XDG_SESSION_PATH
|
|
XDG_SESSION_TYPE
|
|
XDG_VTNR
|
|
_
|
|
mysqlrootpass
|
|
# The function is only copied if using Bash
|
|
my_func2() {
|
|
echo in my_func2 $VAR $1
|
|
}
|
|
export -f my_func2
|
|
VAR=foo
|
|
export VAR
|
|
parallel --env _ -S $SERVER1 'echo $VAR; my_func2' ::: bar
|
|
/bin/bash: my_func2: command not found
|
|
NOT='not exported var'
|
|
alias myecho=echo
|
|
not_ex() {
|
|
myecho in not_exported_func $NOT $1
|
|
}
|
|
env_parallel --env _ -S $SERVER1 'echo $NOT; not_ex' ::: bar
|
|
Unknown option: --env
|
|
Unknown option: _
|
|
Unknown option: -S
|
|
Unknown option: parallel@lo
|
|
Unknown option: echo $NOT; not_ex
|
|
Unknown option: :::
|
|
Unknown option: bar
|
|
env_parallel only works if it is a function. Do the below and restart your shell.
|
|
bash: Put this in $HOME/.bashrc: . `which env_parallel.bash`
|
|
E.g. by doing: echo '. `which env_parallel.bash`' >> $HOME/.bashrc
|
|
Supports: aliases, functions, variables, arrays
|
|
zsh: Put this in $HOME/.zshrc: . `which env_parallel.zsh`
|
|
E.g. by doing: echo '. `which env_parallel.zsh`' >> $HOME/.zshenv
|
|
Supports: functions, variables, arrays
|
|
fish: Put this in $HOME/.config/fish/config.fish:
|
|
. (which env_parallel.fish)
|
|
E.g. by doing:
|
|
echo '. (which env_parallel.fish)' >> $HOME/.config/fish/config.fish
|
|
Supports: aliases, functions, variables, arrays
|
|
ksh: Put this in $HOME/.kshrc: source `which env_parallel.ksh`
|
|
E.g. by doing: echo 'source `which env_parallel.ksh`' >> $HOME/.kshrc
|
|
Supports: aliases, functions, variables, arrays
|
|
mksh: Put this in $HOME/.mkshrc: source `which env_parallel.mksh`
|
|
E.g. by doing: echo 'source `which env_parallel.mksh`' >> $HOME/.mkshrc
|
|
Supports: aliases, functions, variables, arrays
|
|
pdksh: Put this in $HOME/.profile: source `which env_parallel.pdksh`
|
|
E.g. by doing: echo '. `which env_parallel.pdksh`' >> $HOME/.profile
|
|
Supports: aliases, functions, variables, arrays
|
|
ash: Put this in $HOME/.profile: . `which env_parallel.ash`
|
|
E.g. by doing: echo '. `which env_parallel.ash`' >> $HOME/.profile
|
|
Supports: aliases, variables
|
|
dash: Put this in $HOME/.profile: . `which env_parallel.dash`
|
|
E.g. by doing: echo '. `which env_parallel.dash`' >> $HOME/.profile
|
|
Supports: aliases, variables
|
|
csh: Put this in $HOME/.cshrc: source `which env_parallel.csh`
|
|
E.g. by doing: echo 'source `which env_parallel.csh`' >> $HOME/.cshrc
|
|
Supports: aliases, variables, arrays with no special chars
|
|
tcsh: Put this in $HOME/.tcshrc: source `which env_parallel.tcsh`
|
|
E.g. by doing: echo 'source `which env_parallel.tcsh`' >> $HOME/.tcshrc
|
|
Supports: aliases, variables, arrays with no special chars
|
|
To install in all shells run:
|
|
env_parallel --install
|
|
For details: see man env_parallel
|
|
parallel -vv --pipepart --block 1M wc :::: num30000
|
|
<num30000 perl -e 'while(@ARGV){sysseek(STDIN,shift,0)||die;$left=shift;while($read=sysread(STDIN,$buf,$left>131072?131072:$left)){$left-=$read;syswrite(STDOUT,$buf);}}' 0 0 0 168894 |(wc)
|
|
30000 30000 168894
|
|
my_func3() {
|
|
echo in my_func $1 > $1.out
|
|
}
|
|
export -f my_func3
|
|
parallel -vv --workdir ... --nice 17 --env _ --trc {}.out \
|
|
-S $SERVER1 my_func3 {} ::: abc-file
|
|
ssh -l parallel lo -- mkdir -p ./.TMPWORKDIR && rsync --protocol 30 -rlDzR -e'ssh -l parallel' ./abc-file lo:./.TMPWORKDIR;ssh -l parallel lo -- exec perl -e "'"'@GNU_Parallel=split/_/,"use_IPC::Open3;_use_MIME::Base64";eval"@GNU_Parallel";$chld=$SIG{CHLD};$SIG{CHLD}="IGNORE";my$zip=(grep{-x$_}"/usr/local/bin/bzip2")[0]||"bzip2";open3($in,$out,">&STDERR",$zip,"-dc");if(my$perlpid=fork){close$in;$eval=join"",<$out>;close$out;}else{close$out;print$in(decode_base64(join"",@ARGV));close$in;exit;}wait;$SIG{CHLD}=$chld;eval$eval;'"'" BASE64;_EXIT_status=$?; mkdir -p ./. && rsync --protocol 30 --rsync-path='cd ./.TMPWORKDIR/./.; rsync' -rlDzR -e'ssh -l parallel' lo:./abc-file.out ./.;ssh -l parallel lo -- 'rm -f ./.TMPWORKDIR/abc-file; sh -c '"'"'rmdir ./.TMPWORKDIR/ ./.parallel/tmp/ ./.parallel/ 2>/dev/null;'"'"''"'"'rm -rf ./.TMPWORKDIR;'"'";ssh -l parallel lo -- 'rm -f ./.TMPWORKDIR/abc-file.out; sh -c '"'"'rmdir ./.TMPWORKDIR/ ./.parallel/tmp/ ./.parallel/ 2>/dev/null;'"'"''"'"'rm -rf ./.TMPWORKDIR;'"'";ssh -l parallel lo -- rm -rf .TMPWORKDIR;exit $_EXIT_status;
|
|
parset myvar1,myvar2 echo ::: a b
|
|
echo $myvar1
|
|
echo $myvar2
|
|
Unknown option: myvar1,myvar2
|
|
Unknown option: echo
|
|
Unknown option: :::
|
|
Unknown option: a
|
|
Unknown option: b
|
|
parset only works if it is a function. The function is defined as part
|
|
of env_parallel.
|
|
Do the below and restart your shell.
|
|
bash: Put this in $HOME/.bashrc: . `which env_parallel.bash`
|
|
E.g. by doing: echo '. `which env_parallel.bash`' >> $HOME/.bashrc
|
|
Supports: aliases, functions, variables, arrays
|
|
zsh: Put this in $HOME/.zshrc: . `which env_parallel.zsh`
|
|
E.g. by doing: echo '. `which env_parallel.zsh`' >> $HOME/.zshenv
|
|
Supports: functions, variables, arrays
|
|
fish: Unsupported
|
|
ksh: Put this in $HOME/.kshrc: source `which env_parallel.ksh`
|
|
E.g. by doing: echo 'source `which env_parallel.ksh`' >> $HOME/.kshrc
|
|
Supports: aliases, functions, variables, arrays
|
|
mksh: Put this in $HOME/.mkshrc: source `which env_parallel.mksh`
|
|
E.g. by doing: echo 'source `which env_parallel.mksh`' >> $HOME/.mkshrc
|
|
Supports: aliases, functions, variables, arrays
|
|
pdksh: Put this in $HOME/.profile: source `which env_parallel.pdksh`
|
|
E.g. by doing: echo '. `which env_parallel.pdksh`' >> $HOME/.profile
|
|
Supports: aliases, functions, variables, arrays
|
|
ash: Put this in $HOME/.profile: . `which env_parallel.ash`
|
|
E.g. by doing: echo '. `which env_parallel.ash`' >> $HOME/.profile
|
|
Supports: aliases, variables
|
|
dash: Put this in $HOME/.profile: . `which env_parallel.dash`
|
|
E.g. by doing: echo '. `which env_parallel.dash`' >> $HOME/.profile
|
|
Supports: aliases, variables
|
|
csh: Unsupported
|
|
tcsh: Unsupported
|
|
To install in all shells run:
|
|
parset --install
|
|
For details: see man parset
|
|
parset myarray seq {} 5 ::: 1 2 3
|
|
echo "${myarray[1]}"
|
|
Unknown option: myarray
|
|
Unknown option: seq
|
|
Unknown option: {}
|
|
Unknown option: 5
|
|
Unknown option: :::
|
|
Unknown option: 1
|
|
Unknown option: 2
|
|
Unknown option: 3
|
|
parset only works if it is a function. The function is defined as part
|
|
of env_parallel.
|
|
Do the below and restart your shell.
|
|
bash: Put this in $HOME/.bashrc: . `which env_parallel.bash`
|
|
E.g. by doing: echo '. `which env_parallel.bash`' >> $HOME/.bashrc
|
|
Supports: aliases, functions, variables, arrays
|
|
zsh: Put this in $HOME/.zshrc: . `which env_parallel.zsh`
|
|
E.g. by doing: echo '. `which env_parallel.zsh`' >> $HOME/.zshenv
|
|
Supports: functions, variables, arrays
|
|
fish: Unsupported
|
|
ksh: Put this in $HOME/.kshrc: source `which env_parallel.ksh`
|
|
E.g. by doing: echo 'source `which env_parallel.ksh`' >> $HOME/.kshrc
|
|
Supports: aliases, functions, variables, arrays
|
|
mksh: Put this in $HOME/.mkshrc: source `which env_parallel.mksh`
|
|
E.g. by doing: echo 'source `which env_parallel.mksh`' >> $HOME/.mkshrc
|
|
Supports: aliases, functions, variables, arrays
|
|
pdksh: Put this in $HOME/.profile: source `which env_parallel.pdksh`
|
|
E.g. by doing: echo '. `which env_parallel.pdksh`' >> $HOME/.profile
|
|
Supports: aliases, functions, variables, arrays
|
|
ash: Put this in $HOME/.profile: . `which env_parallel.ash`
|
|
E.g. by doing: echo '. `which env_parallel.ash`' >> $HOME/.profile
|
|
Supports: aliases, variables
|
|
dash: Put this in $HOME/.profile: . `which env_parallel.dash`
|
|
E.g. by doing: echo '. `which env_parallel.dash`' >> $HOME/.profile
|
|
Supports: aliases, variables
|
|
csh: Unsupported
|
|
tcsh: Unsupported
|
|
To install in all shells run:
|
|
parset --install
|
|
For details: see man parset
|
|
cmd=("echo '<<joe \"double space\" cartoon>>'" "pwd")
|
|
parset data ::: "${cmd[@]}"
|
|
echo "${data[0]}"
|
|
echo "${data[1]}"
|
|
Unknown option: data
|
|
Unknown option: :::
|
|
Unknown option: echo '<<joe "double space" cartoon>>'
|
|
Unknown option: pwd
|
|
parset only works if it is a function. The function is defined as part
|
|
of env_parallel.
|
|
Do the below and restart your shell.
|
|
bash: Put this in $HOME/.bashrc: . `which env_parallel.bash`
|
|
E.g. by doing: echo '. `which env_parallel.bash`' >> $HOME/.bashrc
|
|
Supports: aliases, functions, variables, arrays
|
|
zsh: Put this in $HOME/.zshrc: . `which env_parallel.zsh`
|
|
E.g. by doing: echo '. `which env_parallel.zsh`' >> $HOME/.zshenv
|
|
Supports: functions, variables, arrays
|
|
fish: Unsupported
|
|
ksh: Put this in $HOME/.kshrc: source `which env_parallel.ksh`
|
|
E.g. by doing: echo 'source `which env_parallel.ksh`' >> $HOME/.kshrc
|
|
Supports: aliases, functions, variables, arrays
|
|
mksh: Put this in $HOME/.mkshrc: source `which env_parallel.mksh`
|
|
E.g. by doing: echo 'source `which env_parallel.mksh`' >> $HOME/.mkshrc
|
|
Supports: aliases, functions, variables, arrays
|
|
pdksh: Put this in $HOME/.profile: source `which env_parallel.pdksh`
|
|
E.g. by doing: echo '. `which env_parallel.pdksh`' >> $HOME/.profile
|
|
Supports: aliases, functions, variables, arrays
|
|
ash: Put this in $HOME/.profile: . `which env_parallel.ash`
|
|
E.g. by doing: echo '. `which env_parallel.ash`' >> $HOME/.profile
|
|
Supports: aliases, variables
|
|
dash: Put this in $HOME/.profile: . `which env_parallel.dash`
|
|
E.g. by doing: echo '. `which env_parallel.dash`' >> $HOME/.profile
|
|
Supports: aliases, variables
|
|
csh: Unsupported
|
|
tcsh: Unsupported
|
|
To install in all shells run:
|
|
parset --install
|
|
For details: see man parset
|
|
parallel --sqlandworker csv:///%2Ftmp/log.csv \
|
|
seq ::: 10 ::: 12 13 14
|
|
cat /tmp/log.csv
|
|
99
|
|
Seq,Host,Starttime,JobRuntime,Send,Receive,Exitval,_Signal,Command,V1,V2,Stdout,Stderr
|
|
1,:,000000000.000,0.000,0,9,0,0,"seq 10 12",10,12,"10
|
|
99
|
|
",
|
|
2,:,000000000.000,0.000,0,12,0,0,"seq 10 13",10,13,"10
|
|
99
|
|
",
|
|
3,:,000000000.000,0.000,0,15,0,0,"seq 10 14",10,14,"10
|
|
99
|
|
",
|
|
parallel --results outdir --sqlandworker csv:///%2Ftmp/log2.csv \
|
|
seq ::: 10 ::: 12 13 14
|
|
cat /tmp/log2.csv
|
|
99
|
|
Seq,Host,Starttime,JobRuntime,Send,Receive,Exitval,_Signal,Command,V1,V2,Stdout,Stderr
|
|
1,:,000000000.000,0.000,0,9,0,0,"seq 10 12",10,12,outdir/1/10/2/12/stdout,outdir/1/10/2/12/stderr
|
|
2,:,000000000.000,0.000,0,12,0,0,"seq 10 13",10,13,outdir/1/10/2/13/stdout,outdir/1/10/2/13/stderr
|
|
3,:,000000000.000,0.000,0,15,0,0,"seq 10 14",10,14,outdir/1/10/2/14/stdout,outdir/1/10/2/14/stderr
|
|
vendor://[[user][:password]@][host][:port]/[database[/table]
|
|
/bin/bash: vendor://[[user][:password]@][host][:port]/[database[/table]: No such file or directory
|
|
sqlite3:///%2Ftmp%2Fmydatabase/mytable
|
|
csv:///%2Ftmp/log.csv
|
|
/bin/bash: sqlite3:///%2Ftmp%2Fmydatabase/mytable: No such file or directory
|
|
/bin/bash: line 1: csv:///%2Ftmp/log.csv: No such file or directory
|
|
DBURL=sqlite3:///%2Ftmp%2Fmydatabase
|
|
DBURLTABLE=$DBURL/mytable
|
|
parallel --sqlandworker $DBURLTABLE echo ::: foo bar ::: baz quuz
|
|
foo baz
|
|
foo quuz
|
|
bar baz
|
|
bar quuz
|
|
sql $DBURL 'SELECT * FROM mytable ORDER BY Seq;'
|
|
Error:
|
|
SELECT * FROM mytable ORDER BY Seq; is not a valid DBURL
|
|
sql [-hnr] [--table-size] [--db-size] [-p pass-through] [-s string] dburl [command]
|
|
parallel --sqlmaster $DBURLTABLE echo ::: foo bar ::: baz quuz
|
|
parallel: Error: echo is not a valid DBURL
|
|
parallel --sqlworker $DBURLTABLE echo ::: foo bar ::: baz quuz
|
|
parallel: Error: echo is not a valid DBURL
|
|
cat num1000000 | parallel --pipe wc
|
|
999999 999999 9999999
|
|
85352 85352 597465
|
|
cat num1000000 | parallel --pipe --block 2M wc
|
|
999999 999999 9999999
|
|
85349 85349 597444
|
|
cat num1000000 | parallel --pipe -j4 --roundrobin wc
|
|
999999 999999 9999999
|
|
cat num1000000 | parallel --pipe -N140000 wc
|
|
140000 140000 868895
|
|
140000 140000 980000
|
|
20000 20000 140001
|
|
cat num1000000 | parallel --pipe -L75 wc
|
|
999999 999999 9999999
|
|
85500 85500 598500
|
|
25 25 176
|
|
cat fixedlen | parallel --pipe --header .{4} --block 3 --recend '' \
|
|
'echo start; cat; echo'
|
|
start
|
|
HHHHAAA
|
|
start
|
|
HHHHBBB
|
|
start
|
|
HHHHCCC
|
|
echo /foo, bar/, /baz, qux/, | \
|
|
parallel -kN1 --recend ', ' --pipe echo JOB{#}\;cat\;echo END
|
|
JOB1
|
|
/foo, END
|
|
JOB2
|
|
bar/, END
|
|
JOB3
|
|
/baz, END
|
|
JOB4
|
|
qux/,
|
|
END
|
|
echo /foo, bar/, /baz, qux/, | \
|
|
parallel -kN1 --recstart / --pipe echo JOB{#}\;cat\;echo END
|
|
JOB1
|
|
/foo, barEND
|
|
JOB2
|
|
/, END
|
|
JOB3
|
|
/baz, quxEND
|
|
JOB4
|
|
/,
|
|
END
|
|
echo /foo, bar/, /baz, qux/, | \
|
|
parallel -kN1 --recend ', ' --recstart / --pipe \
|
|
echo JOB{#}\;cat\;echo END
|
|
JOB1
|
|
/foo, bar/, END
|
|
JOB2
|
|
/baz, qux/,
|
|
END
|
|
echo foo,bar,_baz,__qux, | \
|
|
parallel -kN1 --regexp --recend ,_+ --pipe \
|
|
echo JOB{#}\;cat\;echo END
|
|
JOB1
|
|
foo,bar,_END
|
|
JOB2
|
|
baz,__END
|
|
JOB3
|
|
qux,
|
|
END
|
|
echo foo,bar,_baz,__qux, | \
|
|
parallel -kN1 --rrs --regexp --recend ,_+ --pipe \
|
|
echo JOB{#}\;cat\;echo END
|
|
JOB1
|
|
foo,barEND
|
|
JOB2
|
|
bazEND
|
|
JOB3
|
|
qux,
|
|
END
|
|
cat num_%header | \
|
|
parallel --header '(%.*\n)*' --pipe -N3 echo JOB{#}\;cat
|
|
cat: num_%header: No such file or directory
|
|
cat num_%header | parallel --header 2 --pipe -N3 echo JOB{#}\;cat
|
|
cat: num_%header: No such file or directory
|
|
parallel --pipepart -a num1000000 --block 3m wc
|
|
999999 999999 9999999
|
|
126984 126984 888889
|
|
cat input_file | parallel command
|
|
/bin/bash: This is input_file: command not found
|
|
#!/bin/bash
|
|
#!/usr/bin/parallel --shebang -r echo
|
|
foo
|
|
bar
|
|
baz
|
|
/bin/bash: foo: command not found
|
|
/bin/bash: line 1: bar: command not found
|
|
/bin/bash: line 2: baz: command not found
|
|
cat input_file | parallel command
|
|
parallel command ::: foo bar
|
|
/bin/bash: This is input_file: command not found
|
|
/bin/bash: foo: command not found
|
|
/bin/bash: bar: command not found
|
|
cat input_file | command
|
|
command foo bar
|
|
/bin/bash: line 1: foo: command not found
|
|
#!/usr/bin/perl
|
|
print "@ARGV\n"
|
|
Error: no such file "@ARGV\n"
|
|
parallel perl_echo ::: foo bar
|
|
/bin/bash: perl_echo: command not found
|
|
#!/usr/bin/parallel --shebang-wrap /usr/bin/perl
|
|
print "@ARGV\n"
|
|
Error: no such file "@ARGV\n"
|
|
perl_echo foo bar
|
|
/bin/bash: perl_echo: command not found
|
|
#!/usr/bin/parallel --shebang-wrap /usr/bin/perl
|
|
print "Arguments @ARGV\n";
|
|
Error: no such file "Arguments @ARGV\n"
|
|
#!/usr/bin/parallel --shebang-wrap /usr/bin/python
|
|
print 'Arguments', str(sys.argv)
|
|
/bin/bash: -c: line 3: syntax error near unexpected token `('
|
|
/bin/bash: -c: line 3: ` print 'Arguments', str(sys.argv)'
|
|
#!/usr/bin/parallel --shebang-wrap /bin/bash
|
|
echo Arguments "$@"
|
|
Arguments
|
|
#!/usr/bin/parallel --shebang-wrap /bin/csh
|
|
echo Arguments "$argv"
|
|
Arguments
|
|
#!/usr/bin/parallel --shebang-wrap /usr/bin/tclsh
|
|
puts "Arguments $argv"
|
|
/bin/bash: line 2: puts: command not found
|
|
#!/usr/bin/parallel --shebang-wrap /usr/bin/Rscript --vanilla --slave
|
|
args <- commandArgs(trailingOnly = TRUE)
|
|
print(paste("Arguments ",args))
|
|
/bin/bash: -c: line 2: syntax error near unexpected token `('
|
|
/bin/bash: -c: line 2: ` args <- commandArgs(trailingOnly = TRUE)'
|
|
#!/usr/bin/parallel --shebang-wrap ARG={} /usr/bin/gnuplot
|
|
print "Arguments ", system('echo $ARG')
|
|
/bin/bash: -c: line 2: syntax error near unexpected token `('
|
|
/bin/bash: -c: line 2: ` print "Arguments ", system('echo $ARG')'
|
|
#!/usr/bin/parallel --shebang-wrap /usr/bin/ruby
|
|
print "Arguments "
|
|
puts ARGV
|
|
Error: no such file "Arguments "
|
|
/bin/bash: line 3: puts: command not found
|
|
#!/usr/bin/parallel --shebang-wrap /usr/bin/octave
|
|
printf ("Arguments");
|
|
arg_list = argv ();
|
|
for i = 1:nargin
|
|
printf (" %s", arg_list{i});
|
|
endfor
|
|
printf ("\n");
|
|
/bin/bash: -c: line 2: syntax error near unexpected token `"Arguments"'
|
|
/bin/bash: -c: line 2: ` printf ("Arguments");'
|
|
#!/usr/bin/parallel --shebang-wrap /usr/bin/clisp
|
|
(format t "~&~S~&" 'Arguments)
|
|
(format t "~&~S~&" *args*)
|
|
/bin/bash: -c: line 2: unexpected EOF while looking for matching `''
|
|
/bin/bash: -c: line 4: syntax error: unexpected end of file
|
|
#!/usr/bin/parallel --shebang-wrap /usr/bin/php
|
|
<?php
|
|
echo "Arguments";
|
|
foreach(array_slice($argv,1) as $v)
|
|
{
|
|
echo " $v";
|
|
}
|
|
echo "\n";
|
|
?>
|
|
Arguments
|
|
/bin/bash: line 1: ?php: No such file or directory
|
|
/bin/bash: -c: line 3: syntax error near unexpected token `array_slice'
|
|
/bin/bash: -c: line 3: ` foreach(array_slice($argv,1) as $v)'
|
|
#!/usr/bin/parallel --shebang-wrap /usr/bin/node
|
|
var myArgs = process.argv.slice(2);
|
|
console.log('Arguments ', myArgs);
|
|
/bin/bash: -c: line 0: syntax error near unexpected token `('
|
|
/bin/bash: -c: line 0: ` var myArgs = process.argv.slice(2);'
|
|
#!/usr/bin/parallel --shebang-wrap /usr/bin/lua
|
|
io.write "Arguments"
|
|
for a = 1, #arg do
|
|
io.write(" ")
|
|
io.write(arg[a])
|
|
end
|
|
print("")
|
|
/bin/bash: line 2: io.write: command not found
|
|
/bin/bash: -c: line 3: syntax error near unexpected token `='
|
|
/bin/bash: -c: line 3: ` for a = 1, #arg do'
|
|
#!/usr/bin/parallel --shebang-wrap ARGV={} /usr/bin/csharp
|
|
var argv = Environment.GetEnvironmentVariable("ARGV");
|
|
print("Arguments "+argv);
|
|
/bin/bash: -c: line 2: syntax error near unexpected token `('
|
|
/bin/bash: -c: line 2: ` var argv = Environment.GetEnvironmentVariable("ARGV");'
|
|
sem 'sleep 1; echo The first finished' &&
|
|
echo The first is now running in the background &&
|
|
sem 'sleep 1; echo The second finished' &&
|
|
echo The second is now running in the background
|
|
sem --wait
|
|
The first is now running in the background
|
|
The first finished
|
|
The second is now running in the background
|
|
sem --fg 'sleep 1; echo The first finished' &&
|
|
echo The first finished running in the foreground &&
|
|
sem --fg 'sleep 1; echo The second finished' &&
|
|
echo The second finished running in the foreground
|
|
sem --wait
|
|
The first finished
|
|
The first finished running in the foreground
|
|
The second finished running in the foreground
|
|
sem --id my_id -u 'echo First started; sleep 10; echo First done'
|
|
First started
|
|
First done
|
|
sem --id my_id -u 'echo Second started; sleep 10; echo Second done'
|
|
sem --jobs 3 --id my_id -u 'echo Start 1; sleep 5; echo 1 done' &&
|
|
sem --jobs 3 --id my_id -u 'echo Start 2; sleep 6; echo 2 done' &&
|
|
sem --jobs 3 --id my_id -u 'echo Start 3; sleep 7; echo 3 done' &&
|
|
sem --jobs 3 --id my_id -u 'echo Start 4; sleep 8; echo 4 done' &&
|
|
sem --wait --id my_id
|
|
Start 1
|
|
Start 2
|
|
1 done
|
|
Start 3
|
|
2 done
|
|
Start 4
|
|
3 done
|
|
4 done
|
|
sem --id foo -u 'echo Slow started; sleep 5; echo Slow ended' &&
|
|
sem --id foo --semaphoretimeout 1 'echo Forced running after 1 sec' &&
|
|
sem --id foo --semaphoretimeout -2 'echo Give up after 2 secs'
|
|
sem --id foo --wait
|
|
Slow started
|
|
Forced running after 1 sec
|
|
Slow ended
|
|
parallel: Warning: Semaphore timed out. Stealing the semaphore.
|
|
parallel: Warning: Semaphore timed out. Exiting.
|
|
parallel --help
|
|
Usage:
|
|
parallel [options] [command [arguments]] < list_of_arguments
|
|
parallel [options] [command [arguments]] (::: arguments|:::: argfile(s))...
|
|
cat ... | parallel --pipe [options] [command [arguments]]
|
|
-j n Run n jobs in parallel
|
|
-k Keep same order
|
|
-X Multiple arguments with context replace
|
|
--colsep regexp Split input on regexp for positional replacements
|
|
{} {.} {/} {/.} {#} {%} {= perl code =} Replacement strings
|
|
{3} {3.} {3/} {3/.} {=3 perl code =} Positional replacement strings
|
|
With --plus: {} = {+/}/{/} = {.}.{+.} = {+/}/{/.}.{+.} = {..}.{+..} =
|
|
{+/}/{/..}.{+..} = {...}.{+...} = {+/}/{/...}.{+...}
|
|
-S sshlogin Example: foo@server.example.com
|
|
--slf .. Use ~/.parallel/sshloginfile as the list of sshlogins
|
|
--trc {}.bar Shorthand for --transfer --return {}.bar --cleanup
|
|
--onall Run the given command with argument on all sshlogins
|
|
--nonall Run the given command with no arguments on all sshlogins
|
|
--pipe Split stdin (standard input) to multiple jobs.
|
|
--recend str Record end separator for --pipe.
|
|
--recstart str Record start separator for --pipe.
|
|
See 'man parallel' for details
|
|
Academic tradition requires you to cite works you base your article on.
|
|
If you use programs that use GNU Parallel to process data for an article in a
|
|
scientific publication, please cite:
|
|
O. Tange (2018): GNU Parallel 2018, Mar 2018, ISBN 9781387509881,
|
|
DOI https://doi.org/10.5281/zenodo.1146014
|
|
This helps funding further development; AND IT WON'T COST YOU A CENT.
|
|
If you pay 10000 EUR you should feel free to use GNU Parallel without citing.
|
|
parallel --version
|
|
GNU parallel VERSION
|
|
Copyright (C) 2007-2020 Ole Tange, http://ole.tange.dk and Free Software
|
|
Foundation, Inc.
|
|
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
|
|
This is free software: you are free to change and redistribute it.
|
|
GNU parallel comes with no warranty.
|
|
Web site: http://www.gnu.org/software/parallel
|
|
When using programs that use GNU Parallel to process data for publication
|
|
please cite as described in 'parallel --citation'.
|
|
parallel --minversion VERSION && \
|
|
echo Your version is at least VERSION.
|
|
VERSION
|
|
Your version is at least VERSION.
|
|
parallel --citation
|
|
Academic tradition requires you to cite works you base your article on.
|
|
If you use programs that use GNU Parallel to process data for an article in a
|
|
scientific publication, please cite:
|
|
@book{tange_ole_2018_1146014,
|
|
author = {Tange, Ole},
|
|
title = {GNU Parallel 2018},
|
|
publisher = {Ole Tange},
|
|
month = Mar,
|
|
year = 2018,
|
|
ISBN = {9781387509881},
|
|
doi = {10.5281/zenodo.1146014},
|
|
url = {https://doi.org/10.5281/zenodo.1146014}
|
|
}
|
|
(Feel free to use \nocite{tange_ole_2018_1146014})
|
|
This helps funding further development; AND IT WON'T COST YOU A CENT.
|
|
If you pay 10000 EUR you should feel free to use GNU Parallel without citing.
|
|
More about funding GNU Parallel and the citation notice:
|
|
https://lists.gnu.org/archive/html/parallel/2013-11/msg00006.html
|
|
https://www.gnu.org/software/parallel/parallel_design.html#Citation-notice
|
|
https://git.savannah.gnu.org/cgit/parallel.git/tree/doc/citation-notice-faq.txt
|
|
If you send a copy of your published article to tange@gnu.org, it will be
|
|
mentioned in the release notes of next version of GNU Parallel.
|
|
parallel --number-of-cpus
|
|
parallel --number-of-cores
|
|
9
|
|
echo '--nice 17' > ~/.parallel/nicetimeout
|
|
echo '--timeout 300%' >> ~/.parallel/nicetimeout
|
|
parallel --profile nicetimeout echo ::: A B C
|
|
A
|
|
B
|
|
C
|
|
echo '-vv --dry-run' > ~/.parallel/dryverbose
|
|
parallel --profile dryverbose --profile nicetimeout echo ::: A B C
|
|
echo A
|
|
echo B
|
|
echo C
|
|
8
|