tange/parallel
1
0
Fork 0
mirror of https://git.savannah.gnu.org/git/parallel.git synced 2024-11-22 14:07:55 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity
parallel/testsuite/wanted-results/parallel-tutorial
Ole Tange f90250f2cc Released as 20200222 ('BrexitDay')
2020-02-23 01:10:44 +01:00

1429 lines
46 KiB
Plaintext
Raw Blame History

### test parallel_tutorial
fetch -o - http://pi.dk/3 ) > install.sh
$ sha1sum install.sh
12345678 3374ec53 bacb199b 245af2dd a86df6c9
$ md5sum install.sh
029a9ac0 6e8b5bc6 052eac57 b2c3c9ca
$ sha512sum install.sh
40f53af6 9e20dae5 713ba06c f517006d 9897747b ed8a4694 b1acba1b 1464beb4
60055629 3f2356f3 3e9c4e3c 76e3f3af a9db4b32 bd33322b 975696fc e6b23cfb
$ bash install.sh
/bin/bash: -c: line 0: syntax error near unexpected token `)'
/bin/bash: -c: line 0: ` fetch -o - http://pi.dk/3 ) > install.sh'
parallel -k echo ::: A B C > abc-file
parallel -k echo ::: D E F > def-file
perl -e 'printf "A\0B\0C\0"' > abc0-file
perl -e 'printf "A_B_C_"' > abc_-file
perl -e 'printf "f1\tf2\nA\tB\nC\tD\n"' > tsv-file.tsv
perl -e 'for(1..8){print "$_\n"}' > num8
perl -e 'for(1..128){print "$_\n"}' > num128
perl -e 'for(1..30000){print "$_\n"}' > num30000
perl -e 'for(1..1000000){print "$_\n"}' > num1000000
(echo %head1; echo %head2; \
sleep .3
/bin/bash: -c: line 2: syntax error: unexpected end of file
sleep .3
perl -e 'for(1..10){print "$_\n"}') > num_%header
/bin/bash: -c: line 0: syntax error near unexpected token `)'
/bin/bash: -c: line 0: ` perl -e 'for(1..10){print "$_\n"}') > num_%header'
perl -e 'print "HHHHAAABBBCCC"' > fixedlen
parallel echo ::: A B C
A
B
C
parallel -a abc-file echo
A
B
C
cat abc-file | parallel echo
A
B
C
parallel echo ::: A B C ::: D E F
A D
A E
A F
B D
B E
B F
C D
C E
C F
parallel -a abc-file -a def-file echo
A D
A E
A F
B D
B E
B F
C D
C E
C F
cat abc-file | parallel -a - -a def-file echo
A D
A E
A F
B D
B E
B F
C D
C E
C F
cat abc-file | parallel echo :::: - def-file
A D
A E
A F
B D
B E
B F
C D
C E
C F
parallel echo ::: A B C :::: def-file
A D
A E
A F
B D
B E
B F
C D
C E
C F
parallel --link echo ::: A B C ::: D E F
A D
B E
C F
parallel --link echo ::: A B C D E ::: F G
A F
B G
C F
D G
E F
parallel echo :::: abc-file :::+ G H I :::: def-file
A G D
A G E
A G F
B H D
B H E
B H F
C I D
C I E
C I F
parallel echo :::: abc-file ::: G H I ::::+ def-file
A G D
A H E
A I F
B G D
B H E
B I F
C G D
C H E
C I F
parallel echo ::: A B C D E :::+ F G
A F
B G
parallel --arg-sep ,, echo ,, A B C :::: def-file
A D
A E
A F
B D
B E
B F
C D
C E
C F
parallel --arg-file-sep // echo ::: A B C // def-file
A D
A E
A F
B D
B E
B F
C D
C E
C F
parallel -d _ echo :::: abc_-file
A
B
C
parallel -d '\0' echo :::: abc0-file
A
B
C
parallel -E stop echo ::: A B stop C D
A
B
(echo 1; echo; echo 2) | parallel --no-run-if-empty echo
9
parallel ::: ls 'echo foo' pwd
abc-file
abc0-file
abc_-file
def-file
fixedlen
num1000000
num128
num30000
num8
tsv-file.tsv
foo
~/privat/parallel/testsuite/tmp
my_func() {
echo in my_func $1
}
export -f my_func
parallel my_func ::: 1 2 3
in my_func 1
in my_func 2
in my_func 3
parallel echo ::: A/B.C
A/B.C
parallel echo {} ::: A/B.C
A/B.C
parallel echo {.} ::: A/B.C
A/B
parallel echo {/} ::: A/B.C
B.C
parallel echo {//} ::: A/B.C
A
parallel echo {/.} ::: A/B.C
B
parallel echo {#} ::: A B C
9
parallel -j 2 echo {%} ::: A B C
9
parallel -I ,, echo ,, ::: A/B.C
A/B.C
parallel --extensionreplace ,, echo ,, ::: A/B.C
A/B
parallel --basenamereplace ,, echo ,, ::: A/B.C
B.C
parallel --dirnamereplace ,, echo ,, ::: A/B.C
A
parallel --basenameextensionreplace ,, echo ,, ::: A/B.C
B
parallel --seqreplace ,, echo ,, ::: A B C
9
parallel -j2 --slotreplace ,, echo ,, ::: A B C
9
parallel echo '{= s:\.[^.]+$::;s:\.[^.]+$::; =}' ::: foo.tar.gz
foo
parallel echo Job {#} of {= '$_=total_jobs()' =} ::: {1..5}
Job 1 of 5
Job 2 of 5
Job 3 of 5
Job 4 of 5
Job 5 of 5
parallel echo {} shell quoted is {= '$_=Q($_)' =} ::: '*/!#$'
*/!#$ shell quoted is '*/!#$'
parallel echo {= 'if($_==3) { skip() }' =} ::: {1..5}
9
parallel echo {= 'if($arg[1]==$arg[2]) { skip() }' =} \
::: {1..3} ::: {1..3}
1 2
1 3
2 1
2 3
3 1
3 2
parallel --parens ,,,, echo ',, s:\.[^.]+$::;s:\.[^.]+$::; ,,' \
::: foo.tar.gz
foo
parallel --rpl '.. s:\.[^.]+$::;s:\.[^.]+$::;' echo '..' \
::: foo.tar.gz
foo
parallel --rpl '{..} s:\.[^.]+$::;s:\.[^.]+$::;' echo '{..}'
::: foo.tar.gz
/bin/bash: line 1: :::: command not found
parallel --rpl '{%(.+?)} s/$$1$//;' echo {%.tar.gz}.zip ::: foo.tar.gz
foo.zip
parallel --rpl '{/(.+?)/(.*?)} s/$$1/$$2/;' echo {/tar.gz/zip} \
::: foo.tar.gz
foo.zip
--rpl '{} '
--rpl '{#} $_=$job->seq()'
--rpl '{%} $_=$job->slot()'
--rpl '{/} s:.*/::'
--rpl '{//} $Global::use{"File::Basename"} ||=
eval "use File::Basename; 1;"; $_ = dirname($_);'
--rpl '{/.} s:.*/::; s:\.[^/.]+$::;'
--rpl '{.} s:\.[^/.]+$::'
/bin/bash: --rpl: command not found
/bin/bash: line 1: --rpl: command not found
/bin/bash: line 2: --rpl: command not found
/bin/bash: line 3: --rpl: command not found
/bin/bash: line 5: --rpl: command not found
/bin/bash: line 6: --rpl: command not found
/bin/bash: line 7: --rpl: command not found
parallel echo {1} and {2} ::: A B ::: C D
A and C
A and D
B and C
B and D
parallel echo /={1/} //={1//} /.={1/.} .={1.} ::: A/B.C D/E.F
/=B.C //=A /.=B .=A/B
/=E.F //=D /.=E .=D/E
parallel echo 1={1} 2={2} 3={3} -1={-1} -2={-2} -3={-3} \
::: A B ::: C D ::: E F
1=A 2=C 3=E -1=E -2=C -3=A
1=A 2=C 3=F -1=F -2=C -3=A
1=A 2=D 3=E -1=E -2=D -3=A
1=A 2=D 3=F -1=F -2=D -3=A
1=B 2=C 3=E -1=E -2=C -3=B
1=B 2=C 3=F -1=F -2=C -3=B
1=B 2=D 3=E -1=E -2=D -3=B
1=B 2=D 3=F -1=F -2=D -3=B
parallel echo '{=2 s:\.[^.]+$::;s:\.[^.]+$::; =} {1}' \
::: bar ::: foo.tar.gz
foo bar
parallel --rpl '{..} s:\.[^.]+$::;s:\.[^.]+$::;' echo '{2..} {1}' \
::: bar ::: foo.tar.gz
foo bar
parallel --colsep '\t' echo 1={1} 2={2} :::: tsv-file.tsv
1=f1 2=f2
1=A 2=B
1=C 2=D
parallel --header : echo f1={f1} f2={f2} ::: f1 A B ::: f2 C D
f1=A f2=C
f1=A f2=D
f1=B f2=C
f1=B f2=D
parallel --header : --colsep '\t' echo f1={f1} f2={f2} \
:::: tsv-file.tsv
f1=A f2=B
f1=C f2=D
parallel --plus echo {} ::: dir/sub/file.ex1.ex2.ex3
parallel --plus echo {+/}/{/} ::: dir/sub/file.ex1.ex2.ex3
parallel --plus echo {.}.{+.} ::: dir/sub/file.ex1.ex2.ex3
parallel --plus echo {+/}/{/.}.{+.} ::: dir/sub/file.ex1.ex2.ex3
parallel --plus echo {..}.{+..} ::: dir/sub/file.ex1.ex2.ex3
parallel --plus echo {+/}/{/..}.{+..} ::: dir/sub/file.ex1.ex2.ex3
parallel --plus echo {...}.{+...} ::: dir/sub/file.ex1.ex2.ex3
parallel --plus echo {+/}/{/...}.{+...} ::: dir/sub/file.ex1.ex2.ex3
dir/sub/file.ex1.ex2.ex3
parallel --plus echo Job {#} of {##} ::: {1..5}
Job 1 of 5
Job 2 of 5
Job 3 of 5
Job 4 of 5
Job 5 of 5
unset myvar
echo ${myvar:-myval}
parallel --plus echo {:-myval} ::: "$myvar"
myval
myvar=abcAaAdef
echo ${myvar:2}
parallel --plus echo {:2} ::: "$myvar"
cAaAdef
echo ${myvar:2:3}
parallel --plus echo {:2:3} ::: "$myvar"
echo ${myvar#bc}
parallel --plus echo {#bc} ::: "$myvar"
echo ${myvar#abc}
parallel --plus echo {#abc} ::: "$myvar"
echo ${myvar%de}
parallel --plus echo {%de} ::: "$myvar"
echo ${myvar%def}
parallel --plus echo {%def} ::: "$myvar"
echo ${myvar/def/ghi}
parallel --plus echo {/def/ghi} ::: "$myvar"
echo ${myvar^a}
parallel --plus echo {^a} ::: "$myvar"
echo ${myvar^^a}
parallel --plus echo {^^a} ::: "$myvar"
myvar=AbcAaAdef
echo ${myvar,A}
parallel --plus echo '{,A}' ::: "$myvar"
echo ${myvar,,A}
parallel --plus echo '{,,A}' ::: "$myvar"
abcAaAdef
abcaaadef
cat num30000 | parallel --xargs echo | wc -l
9
cat num30000 | parallel --xargs -s 10000 echo | wc -l
99
cat num30000 | parallel --jobs 4 -m echo | wc -l
9
parallel --jobs 4 -m echo ::: 1 2 3 4 5 6 7 8 9 10
1 2 3
4 5 6
7 8 9
99
parallel --jobs 4 -X echo pre-{}-post ::: A B C D E F G
pre-A-post pre-B-post
pre-C-post pre-D-post
pre-E-post pre-F-post
pre-G-post
parallel -N3 echo ::: A B C D E F G H
A B C
D E F
G H
parallel -N3 echo 1={1} 2={2} 3={3} ::: A B C D E F G H
1=A 2=B 3=C
1=D 2=E 3=F
1=G 2=H 3=
parallel -N0 echo foo ::: 1 2 3
foo
perl -e 'print "@ARGV\n"' A
A
parallel perl -e 'print "@ARGV\n"' ::: This wont work
Warning: Input is read from the terminal. You either know what you
Warning: are doing (in which case: YOU ARE AWESOME!) or you forgot
Warning: ::: or :::: or to pipe data into parallel. If so
Warning: Press CTRL-D to exit.
perl -e 'print "@ARGV\n"'
[CTRL-D]
/bin/bash: line 1: Warning:: command not found
/bin/bash: -c: line 2: syntax error near unexpected token `('
/bin/bash: -c: line 2: ` Warning: are doing (in which case: YOU ARE AWESOME!) or you forgot'
parallel --trim r echo pre-{}-post ::: ' A '
pre- A-post
parallel --trim l echo pre-{}-post ::: ' A '
pre-A -post
parallel --trim lr echo pre-{}-post ::: ' A '
pre-A-post
parallel echo \={} ::: zsh bash ls
=zsh
=bash
=ls
parallel 'set a="{}"; if( { test -d "$a" } ) echo "$a is a dir"' ::: *
/bin/bash: -c: line 0: syntax error near unexpected token `)'
/bin/bash: -c: line 0: `set a="abc-file"; if( { test -d "$a" } ) echo "$a is a dir"'
/bin/bash: -c: line 0: syntax error near unexpected token `)'
/bin/bash: -c: line 0: `set a="abc0-file"; if( { test -d "$a" } ) echo "$a is a dir"'
/bin/bash: -c: line 0: syntax error near unexpected token `)'
/bin/bash: -c: line 0: `set a="abc_-file"; if( { test -d "$a" } ) echo "$a is a dir"'
/bin/bash: -c: line 0: syntax error near unexpected token `)'
/bin/bash: -c: line 0: `set a="def-file"; if( { test -d "$a" } ) echo "$a is a dir"'
/bin/bash: -c: line 0: syntax error near unexpected token `)'
/bin/bash: -c: line 0: `set a="fixedlen"; if( { test -d "$a" } ) echo "$a is a dir"'
/bin/bash: -c: line 0: syntax error near unexpected token `)'
/bin/bash: -c: line 0: `set a="num1000000"; if( { test -d "$a" } ) echo "$a is a dir"'
/bin/bash: -c: line 0: syntax error near unexpected token `)'
/bin/bash: -c: line 0: `set a="num128"; if( { test -d "$a" } ) echo "$a is a dir"'
/bin/bash: -c: line 0: syntax error near unexpected token `)'
/bin/bash: -c: line 0: `set a="num30000"; if( { test -d "$a" } ) echo "$a is a dir"'
/bin/bash: -c: line 0: syntax error near unexpected token `)'
/bin/bash: -c: line 0: `set a="num8"; if( { test -d "$a" } ) echo "$a is a dir"'
/bin/bash: -c: line 0: syntax error near unexpected token `)'
/bin/bash: -c: line 0: `set a="tsv-file.tsv"; if( { test -d "$a" } ) echo "$a is a dir"'
parallel --tag echo foo-{} ::: A B C
A foo-A
B foo-B
C foo-C
parallel --tagstring {}-bar echo foo-{} ::: A B C
A-bar foo-A
B-bar foo-B
C-bar foo-C
parallel --dryrun echo {} ::: A B C
echo A
echo B
echo C
parallel --verbose echo {} ::: A B C
A
B
C
echo A
echo B
echo C
parallel -j2 'printf "%s-start\n%s" {} {};
sleep {};printf "%s\n" -middle;echo {}-end' ::: 4 2 1
4-start
2-start
2-middle
2-end
1-start
1-middle
1-end
parallel -j2 --ungroup 'printf "%s-start\n%s" {} {};
sleep {};printf "%s\n" -middle;echo {}-end' ::: 4 2 1
4-start
42-start
2-middle
2-end
1-start
1-middle
1-end
-middle
parallel -j2 --linebuffer 'printf "%s-start\n%s" {} {};
sleep {};printf "%s\n" -middle;echo {}-end' ::: 4 2 1
4-start
2-start
2-middle
2-end
1-start
1-middle
1-end
parallel -j2 -k 'printf "%s-start\n%s" {} {};
sleep {};printf "%s\n" -middle;echo {}-end' ::: 4 2 1
4-start
2-start
2-middle
2-end
1-start
1-middle
1-end
parallel --files echo ::: A B C
/tmp/parallel-tutorial-tmpdir/parXXXXX.par
parallel --tmpdir /var/tmp --files echo ::: A B C
/var/tmp/parXXXXX.par
parallel --results outdir echo ::: A B C
A
B
C
outdir/1/A/seq
outdir/1/A/stderr
outdir/1/A/stdout
outdir/1/B/seq
outdir/1/B/stderr
outdir/1/B/stdout
outdir/1/C/seq
outdir/1/C/stderr
outdir/1/C/stdout
/bin/bash: outdir/1/A/seq: No such file or directory
/bin/bash: line 1: outdir/1/A/stderr: No such file or directory
/bin/bash: line 2: outdir/1/A/stdout: No such file or directory
/bin/bash: line 3: outdir/1/B/seq: No such file or directory
/bin/bash: line 4: outdir/1/B/stderr: No such file or directory
/bin/bash: line 5: outdir/1/B/stdout: No such file or directory
/bin/bash: line 6: outdir/1/C/seq: No such file or directory
/bin/bash: line 7: outdir/1/C/stderr: No such file or directory
/bin/bash: line 8: outdir/1/C/stdout: No such file or directory
parallel --header : --results outdir echo ::: f1 A B ::: f2 C D
A C
A D
B C
B D
outdir/f1/A/f2/C/seq
outdir/f1/A/f2/C/stderr
outdir/f1/A/f2/C/stdout
outdir/f1/A/f2/D/seq
outdir/f1/A/f2/D/stderr
outdir/f1/A/f2/D/stdout
outdir/f1/B/f2/C/seq
outdir/f1/B/f2/C/stderr
outdir/f1/B/f2/C/stdout
outdir/f1/B/f2/D/seq
outdir/f1/B/f2/D/stderr
outdir/f1/B/f2/D/stdout
/bin/bash: outdir/f1/A/f2/C/seq: No such file or directory
/bin/bash: line 1: outdir/f1/A/f2/C/stderr: No such file or directory
/bin/bash: line 2: outdir/f1/A/f2/C/stdout: No such file or directory
/bin/bash: line 3: outdir/f1/A/f2/D/seq: No such file or directory
/bin/bash: line 4: outdir/f1/A/f2/D/stderr: No such file or directory
/bin/bash: line 5: outdir/f1/A/f2/D/stdout: No such file or directory
/bin/bash: line 6: outdir/f1/B/f2/C/seq: No such file or directory
/bin/bash: line 7: outdir/f1/B/f2/C/stderr: No such file or directory
/bin/bash: line 8: outdir/f1/B/f2/C/stdout: No such file or directory
/bin/bash: line 9: outdir/f1/B/f2/D/seq: No such file or directory
/bin/bash: line 10: outdir/f1/B/f2/D/stderr: No such file or directory
/bin/bash: line 11: outdir/f1/B/f2/D/stdout: No such file or directory
/usr/bin/time -f %e parallel -N0 -j64 sleep 1 :::: num128
9
/usr/bin/time -f %e parallel -N0 sleep 1 :::: num128
99
/usr/bin/time -f %e parallel -N0 --jobs 200% sleep 1 :::: num128
9
/usr/bin/time -f %e parallel -N0 --jobs 0 sleep 1 :::: num128
9
echo 50% > my_jobs
/usr/bin/time -f %e parallel -N0 --jobs my_jobs sleep 1 :::: num128 &
sleep 1
echo 0 > my_jobs
wait
9
parallel --use-cpus-instead-of-cores -N0 sleep 1 :::: num8
parallel --shuf echo ::: 1 2 3 ::: a b c ::: A B C
123 abc ABC
seq 10 20 | parallel --tmux 'echo start {}; sleep {}; echo done {}'
See output with: tmux -S /tmp/parallel-tutorial-tmpdir/tmsXXXXX attach
tmux -S /tmp/tmsXXXXX attach
no sessions
parallel --delay 2.5 echo Starting {}\;date ::: 1 2 3
Starting 1
DATE OUTPUT
Starting 2
DATE OUTPUT
Starting 3
DATE OUTPUT
parallel --timeout 4.1 sleep {}\; echo {} ::: 2 4 6 8
9
parallel: Warning: This job was killed because it timed out:
parallel: Warning: sleep 6; echo 6
parallel: Warning: This job was killed because it timed out:
parallel: Warning: sleep 8; echo 8
parallel --timeout 200% sleep {}\; echo {} ::: 2.1 2.2 3 7 2.3
9
parallel: Warning: This job was killed because it timed out:
parallel: Warning: sleep 7; echo 7
parallel --eta sleep ::: 1 3 2 2 1 3 3 2 1
Computers / CPU cores / Max jobs to run
1:local / 8 / 8
Computer:jobs running/jobs completed/%of started jobs/Average seconds to complete
parallel --progress sleep ::: 1 3 2 2 1 3 3 2 1
Computers / CPU cores / Max jobs to run
1:local / 8 / 8
Computer:jobs running/jobs completed/%of started jobs/Average seconds to complete
seq 1000 | parallel -j10 --bar '(echo -n {};sleep 0.1)' \
2> >(zenity --timeout=15 --progress --auto-kill --auto-close)
parallel --joblog /tmp/log exit ::: 1 2 3 0
cat /tmp/log;
parallel --joblog /tmp/log exit ::: 1 2 3 0
cat /tmp/log; parallel --resume --joblog /tmp/log exit ::: 1 2 3 0 0 0
cat /tmp/log;
parallel --resume-failed --joblog /tmp/log exit ::: 1 2 3 0 0 0
cat /tmp/log;
parallel --retry-failed --joblog /tmp/log
cat /tmp/log;
parallel -j2 --halt soon,fail=1 echo {}\; exit {} ::: 0 0 1 2 3
Seq Host Starttime JobRuntime Send Receive Exitval Signal Command
1 : TIMESTAMP 9.999 0 0 1 0 exit 1
2 : TIMESTAMP 9.999 0 0 2 0 exit 2
3 : TIMESTAMP 9.999 0 0 3 0 exit 3
4 : TIMESTAMP 9.999 0 0 0 0 exit 0
Seq Host Starttime JobRuntime Send Receive Exitval Signal Command
1 : TIMESTAMP 9.999 0 0 1 0 exit 1
2 : TIMESTAMP 9.999 0 0 2 0 exit 2
3 : TIMESTAMP 9.999 0 0 3 0 exit 3
4 : TIMESTAMP 9.999 0 0 0 0 exit 0
Seq Host Starttime JobRuntime Send Receive Exitval Signal Command
1 : TIMESTAMP 9.999 0 0 1 0 exit 1
2 : TIMESTAMP 9.999 0 0 2 0 exit 2
3 : TIMESTAMP 9.999 0 0 3 0 exit 3
4 : TIMESTAMP 9.999 0 0 0 0 exit 0
5 : TIMESTAMP 9.999 0 0 0 0 exit 0
6 : TIMESTAMP 9.999 0 0 0 0 exit 0
Seq Host Starttime JobRuntime Send Receive Exitval Signal Command
1 : TIMESTAMP 9.999 0 0 1 0 exit 1
2 : TIMESTAMP 9.999 0 0 2 0 exit 2
3 : TIMESTAMP 9.999 0 0 3 0 exit 3
4 : TIMESTAMP 9.999 0 0 0 0 exit 0
5 : TIMESTAMP 9.999 0 0 0 0 exit 0
6 : TIMESTAMP 9.999 0 0 0 0 exit 0
1 : TIMESTAMP 9.999 0 0 1 0 exit 1
2 : TIMESTAMP 9.999 0 0 2 0 exit 2
3 : TIMESTAMP 9.999 0 0 3 0 exit 3
Seq Host Starttime JobRuntime Send Receive Exitval Signal Command
1 : TIMESTAMP 9.999 0 0 1 0 exit 1
2 : TIMESTAMP 9.999 0 0 2 0 exit 2
3 : TIMESTAMP 9.999 0 0 3 0 exit 3
4 : TIMESTAMP 9.999 0 0 0 0 exit 0
5 : TIMESTAMP 9.999 0 0 0 0 exit 0
6 : TIMESTAMP 9.999 0 0 0 0 exit 0
1 : TIMESTAMP 9.999 0 0 1 0 exit 1
2 : TIMESTAMP 9.999 0 0 2 0 exit 2
3 : TIMESTAMP 9.999 0 0 3 0 exit 3
1 : TIMESTAMP 9.999 0 0 1 0 exit 1
2 : TIMESTAMP 9.999 0 0 2 0 exit 2
3 : TIMESTAMP 9.999 0 0 3 0 exit 3
9
parallel: This job failed:
echo X; exit X
parallel: Starting no more jobs. Waiting for 1 jobs to finish.
parallel: This job failed:
echo X; exit X
parallel -j2 --halt now,fail=1 echo {}\; exit {} ::: 0 0 1 2 3
9
parallel: This job failed:
echo X; exit X
parallel -j2 --halt soon,fail=20% echo {}\; exit {} \
::: 0 1 2 3 4 5 6 7 8 9
9
parallel: This job failed:
echo X; exit X
parallel: This job failed:
echo X; exit X
parallel: Starting no more jobs. Waiting for 1 jobs to finish.
parallel: This job failed:
echo X; exit X
parallel -j2 --halt now,success=1 echo {}\; exit {} ::: 1 2 3 0 4 5 6
9
parallel: This job succeeded:
echo X; exit X
parallel -k --retries 3 \
'echo tried {} >>/tmp/runs; echo completed {}; exit {}' ::: 1 2 0
cat /tmp/runs
completed 1
completed 2
completed 0
tried 0
show_signals() {
perl -e 'for(keys %SIG) {
$SIG{$_} = eval "sub { print \"Got $_\\n\"; }";
}
while(1){sleep 1}'
}
export -f show_signals
echo | parallel --termseq TERM,200,TERM,100,TERM,50,KILL,25 \
-u --timeout 1 show_signals
parallel: Warning: This job was killed because it timed out:
parallel: Warning: show_signals ''
echo | parallel --termseq INT,200,TERM,100,KILL,25 \
-u --timeout 1 show_signals
/bin/bash: show_signals: command not found
parallel --load 100% echo load is less than {} job per cpu ::: 1
load is less than 1 job per cpu
parallel --noswap echo the system is not swapping ::: now
the system is not swapping now
parallel --memfree 1G echo will run if more than 1 GB is ::: free
will run if more than 1 GB is free
parallel --nice 17 echo this is being run with nice -n ::: 17
this is being run with nice -n 17
parallel -S $SERVER1 echo running on ::: $SERVER1
running on parallel@lo
parallel -S $SERVER1 echo running on ::: username@$SERVER1
running on username@parallel@lo
parallel -S : echo running on ::: the_local_machine
running on the_local_machine
parallel -S '/usr/bin/ssh '$SERVER1 echo custom ::: ssh
custom ssh
parallel --ssh /usr/bin/ssh -S $SERVER1 echo custom ::: ssh
custom ssh
export PARALLEL_SSH=/usr/bin/ssh
parallel -S $SERVER1 echo custom ::: ssh
custom ssh
parallel -S $SERVER1 -S $SERVER2 echo ::: running on more hosts
running
on
more
hosts
echo $SERVER1 > nodefile
# Force 4 cores, special ssh-command, username
echo 4//usr/bin/ssh $SERVER2 >> nodefile
parallel --sshloginfile nodefile echo ::: running on more hosts
running
on
more
hosts
parallel -S 4/$SERVER1 echo force {} cpus on server ::: 4
force 4 cpus on server
parallel --hostgroup -S @grp1/$SERVER1 -S @grp2/$SERVER2 echo {} \
::: run_on_grp1@grp1 run_on_grp2@grp2
run_on_grp1
run_on_grp2
echo This is input_file > input_file
parallel -S $SERVER1 --transferfile {} cat ::: input_file
This is input_file
echo This is input_file > input_file
parallel -S $SERVER1 --transferfile {} --return {}.out \
cat {} ">"{}.out ::: input_file
cat input_file.out
This is input_file
echo This is input_file > input_file
parallel -S $SERVER1 --transferfile {} --return {}.out --cleanup \
cat {} ">"{}.out ::: input_file
cat input_file.out
This is input_file
echo This is input_file > input_file
parallel -S $SERVER1 --trc {}.out cat {} ">"{}.out ::: input_file
cat input_file.out
This is input_file
echo common data > common_file
parallel --basefile common_file -S $SERVER1 \
cat common_file\; echo {} ::: foo
common data
foo
parallel -S $SERVER1 pwd ::: ""
parallel --workdir . -S $SERVER1 pwd ::: ""
parallel --workdir ... -S $SERVER1 pwd ::: ""
/mnt/4tb/home/parallel
BASE64/mnt/4tb/home/parallel/.TMPWORKDIR
parallel -S $SERVER1 --sshdelay 0.2 echo ::: 1 2 3
9
parallel --controlmaster -S $SERVER1 echo ::: 1 2 3
9
parallel --filter-hosts -S 173.194.32.46,$SERVER1 echo ::: bar
bar
parallel: Warning: Removed 173.194.32.46.
parallel --onall -S $SERVER1,$SERVER2 echo ::: foo bar
foo
bar
foo
bar
parallel --nonall --tag -S $SERVER1,$SERVER2 echo foo bar
csh@lo foo bar
parallel@lo foo bar
source `which env_parallel.bash`
alias myecho=echo
myvar="Joe's var is"
env_parallel -S $SERVER1 'myecho $myvar' ::: green
Unknown option: -S
Unknown option: parallel@lo
Unknown option: myecho $myvar
Unknown option: :::
Unknown option: green
env_parallel only works if it is a function. Do the below and restart your shell.
bash: Put this in $HOME/.bashrc: . `which env_parallel.bash`
E.g. by doing: echo '. `which env_parallel.bash`' >> $HOME/.bashrc
Supports: aliases, functions, variables, arrays
zsh: Put this in $HOME/.zshrc: . `which env_parallel.zsh`
E.g. by doing: echo '. `which env_parallel.zsh`' >> $HOME/.zshenv
Supports: functions, variables, arrays
fish: Put this in $HOME/.config/fish/config.fish:
. (which env_parallel.fish)
E.g. by doing:
echo '. (which env_parallel.fish)' >> $HOME/.config/fish/config.fish
Supports: aliases, functions, variables, arrays
ksh: Put this in $HOME/.kshrc: source `which env_parallel.ksh`
E.g. by doing: echo 'source `which env_parallel.ksh`' >> $HOME/.kshrc
Supports: aliases, functions, variables, arrays
mksh: Put this in $HOME/.mkshrc: source `which env_parallel.mksh`
E.g. by doing: echo 'source `which env_parallel.mksh`' >> $HOME/.mkshrc
Supports: aliases, functions, variables, arrays
pdksh: Put this in $HOME/.profile: source `which env_parallel.pdksh`
E.g. by doing: echo '. `which env_parallel.pdksh`' >> $HOME/.profile
Supports: aliases, functions, variables, arrays
ash: Put this in $HOME/.profile: . `which env_parallel.ash`
E.g. by doing: echo '. `which env_parallel.ash`' >> $HOME/.profile
Supports: aliases, variables
dash: Put this in $HOME/.profile: . `which env_parallel.dash`
E.g. by doing: echo '. `which env_parallel.dash`' >> $HOME/.profile
Supports: aliases, variables
csh: Put this in $HOME/.cshrc: source `which env_parallel.csh`
E.g. by doing: echo 'source `which env_parallel.csh`' >> $HOME/.cshrc
Supports: aliases, variables, arrays with no special chars
tcsh: Put this in $HOME/.tcshrc: source `which env_parallel.tcsh`
E.g. by doing: echo 'source `which env_parallel.tcsh`' >> $HOME/.tcshrc
Supports: aliases, variables, arrays with no special chars
To install in all shells run:
env_parallel --install
For details: see man env_parallel
MYVAR='foo bar'
export MYVAR
parallel --env MYVAR -S $SERVER1 echo '$MYVAR' ::: baz
foo bar baz
my_func() {
echo in my_func $1
}
export -f my_func
parallel --env my_func -S $SERVER1 my_func ::: baz
in my_func baz
parallel --record-env
cat ~/.parallel/ignored_vars|sort
BASH_FUNC_run_once%%
BASH_FUNC_run_test%%
COLORFGBG
COLORTERM
DBUS_SESSION_BUS_ADDRESS
DEBEMAIL
DEBFULLNAME
DEFAULTS_PATH
DESKTOP_SESSION
DISPLAY
GOPATH
GPG_AGENT_INFO
GS_LIB
HISTCONTROL
HOME
JAVA_HOME
JDK_HOME
KDE_FULL_SESSION
KDE_SESSION_UID
KDE_SESSION_VERSION
KONSOLE_DBUS_SERVICE
KONSOLE_DBUS_SESSION
KONSOLE_DBUS_WINDOW
KONSOLE_PROFILE_NAME
LANG
LANGUAGE
LC_ALL
LC_TIME
LESS
LESSCLOSE
LESSOPEN
LOGNAME
LS_COLORS
MAKEFLAGS
MAKELEVEL
MAKE_TERMERR
MAKE_TERMOUT
MANDATORY_PATH
MFLAGS
OLDPWD
ORACLE_HOME
ORACLE_SID
PARALLEL
PARALLEL_HOME
PARALLEL_PID
PARALLEL_RSYNC_OPTS
PARALLEL_SEQ
PARALLEL_SSHLOGIN
PARALLEL_TMP
PATH
PERL_MB_OPT
PERL_MM_OPT
PROFILEHOME
PWD
QT_ACCESSIBILITY
QT_AUTO_SCREEN_SCALE_FACTOR
SERVER1
SERVER2
SESSION_MANAGER
SHELL
SHELL_SESSION_ID
SHLVL
SSH_AGENT_PID
SSH_AUTH_SOCK
S_COLORS
TERM
TIMEOUT
TMPDIR
TRIES
USER
VISUAL
WINDOWID
XAUTHORITY
XCURSOR_SIZE
XCURSOR_THEME
XDG_CONFIG_DIRS
XDG_CURRENT_DESKTOP
XDG_DATA_DIRS
XDG_RUNTIME_DIR
XDG_SEAT
XDG_SEAT_PATH
XDG_SESSION_CLASS
XDG_SESSION_DESKTOP
XDG_SESSION_ID
XDG_SESSION_PATH
XDG_SESSION_TYPE
XDG_VTNR
_
mysqlrootpass
# The function is only copied if using Bash
my_func2() {
echo in my_func2 $VAR $1
}
export -f my_func2
VAR=foo
export VAR
parallel --env _ -S $SERVER1 'echo $VAR; my_func2' ::: bar
/bin/bash: my_func2: command not found
NOT='not exported var'
alias myecho=echo
not_ex() {
myecho in not_exported_func $NOT $1
}
env_parallel --env _ -S $SERVER1 'echo $NOT; not_ex' ::: bar
Unknown option: --env
Unknown option: _
Unknown option: -S
Unknown option: parallel@lo
Unknown option: echo $NOT; not_ex
Unknown option: :::
Unknown option: bar
env_parallel only works if it is a function. Do the below and restart your shell.
bash: Put this in $HOME/.bashrc: . `which env_parallel.bash`
E.g. by doing: echo '. `which env_parallel.bash`' >> $HOME/.bashrc
Supports: aliases, functions, variables, arrays
zsh: Put this in $HOME/.zshrc: . `which env_parallel.zsh`
E.g. by doing: echo '. `which env_parallel.zsh`' >> $HOME/.zshenv
Supports: functions, variables, arrays
fish: Put this in $HOME/.config/fish/config.fish:
. (which env_parallel.fish)
E.g. by doing:
echo '. (which env_parallel.fish)' >> $HOME/.config/fish/config.fish
Supports: aliases, functions, variables, arrays
ksh: Put this in $HOME/.kshrc: source `which env_parallel.ksh`
E.g. by doing: echo 'source `which env_parallel.ksh`' >> $HOME/.kshrc
Supports: aliases, functions, variables, arrays
mksh: Put this in $HOME/.mkshrc: source `which env_parallel.mksh`
E.g. by doing: echo 'source `which env_parallel.mksh`' >> $HOME/.mkshrc
Supports: aliases, functions, variables, arrays
pdksh: Put this in $HOME/.profile: source `which env_parallel.pdksh`
E.g. by doing: echo '. `which env_parallel.pdksh`' >> $HOME/.profile
Supports: aliases, functions, variables, arrays
ash: Put this in $HOME/.profile: . `which env_parallel.ash`
E.g. by doing: echo '. `which env_parallel.ash`' >> $HOME/.profile
Supports: aliases, variables
dash: Put this in $HOME/.profile: . `which env_parallel.dash`
E.g. by doing: echo '. `which env_parallel.dash`' >> $HOME/.profile
Supports: aliases, variables
csh: Put this in $HOME/.cshrc: source `which env_parallel.csh`
E.g. by doing: echo 'source `which env_parallel.csh`' >> $HOME/.cshrc
Supports: aliases, variables, arrays with no special chars
tcsh: Put this in $HOME/.tcshrc: source `which env_parallel.tcsh`
E.g. by doing: echo 'source `which env_parallel.tcsh`' >> $HOME/.tcshrc
Supports: aliases, variables, arrays with no special chars
To install in all shells run:
env_parallel --install
For details: see man env_parallel
parallel -vv --pipepart --block 1M wc :::: num30000
<num30000 perl -e 'while(@ARGV){sysseek(STDIN,shift,0)||die;$left=shift;while($read=sysread(STDIN,$buf,$left>131072?131072:$left)){$left-=$read;syswrite(STDOUT,$buf);}}' 0 0 0 168894 |(wc)
30000 30000 168894
my_func3() {
echo in my_func $1 > $1.out
}
export -f my_func3
parallel -vv --workdir ... --nice 17 --env _ --trc {}.out \
-S $SERVER1 my_func3 {} ::: abc-file
ssh -l parallel lo -- mkdir -p ./.TMPWORKDIR && rsync --protocol 30 -rlDzR -e'ssh -l parallel' ./abc-file lo:./.TMPWORKDIR;ssh -l parallel lo -- exec perl -e "'"'@GNU_Parallel=split/_/,"use_IPC::Open3;_use_MIME::Base64";eval"@GNU_Parallel";$chld=$SIG{CHLD};$SIG{CHLD}="IGNORE";my$zip=(grep{-x$_}"/usr/local/bin/bzip2")[0]||"bzip2";open3($in,$out,">&STDERR",$zip,"-dc");if(my$perlpid=fork){close$in;$eval=join"",<$out>;close$out;}else{close$out;print$in(decode_base64(join"",@ARGV));close$in;exit;}wait;$SIG{CHLD}=$chld;eval$eval;'"'" BASE64;_EXIT_status=$?; mkdir -p ./. && rsync --protocol 30 --rsync-path='cd ./.TMPWORKDIR/./.; rsync' -rlDzR -e'ssh -l parallel' lo:./abc-file.out ./.;ssh -l parallel lo -- 'rm -f ./.TMPWORKDIR/abc-file; sh -c '"'"'rmdir ./.TMPWORKDIR/ ./.parallel/tmp/ ./.parallel/ 2>/dev/null;'"'"''"'"'rm -rf ./.TMPWORKDIR;'"'";ssh -l parallel lo -- 'rm -f ./.TMPWORKDIR/abc-file.out; sh -c '"'"'rmdir ./.TMPWORKDIR/ ./.parallel/tmp/ ./.parallel/ 2>/dev/null;'"'"''"'"'rm -rf ./.TMPWORKDIR;'"'";ssh -l parallel lo -- rm -rf .TMPWORKDIR;exit $_EXIT_status;
parset myvar1,myvar2 echo ::: a b
echo $myvar1
echo $myvar2
Unknown option: myvar1,myvar2
Unknown option: echo
Unknown option: :::
Unknown option: a
Unknown option: b
parset only works if it is a function. The function is defined as part
of env_parallel.
Do the below and restart your shell.
bash: Put this in $HOME/.bashrc: . `which env_parallel.bash`
E.g. by doing: echo '. `which env_parallel.bash`' >> $HOME/.bashrc
Supports: aliases, functions, variables, arrays
zsh: Put this in $HOME/.zshrc: . `which env_parallel.zsh`
E.g. by doing: echo '. `which env_parallel.zsh`' >> $HOME/.zshenv
Supports: functions, variables, arrays
fish: Unsupported
ksh: Put this in $HOME/.kshrc: source `which env_parallel.ksh`
E.g. by doing: echo 'source `which env_parallel.ksh`' >> $HOME/.kshrc
Supports: aliases, functions, variables, arrays
mksh: Put this in $HOME/.mkshrc: source `which env_parallel.mksh`
E.g. by doing: echo 'source `which env_parallel.mksh`' >> $HOME/.mkshrc
Supports: aliases, functions, variables, arrays
pdksh: Put this in $HOME/.profile: source `which env_parallel.pdksh`
E.g. by doing: echo '. `which env_parallel.pdksh`' >> $HOME/.profile
Supports: aliases, functions, variables, arrays
ash: Put this in $HOME/.profile: . `which env_parallel.ash`
E.g. by doing: echo '. `which env_parallel.ash`' >> $HOME/.profile
Supports: aliases, variables
dash: Put this in $HOME/.profile: . `which env_parallel.dash`
E.g. by doing: echo '. `which env_parallel.dash`' >> $HOME/.profile
Supports: aliases, variables
csh: Unsupported
tcsh: Unsupported
To install in all shells run:
parset --install
For details: see man parset
parset myarray seq {} 5 ::: 1 2 3
echo "${myarray[1]}"
Unknown option: myarray
Unknown option: seq
Unknown option: {}
Unknown option: 5
Unknown option: :::
Unknown option: 1
Unknown option: 2
Unknown option: 3
parset only works if it is a function. The function is defined as part
of env_parallel.
Do the below and restart your shell.
bash: Put this in $HOME/.bashrc: . `which env_parallel.bash`
E.g. by doing: echo '. `which env_parallel.bash`' >> $HOME/.bashrc
Supports: aliases, functions, variables, arrays
zsh: Put this in $HOME/.zshrc: . `which env_parallel.zsh`
E.g. by doing: echo '. `which env_parallel.zsh`' >> $HOME/.zshenv
Supports: functions, variables, arrays
fish: Unsupported
ksh: Put this in $HOME/.kshrc: source `which env_parallel.ksh`
E.g. by doing: echo 'source `which env_parallel.ksh`' >> $HOME/.kshrc
Supports: aliases, functions, variables, arrays
mksh: Put this in $HOME/.mkshrc: source `which env_parallel.mksh`
E.g. by doing: echo 'source `which env_parallel.mksh`' >> $HOME/.mkshrc
Supports: aliases, functions, variables, arrays
pdksh: Put this in $HOME/.profile: source `which env_parallel.pdksh`
E.g. by doing: echo '. `which env_parallel.pdksh`' >> $HOME/.profile
Supports: aliases, functions, variables, arrays
ash: Put this in $HOME/.profile: . `which env_parallel.ash`
E.g. by doing: echo '. `which env_parallel.ash`' >> $HOME/.profile
Supports: aliases, variables
dash: Put this in $HOME/.profile: . `which env_parallel.dash`
E.g. by doing: echo '. `which env_parallel.dash`' >> $HOME/.profile
Supports: aliases, variables
csh: Unsupported
tcsh: Unsupported
To install in all shells run:
parset --install
For details: see man parset
cmd=("echo '<<joe \"double space\" cartoon>>'" "pwd")
parset data ::: "${cmd[@]}"
echo "${data[0]}"
echo "${data[1]}"
Unknown option: data
Unknown option: :::
Unknown option: echo '<<joe "double space" cartoon>>'
Unknown option: pwd
parset only works if it is a function. The function is defined as part
of env_parallel.
Do the below and restart your shell.
bash: Put this in $HOME/.bashrc: . `which env_parallel.bash`
E.g. by doing: echo '. `which env_parallel.bash`' >> $HOME/.bashrc
Supports: aliases, functions, variables, arrays
zsh: Put this in $HOME/.zshrc: . `which env_parallel.zsh`
E.g. by doing: echo '. `which env_parallel.zsh`' >> $HOME/.zshenv
Supports: functions, variables, arrays
fish: Unsupported
ksh: Put this in $HOME/.kshrc: source `which env_parallel.ksh`
E.g. by doing: echo 'source `which env_parallel.ksh`' >> $HOME/.kshrc
Supports: aliases, functions, variables, arrays
mksh: Put this in $HOME/.mkshrc: source `which env_parallel.mksh`
E.g. by doing: echo 'source `which env_parallel.mksh`' >> $HOME/.mkshrc
Supports: aliases, functions, variables, arrays
pdksh: Put this in $HOME/.profile: source `which env_parallel.pdksh`
E.g. by doing: echo '. `which env_parallel.pdksh`' >> $HOME/.profile
Supports: aliases, functions, variables, arrays
ash: Put this in $HOME/.profile: . `which env_parallel.ash`
E.g. by doing: echo '. `which env_parallel.ash`' >> $HOME/.profile
Supports: aliases, variables
dash: Put this in $HOME/.profile: . `which env_parallel.dash`
E.g. by doing: echo '. `which env_parallel.dash`' >> $HOME/.profile
Supports: aliases, variables
csh: Unsupported
tcsh: Unsupported
To install in all shells run:
parset --install
For details: see man parset
parallel --sqlandworker csv:////%2Ftmp%2Flog.csv \
seq ::: 10 ::: 12 13 14
cat /tmp/log.csv
99
Seq,Host,Starttime,JobRuntime,Send,Receive,Exitval,_Signal,Command,V1,V2,Stdout,Stderr
1,:,000000000.000,0.000,0,9,0,0,"seq 10 12",10,12,"10
99
",
2,:,000000000.000,0.000,0,12,0,0,"seq 10 13",10,13,"10
99
",
3,:,000000000.000,0.000,0,15,0,0,"seq 10 14",10,14,"10
99
",
parallel --results outdir --sqlandworker csv:////%2Ftmp%2Flog2.csv \
seq ::: 10 ::: 12 13 14
cat /tmp/log2.csv
99
Seq,Host,Starttime,JobRuntime,Send,Receive,Exitval,_Signal,Command,V1,V2,Stdout,Stderr
1,:,000000000.000,0.000,0,9,0,0,"seq 10 12",10,12,outdir/1/10/2/12/stdout,outdir/1/10/2/12/stderr
2,:,000000000.000,0.000,0,12,0,0,"seq 10 13",10,13,outdir/1/10/2/13/stdout,outdir/1/10/2/13/stderr
3,:,000000000.000,0.000,0,15,0,0,"seq 10 14",10,14,outdir/1/10/2/14/stdout,outdir/1/10/2/14/stderr
vendor://[[user][:password]@][host][:port]/[database[/table]
/bin/bash: vendor://[[user][:password]@][host][:port]/[database[/table]: No such file or directory
sqlite3:///%2Ftmp%2Fmydatabase/mytable
csv:////%2Ftmp%2Flog.csv
/bin/bash: sqlite3:///%2Ftmp%2Fmydatabase/mytable: No such file or directory
/bin/bash: line 1: csv:////%2Ftmp%2Flog.csv: No such file or directory
DBURL=sqlite3:///%2Ftmp%2Fmydatabase
DBURLTABLE=$DBURL/mytable
parallel --sqlandworker $DBURLTABLE echo ::: foo bar ::: baz quuz
foo baz
foo quuz
bar baz
bar quuz
sql $DBURL 'SELECT * FROM mytable ORDER BY Seq;'
Error:
SELECT * FROM mytable ORDER BY Seq; is not a valid DBURL
sql [-hnr] [--table-size] [--db-size] [-p pass-through] [-s string] dburl [command]
parallel --sqlmaster $DBURLTABLE echo ::: foo bar ::: baz quuz
parallel: Error: echo is not a valid DBURL
parallel --sqlworker $DBURLTABLE echo ::: foo bar ::: baz quuz
parallel: Error: echo is not a valid DBURL
cat num1000000 | parallel --pipe wc
999999 999999 9999999
85352 85352 597465
cat num1000000 | parallel --pipe --block 2M wc
999999 999999 9999999
85349 85349 597444
cat num1000000 | parallel --pipe -j4 --roundrobin wc
999999 999999 9999999
cat num1000000 | parallel --pipe -N140000 wc
140000 140000 868895
140000 140000 980000
20000 20000 140001
cat num1000000 | parallel --pipe -L75 wc
999999 999999 9999999
85500 85500 598500
25 25 176
cat fixedlen | parallel --pipe --header .{4} --block 3 --recend '' \
'echo start; cat; echo'
start
HHHHAAA
start
HHHHBBB
start
HHHHCCC
echo /foo, bar/, /baz, qux/, | \
parallel -kN1 --recend ', ' --pipe echo JOB{#}\;cat\;echo END
JOB1
/foo, END
JOB2
bar/, END
JOB3
/baz, END
JOB4
qux/,
END
echo /foo, bar/, /baz, qux/, | \
parallel -kN1 --recstart / --pipe echo JOB{#}\;cat\;echo END
JOB1
/foo, barEND
JOB2
/, END
JOB3
/baz, quxEND
JOB4
/,
END
echo /foo, bar/, /baz, qux/, | \
parallel -kN1 --recend ', ' --recstart / --pipe \
echo JOB{#}\;cat\;echo END
JOB1
/foo, bar/, END
JOB2
/baz, qux/,
END
echo foo,bar,_baz,__qux, | \
parallel -kN1 --regexp --recend ,_+ --pipe \
echo JOB{#}\;cat\;echo END
JOB1
foo,bar,_END
JOB2
baz,__END
JOB3
qux,
END
echo foo,bar,_baz,__qux, | \
parallel -kN1 --rrs --regexp --recend ,_+ --pipe \
echo JOB{#}\;cat\;echo END
JOB1
foo,barEND
JOB2
bazEND
JOB3
qux,
END
cat num_%header | \
parallel --header '(%.*\n)*' --pipe -N3 echo JOB{#}\;cat
cat: num_%header: No such file or directory
cat num_%header | parallel --header 2 --pipe -N3 echo JOB{#}\;cat
cat: num_%header: No such file or directory
parallel --pipepart -a num1000000 --block 3m wc
999999 999999 9999999
126984 126984 888889
cat input_file | parallel command
/bin/bash: This is input_file: command not found
#!/bin/bash
#!/usr/bin/parallel --shebang -r echo
foo
bar
baz
/bin/bash: foo: command not found
/bin/bash: line 1: bar: command not found
/bin/bash: line 2: baz: command not found
cat input_file | parallel command
parallel command ::: foo bar
/bin/bash: This is input_file: command not found
/bin/bash: foo: command not found
/bin/bash: bar: command not found
cat input_file | command
command foo bar
/bin/bash: line 1: foo: command not found
#!/usr/bin/perl
print "@ARGV\n"
Error: no such file "@ARGV\n"
parallel perl_echo ::: foo bar
/bin/bash: perl_echo: command not found
#!/usr/bin/parallel --shebang-wrap /usr/bin/perl
print "@ARGV\n"
Error: no such file "@ARGV\n"
perl_echo foo bar
/bin/bash: perl_echo: command not found
#!/usr/bin/parallel --shebang-wrap /usr/bin/perl
print "Arguments @ARGV\n";
Error: no such file "Arguments @ARGV\n"
#!/usr/bin/parallel --shebang-wrap /usr/bin/python
print 'Arguments', str(sys.argv)
/bin/bash: -c: line 3: syntax error near unexpected token `('
/bin/bash: -c: line 3: ` print 'Arguments', str(sys.argv)'
#!/usr/bin/parallel --shebang-wrap /bin/bash
echo Arguments "$@"
Arguments
#!/usr/bin/parallel --shebang-wrap /bin/csh
echo Arguments "$argv"
Arguments
#!/usr/bin/parallel --shebang-wrap /usr/bin/tclsh
puts "Arguments $argv"
/bin/bash: line 2: puts: command not found
#!/usr/bin/parallel --shebang-wrap /usr/bin/Rscript --vanilla --slave
args <- commandArgs(trailingOnly = TRUE)
print(paste("Arguments ",args))
/bin/bash: -c: line 2: syntax error near unexpected token `('
/bin/bash: -c: line 2: ` args <- commandArgs(trailingOnly = TRUE)'
#!/usr/bin/parallel --shebang-wrap ARG={} /usr/bin/gnuplot
print "Arguments ", system('echo $ARG')
/bin/bash: -c: line 2: syntax error near unexpected token `('
/bin/bash: -c: line 2: ` print "Arguments ", system('echo $ARG')'
#!/usr/bin/parallel --shebang-wrap /usr/bin/ruby
print "Arguments "
puts ARGV
Error: no such file "Arguments "
/bin/bash: line 3: puts: command not found
#!/usr/bin/parallel --shebang-wrap /usr/bin/octave
printf ("Arguments");
arg_list = argv ();
for i = 1:nargin
printf (" %s", arg_list{i});
endfor
printf ("\n");
/bin/bash: -c: line 2: syntax error near unexpected token `"Arguments"'
/bin/bash: -c: line 2: ` printf ("Arguments");'
#!/usr/bin/parallel --shebang-wrap /usr/bin/clisp
(format t "~&~S~&" 'Arguments)
(format t "~&~S~&" *args*)
/bin/bash: -c: line 2: unexpected EOF while looking for matching `''
/bin/bash: -c: line 4: syntax error: unexpected end of file
#!/usr/bin/parallel --shebang-wrap /usr/bin/php
<?php
echo "Arguments";
foreach(array_slice($argv,1) as $v)
{
echo " $v";
}
echo "\n";
?>
Arguments
/bin/bash: line 1: ?php: No such file or directory
/bin/bash: -c: line 3: syntax error near unexpected token `array_slice'
/bin/bash: -c: line 3: ` foreach(array_slice($argv,1) as $v)'
#!/usr/bin/parallel --shebang-wrap /usr/bin/node
var myArgs = process.argv.slice(2);
console.log('Arguments ', myArgs);
/bin/bash: -c: line 0: syntax error near unexpected token `('
/bin/bash: -c: line 0: ` var myArgs = process.argv.slice(2);'
#!/usr/bin/parallel --shebang-wrap /usr/bin/lua
io.write "Arguments"
for a = 1, #arg do
io.write(" ")
io.write(arg[a])
end
print("")
/bin/bash: line 2: io.write: command not found
/bin/bash: -c: line 3: syntax error near unexpected token `='
/bin/bash: -c: line 3: ` for a = 1, #arg do'
#!/usr/bin/parallel --shebang-wrap ARGV={} /usr/bin/csharp
var argv = Environment.GetEnvironmentVariable("ARGV");
print("Arguments "+argv);
/bin/bash: -c: line 2: syntax error near unexpected token `('
/bin/bash: -c: line 2: ` var argv = Environment.GetEnvironmentVariable("ARGV");'
sem 'sleep 1; echo The first finished' &&
echo The first is now running in the background &&
sem 'sleep 1; echo The second finished' &&
echo The second is now running in the background
sem --wait
The first is now running in the background
The first finished
The second is now running in the background
sem --fg 'sleep 1; echo The first finished' &&
echo The first finished running in the foreground &&
sem --fg 'sleep 1; echo The second finished' &&
echo The second finished running in the foreground
sem --wait
The first finished
The first finished running in the foreground
The second finished running in the foreground
sem --id my_id -u 'echo First started; sleep 10; echo First done'
First started
First done
sem --id my_id -u 'echo Second started; sleep 10; echo Second done'
sem --jobs 3 --id my_id -u 'echo Start 1; sleep 5; echo 1 done' &&
sem --jobs 3 --id my_id -u 'echo Start 2; sleep 6; echo 2 done' &&
sem --jobs 3 --id my_id -u 'echo Start 3; sleep 7; echo 3 done' &&
sem --jobs 3 --id my_id -u 'echo Start 4; sleep 8; echo 4 done' &&
sem --wait --id my_id
Start 1
Start 2
1 done
Start 3
2 done
Start 4
3 done
4 done
sem --id foo -u 'echo Slow started; sleep 5; echo Slow ended' &&
sem --id foo --semaphoretimeout 1 'echo Forced running after 1 sec' &&
sem --id foo --semaphoretimeout -2 'echo Give up after 2 secs'
sem --id foo --wait
Slow started
Forced running after 1 sec
Slow ended
parallel: Warning: Semaphore timed out. Stealing the semaphore.
parallel: Warning: Semaphore timed out. Exiting.
parallel --help
Usage:
parallel [options] [command [arguments]] < list_of_arguments
parallel [options] [command [arguments]] (::: arguments|:::: argfile(s))...
cat ... | parallel --pipe [options] [command [arguments]]
-j n Run n jobs in parallel
-k Keep same order
-X Multiple arguments with context replace
--colsep regexp Split input on regexp for positional replacements
{} {.} {/} {/.} {#} {%} {= perl code =} Replacement strings
{3} {3.} {3/} {3/.} {=3 perl code =} Positional replacement strings
With --plus: {} = {+/}/{/} = {.}.{+.} = {+/}/{/.}.{+.} = {..}.{+..} =
{+/}/{/..}.{+..} = {...}.{+...} = {+/}/{/...}.{+...}
-S sshlogin Example: foo@server.example.com
--slf .. Use ~/.parallel/sshloginfile as the list of sshlogins
--trc {}.bar Shorthand for --transfer --return {}.bar --cleanup
--onall Run the given command with argument on all sshlogins
--nonall Run the given command with no arguments on all sshlogins
--pipe Split stdin (standard input) to multiple jobs.
--recend str Record end separator for --pipe.
--recstart str Record start separator for --pipe.
See 'man parallel' for details
Academic tradition requires you to cite works you base your article on.
If you use programs that use GNU Parallel to process data for an article in a
scientific publication, please cite:
O. Tange (2018): GNU Parallel 2018, Mar 2018, ISBN 9781387509881,
DOI https://doi.org/10.5281/zenodo.1146014
This helps funding further development; AND IT WON'T COST YOU A CENT.
If you pay 10000 EUR you should feel free to use GNU Parallel without citing.
parallel --version
GNU parallel VERSION
Copyright (C) 2007-2020 Ole Tange, http://ole.tange.dk and Free Software
Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
GNU parallel comes with no warranty.
Web site: http://www.gnu.org/software/parallel
When using programs that use GNU Parallel to process data for publication
please cite as described in 'parallel --citation'.
parallel --minversion VERSION && \
echo Your version is at least VERSION.
VERSION
Your version is at least VERSION.
parallel --citation
Academic tradition requires you to cite works you base your article on.
If you use programs that use GNU Parallel to process data for an article in a
scientific publication, please cite:
@book{tange_ole_2018_1146014,
author = {Tange, Ole},
title = {GNU Parallel 2018},
publisher = {Ole Tange},
month = Mar,
year = 2018,
ISBN = {9781387509881},
doi = {10.5281/zenodo.1146014},
url = {https://doi.org/10.5281/zenodo.1146014}
}
(Feel free to use \nocite{tange_ole_2018_1146014})
This helps funding further development; AND IT WON'T COST YOU A CENT.
If you pay 10000 EUR you should feel free to use GNU Parallel without citing.
More about funding GNU Parallel and the citation notice:
https://lists.gnu.org/archive/html/parallel/2013-11/msg00006.html
https://www.gnu.org/software/parallel/parallel_design.html#Citation-notice
https://git.savannah.gnu.org/cgit/parallel.git/tree/doc/citation-notice-faq.txt
If you send a copy of your published article to tange@gnu.org, it will be
mentioned in the release notes of next version of GNU Parallel.
parallel --number-of-cpus
parallel --number-of-cores
9
echo '--nice 17' > ~/.parallel/nicetimeout
echo '--timeout 300%' >> ~/.parallel/nicetimeout
parallel --profile nicetimeout echo ::: A B C
A
B
C
echo '-vv --dry-run' > ~/.parallel/dryverbose
parallel --profile dryverbose --profile nicetimeout echo ::: A B C
echo A
echo B
echo C
7
Reference in a new issue View git blame Copy permalink