|
|
|
|
@ -281,46 +281,47 @@ start. GNU B<parallel> only requires one step.
|
|
|
|
|
Here are the examples from B<ppss>'s manual page with the equivalent
|
|
|
|
|
using GNU B<parallel>:
|
|
|
|
|
|
|
|
|
|
B<1> ./ppss.sh standalone -d /path/to/files -c 'gzip '
|
|
|
|
|
1$ ./ppss.sh standalone -d /path/to/files -c 'gzip '
|
|
|
|
|
|
|
|
|
|
B<1> find /path/to/files -type f | parallel gzip
|
|
|
|
|
1$ find /path/to/files -type f | parallel gzip
|
|
|
|
|
|
|
|
|
|
B<2> ./ppss.sh standalone -d /path/to/files -c 'cp "$ITEM" /destination/dir '
|
|
|
|
|
2$ ./ppss.sh standalone -d /path/to/files -c 'cp "$ITEM" /destination/dir '
|
|
|
|
|
|
|
|
|
|
B<2> find /path/to/files -type f | parallel cp {} /destination/dir
|
|
|
|
|
2$ find /path/to/files -type f | parallel cp {} /destination/dir
|
|
|
|
|
|
|
|
|
|
B<3> ./ppss.sh standalone -f list-of-urls.txt -c 'wget -q '
|
|
|
|
|
3$ ./ppss.sh standalone -f list-of-urls.txt -c 'wget -q '
|
|
|
|
|
|
|
|
|
|
B<3> parallel -a list-of-urls.txt wget -q
|
|
|
|
|
3$ parallel -a list-of-urls.txt wget -q
|
|
|
|
|
|
|
|
|
|
B<4> ./ppss.sh standalone -f list-of-urls.txt -c 'wget -q "$ITEM"'
|
|
|
|
|
4$ ./ppss.sh standalone -f list-of-urls.txt -c 'wget -q "$ITEM"'
|
|
|
|
|
|
|
|
|
|
B<4> parallel -a list-of-urls.txt wget -q {}
|
|
|
|
|
4$ parallel -a list-of-urls.txt wget -q {}
|
|
|
|
|
|
|
|
|
|
B<5> ./ppss config -C config.cfg -c 'encode.sh ' -d /source/dir -m
|
|
|
|
|
192.168.1.100 -u ppss -k ppss-key.key -S ./encode.sh -n nodes.txt -o
|
|
|
|
|
/some/output/dir --upload --download ; ./ppss deploy -C config.cfg ;
|
|
|
|
|
./ppss start -C config
|
|
|
|
|
5$ ./ppss config -C config.cfg -c 'encode.sh ' -d /source/dir \
|
|
|
|
|
-m 192.168.1.100 -u ppss -k ppss-key.key -S ./encode.sh \
|
|
|
|
|
-n nodes.txt -o /some/output/dir --upload --download;
|
|
|
|
|
./ppss deploy -C config.cfg
|
|
|
|
|
./ppss start -C config
|
|
|
|
|
|
|
|
|
|
B<5> # parallel does not use configs. If you want a different username put it in nodes.txt: user@hostname
|
|
|
|
|
5$ # parallel does not use configs. If you want a different username put it in nodes.txt: user@hostname
|
|
|
|
|
find source/dir -type f |
|
|
|
|
|
parallel --sshloginfile nodes.txt --trc {.}.mp3 lame -a {} -o {.}.mp3 --preset standard --quiet
|
|
|
|
|
|
|
|
|
|
B<5> find source/dir -type f | parallel --sshloginfile nodes.txt --trc {.}.mp3 lame -a {} -o {.}.mp3 --preset standard --quiet
|
|
|
|
|
6$ ./ppss stop -C config.cfg
|
|
|
|
|
|
|
|
|
|
B<6> ./ppss stop -C config.cfg
|
|
|
|
|
6$ killall -TERM parallel
|
|
|
|
|
|
|
|
|
|
B<6> killall -TERM parallel
|
|
|
|
|
7$ ./ppss pause -C config.cfg
|
|
|
|
|
|
|
|
|
|
B<7> ./ppss pause -C config.cfg
|
|
|
|
|
7$ Press: CTRL-Z or killall -SIGTSTP parallel
|
|
|
|
|
|
|
|
|
|
B<7> Press: CTRL-Z or killall -SIGTSTP parallel
|
|
|
|
|
8$ ./ppss continue -C config.cfg
|
|
|
|
|
|
|
|
|
|
B<8> ./ppss continue -C config.cfg
|
|
|
|
|
8$ Enter: fg or killall -SIGCONT parallel
|
|
|
|
|
|
|
|
|
|
B<8> Enter: fg or killall -SIGCONT parallel
|
|
|
|
|
9$ ./ppss.sh status -C config.cfg
|
|
|
|
|
|
|
|
|
|
B<9> ./ppss.sh status -C config.cfg
|
|
|
|
|
|
|
|
|
|
B<9> killall -SIGUSR2 parallel
|
|
|
|
|
9$ killall -SIGUSR2 parallel
|
|
|
|
|
|
|
|
|
|
https://github.com/louwrentius/PPSS
|
|
|
|
|
|
|
|
|
|
@ -342,55 +343,54 @@ B<pexec> is also a tool for running jobs in parallel.
|
|
|
|
|
Here are the examples from B<pexec>'s info page with the equivalent
|
|
|
|
|
using GNU B<parallel>:
|
|
|
|
|
|
|
|
|
|
B<1> pexec -o sqrt-%s.dat -p "$(seq 10)" -e NUM -n 4 -c -- \
|
|
|
|
|
'echo "scale=10000;sqrt($NUM)" | bc'
|
|
|
|
|
1$ pexec -o sqrt-%s.dat -p "$(seq 10)" -e NUM -n 4 -c -- \
|
|
|
|
|
'echo "scale=10000;sqrt($NUM)" | bc'
|
|
|
|
|
|
|
|
|
|
B<1> seq 10 | parallel -j4 'echo "scale=10000;sqrt({})" | bc > sqrt-{}.dat'
|
|
|
|
|
1$ seq 10 | parallel -j4 'echo "scale=10000;sqrt({})" | \
|
|
|
|
|
bc > sqrt-{}.dat'
|
|
|
|
|
|
|
|
|
|
B<2> pexec -p "$(ls myfiles*.ext)" -i %s -o %s.sort -- sort
|
|
|
|
|
2$ pexec -p "$(ls myfiles*.ext)" -i %s -o %s.sort -- sort
|
|
|
|
|
|
|
|
|
|
B<2> ls myfiles*.ext | parallel sort {} ">{}.sort"
|
|
|
|
|
2$ ls myfiles*.ext | parallel sort {} ">{}.sort"
|
|
|
|
|
|
|
|
|
|
B<3> pexec -f image.list -n auto -e B -u star.log -c -- \
|
|
|
|
|
'fistar $B.fits -f 100 -F id,x,y,flux -o $B.star'
|
|
|
|
|
3$ pexec -f image.list -n auto -e B -u star.log -c -- \
|
|
|
|
|
'fistar $B.fits -f 100 -F id,x,y,flux -o $B.star'
|
|
|
|
|
|
|
|
|
|
B<3> parallel -a image.list \
|
|
|
|
|
'fistar {}.fits -f 100 -F id,x,y,flux -o {}.star' 2>star.log
|
|
|
|
|
3$ parallel -a image.list \
|
|
|
|
|
'fistar {}.fits -f 100 -F id,x,y,flux -o {}.star' 2>star.log
|
|
|
|
|
|
|
|
|
|
B<4> pexec -r *.png -e IMG -c -o - -- \
|
|
|
|
|
'convert $IMG ${IMG%.png}.jpeg ; "echo $IMG: done"'
|
|
|
|
|
4$ pexec -r *.png -e IMG -c -o - -- \
|
|
|
|
|
'convert $IMG ${IMG%.png}.jpeg ; "echo $IMG: done"'
|
|
|
|
|
|
|
|
|
|
B<4> ls *.png | parallel 'convert {} {.}.jpeg; echo {}: done'
|
|
|
|
|
4$ ls *.png | parallel 'convert {} {.}.jpeg; echo {}: done'
|
|
|
|
|
|
|
|
|
|
B<5> pexec -r *.png -i %s -o %s.jpg -c 'pngtopnm | pnmtojpeg'
|
|
|
|
|
5$ pexec -r *.png -i %s -o %s.jpg -c 'pngtopnm | pnmtojpeg'
|
|
|
|
|
|
|
|
|
|
B<5> ls *.png | parallel 'pngtopnm < {} | pnmtojpeg > {}.jpg'
|
|
|
|
|
5$ ls *.png | parallel 'pngtopnm < {} | pnmtojpeg > {}.jpg'
|
|
|
|
|
|
|
|
|
|
B<6> for p in *.png ; do echo ${p%.png} ; done | \
|
|
|
|
|
pexec -f - -i %s.png -o %s.jpg -c 'pngtopnm | pnmtojpeg'
|
|
|
|
|
6$ for p in *.png ; do echo ${p%.png} ; done | \
|
|
|
|
|
pexec -f - -i %s.png -o %s.jpg -c 'pngtopnm | pnmtojpeg'
|
|
|
|
|
|
|
|
|
|
B<6> ls *.png | parallel 'pngtopnm < {} | pnmtojpeg > {.}.jpg'
|
|
|
|
|
6$ ls *.png | parallel 'pngtopnm < {} | pnmtojpeg > {.}.jpg'
|
|
|
|
|
|
|
|
|
|
B<7> LIST=$(for p in *.png ; do echo ${p%.png} ; done)
|
|
|
|
|
pexec -r $LIST -i %s.png -o %s.jpg -c 'pngtopnm | pnmtojpeg'
|
|
|
|
|
7$ LIST=$(for p in *.png ; do echo ${p%.png} ; done)
|
|
|
|
|
pexec -r $LIST -i %s.png -o %s.jpg -c 'pngtopnm | pnmtojpeg'
|
|
|
|
|
|
|
|
|
|
B<7> ls *.png | parallel 'pngtopnm < {} | pnmtojpeg > {.}.jpg'
|
|
|
|
|
7$ ls *.png | parallel 'pngtopnm < {} | pnmtojpeg > {.}.jpg'
|
|
|
|
|
|
|
|
|
|
B<8> pexec -n 8 -r *.jpg -y unix -e IMG -c \
|
|
|
|
|
'pexec -j -m blockread -d $IMG | \
|
|
|
|
|
jpegtopnm | pnmscale 0.5 | pnmtojpeg | \
|
|
|
|
|
pexec -j -m blockwrite -s th_$IMG'
|
|
|
|
|
8$ pexec -n 8 -r *.jpg -y unix -e IMG -c \
|
|
|
|
|
'pexec -j -m blockread -d $IMG | \
|
|
|
|
|
jpegtopnm | pnmscale 0.5 | pnmtojpeg | \
|
|
|
|
|
pexec -j -m blockwrite -s th_$IMG'
|
|
|
|
|
|
|
|
|
|
B<8> Combining GNU B<parallel> and GNU B<sem>.
|
|
|
|
|
8$ # Combining GNU B<parallel> and GNU B<sem>.
|
|
|
|
|
ls *jpg | parallel -j8 'sem --id blockread cat {} | jpegtopnm |' \
|
|
|
|
|
'pnmscale 0.5 | pnmtojpeg | sem --id blockwrite cat > th_{}'
|
|
|
|
|
|
|
|
|
|
B<8> ls *jpg | parallel -j8 'sem --id blockread cat {} | jpegtopnm |' \
|
|
|
|
|
'pnmscale 0.5 | pnmtojpeg | sem --id blockwrite cat > th_{}'
|
|
|
|
|
|
|
|
|
|
B<8> If reading and writing is done to the same disk, this may be
|
|
|
|
|
faster as only one process will be either reading or writing:
|
|
|
|
|
|
|
|
|
|
B<8> ls *jpg | parallel -j8 'sem --id diskio cat {} | jpegtopnm |' \
|
|
|
|
|
'pnmscale 0.5 | pnmtojpeg | sem --id diskio cat > th_{}'
|
|
|
|
|
# If reading and writing is done to the same disk, this may be
|
|
|
|
|
# faster as only one process will be either reading or writing:
|
|
|
|
|
ls *jpg | parallel -j8 'sem --id diskio cat {} | jpegtopnm |' \
|
|
|
|
|
'pnmscale 0.5 | pnmtojpeg | sem --id diskio cat > th_{}'
|
|
|
|
|
|
|
|
|
|
https://www.gnu.org/software/pexec/
|
|
|
|
|
|
|
|
|
|
@ -403,38 +403,40 @@ running jobs on your local computer.
|
|
|
|
|
B<xjobs> deals badly with special characters just like B<xargs>. See
|
|
|
|
|
the section B<DIFFERENCES BETWEEN xargs AND GNU Parallel>.
|
|
|
|
|
|
|
|
|
|
=head3 EXAMPLES FROM xjobs MANUAL
|
|
|
|
|
|
|
|
|
|
Here are the examples from B<xjobs>'s man page with the equivalent
|
|
|
|
|
using GNU B<parallel>:
|
|
|
|
|
|
|
|
|
|
B<1> ls -1 *.zip | xjobs unzip
|
|
|
|
|
1$ ls -1 *.zip | xjobs unzip
|
|
|
|
|
|
|
|
|
|
B<1> ls *.zip | parallel unzip
|
|
|
|
|
1$ ls *.zip | parallel unzip
|
|
|
|
|
|
|
|
|
|
B<2> ls -1 *.zip | xjobs -n unzip
|
|
|
|
|
2$ ls -1 *.zip | xjobs -n unzip
|
|
|
|
|
|
|
|
|
|
B<2> ls *.zip | parallel unzip >/dev/null
|
|
|
|
|
2$ ls *.zip | parallel unzip >/dev/null
|
|
|
|
|
|
|
|
|
|
B<3> find . -name '*.bak' | xjobs gzip
|
|
|
|
|
3$ find . -name '*.bak' | xjobs gzip
|
|
|
|
|
|
|
|
|
|
B<3> find . -name '*.bak' | parallel gzip
|
|
|
|
|
3$ find . -name '*.bak' | parallel gzip
|
|
|
|
|
|
|
|
|
|
B<4> ls -1 *.jar | sed 's/\(.*\)/\1 > \1.idx/' | xjobs jar tf
|
|
|
|
|
4$ ls -1 *.jar | sed 's/\(.*\)/\1 > \1.idx/' | xjobs jar tf
|
|
|
|
|
|
|
|
|
|
B<4> ls *.jar | parallel jar tf {} '>' {}.idx
|
|
|
|
|
4$ ls *.jar | parallel jar tf {} '>' {}.idx
|
|
|
|
|
|
|
|
|
|
B<5> xjobs -s script
|
|
|
|
|
5$ xjobs -s script
|
|
|
|
|
|
|
|
|
|
B<5> cat script | parallel
|
|
|
|
|
5$ cat script | parallel
|
|
|
|
|
|
|
|
|
|
B<6> mkfifo /var/run/my_named_pipe;
|
|
|
|
|
xjobs -s /var/run/my_named_pipe &
|
|
|
|
|
echo unzip 1.zip >> /var/run/my_named_pipe;
|
|
|
|
|
echo tar cf /backup/myhome.tar /home/me >> /var/run/my_named_pipe
|
|
|
|
|
6$ mkfifo /var/run/my_named_pipe;
|
|
|
|
|
xjobs -s /var/run/my_named_pipe &
|
|
|
|
|
echo unzip 1.zip >> /var/run/my_named_pipe;
|
|
|
|
|
echo tar cf /backup/myhome.tar /home/me >> /var/run/my_named_pipe
|
|
|
|
|
|
|
|
|
|
B<6> mkfifo /var/run/my_named_pipe;
|
|
|
|
|
cat /var/run/my_named_pipe | parallel &
|
|
|
|
|
echo unzip 1.zip >> /var/run/my_named_pipe;
|
|
|
|
|
echo tar cf /backup/myhome.tar /home/me >> /var/run/my_named_pipe
|
|
|
|
|
6$ mkfifo /var/run/my_named_pipe;
|
|
|
|
|
cat /var/run/my_named_pipe | parallel &
|
|
|
|
|
echo unzip 1.zip >> /var/run/my_named_pipe;
|
|
|
|
|
echo tar cf /backup/myhome.tar /home/me >> /var/run/my_named_pipe
|
|
|
|
|
|
|
|
|
|
http://www.maier-komor.de/xjobs.html (Last checked: 2019-01)
|
|
|
|
|
|
|
|
|
|
@ -452,11 +454,14 @@ B<prll> generates a lot of status information on stderr (standard
|
|
|
|
|
error) which makes it harder to use the stderr (standard error) output
|
|
|
|
|
of the job directly as input for another program.
|
|
|
|
|
|
|
|
|
|
=head3 EXAMPLES FROM prll's MANUAL
|
|
|
|
|
|
|
|
|
|
Here is the example from B<prll>'s man page with the equivalent
|
|
|
|
|
using GNU B<parallel>:
|
|
|
|
|
|
|
|
|
|
prll -s 'mogrify -flip $1' *.jpg
|
|
|
|
|
parallel mogrify -flip ::: *.jpg
|
|
|
|
|
1$ prll -s 'mogrify -flip $1' *.jpg
|
|
|
|
|
|
|
|
|
|
1$ parallel mogrify -flip ::: *.jpg
|
|
|
|
|
|
|
|
|
|
https://github.com/exzombie/prll (Last checked: 2019-01)
|
|
|
|
|
|
|
|
|
|
@ -477,13 +482,15 @@ semicomplete.com/blog/geekery/distributed-xargs.html (Last checked: 2019-01)
|
|
|
|
|
|
|
|
|
|
middleman(mdm) is also a tool for running jobs in parallel.
|
|
|
|
|
|
|
|
|
|
=head3 EXAMPLES FROM middleman's WEBSITE
|
|
|
|
|
|
|
|
|
|
Here are the shellscripts of
|
|
|
|
|
https://web.archive.org/web/20110728064735/http://mdm.
|
|
|
|
|
berlios.de/usage.html ported to GNU B<parallel>:
|
|
|
|
|
|
|
|
|
|
seq 19 | parallel buffon -o - | sort -n > result
|
|
|
|
|
cat files | parallel cmd
|
|
|
|
|
find dir -execdir sem cmd {} \;
|
|
|
|
|
1$ seq 19 | parallel buffon -o - | sort -n > result
|
|
|
|
|
cat files | parallel cmd
|
|
|
|
|
find dir -execdir sem cmd {} \;
|
|
|
|
|
|
|
|
|
|
https://github.com/cklin/mdm (Last checked: 2019-01)
|
|
|
|
|
|
|
|
|
|
@ -492,56 +499,58 @@ https://github.com/cklin/mdm (Last checked: 2019-01)
|
|
|
|
|
|
|
|
|
|
B<xapply> can run jobs in parallel on the local computer.
|
|
|
|
|
|
|
|
|
|
=head3 EXAMPLES FROM xapply's MANUAL
|
|
|
|
|
|
|
|
|
|
Here are the examples from B<xapply>'s man page with the equivalent
|
|
|
|
|
using GNU B<parallel>:
|
|
|
|
|
|
|
|
|
|
B<1> xapply '(cd %1 && make all)' */
|
|
|
|
|
1$ xapply '(cd %1 && make all)' */
|
|
|
|
|
|
|
|
|
|
B<1> parallel 'cd {} && make all' ::: */
|
|
|
|
|
1$ parallel 'cd {} && make all' ::: */
|
|
|
|
|
|
|
|
|
|
B<2> xapply -f 'diff %1 ../version5/%1' manifest | more
|
|
|
|
|
2$ xapply -f 'diff %1 ../version5/%1' manifest | more
|
|
|
|
|
|
|
|
|
|
B<2> parallel diff {} ../version5/{} < manifest | more
|
|
|
|
|
2$ parallel diff {} ../version5/{} < manifest | more
|
|
|
|
|
|
|
|
|
|
B<3> xapply -p/dev/null -f 'diff %1 %2' manifest1 checklist1
|
|
|
|
|
3$ xapply -p/dev/null -f 'diff %1 %2' manifest1 checklist1
|
|
|
|
|
|
|
|
|
|
B<3> parallel --link diff {1} {2} :::: manifest1 checklist1
|
|
|
|
|
3$ parallel --link diff {1} {2} :::: manifest1 checklist1
|
|
|
|
|
|
|
|
|
|
B<4> xapply 'indent' *.c
|
|
|
|
|
4$ xapply 'indent' *.c
|
|
|
|
|
|
|
|
|
|
B<4> parallel indent ::: *.c
|
|
|
|
|
4$ parallel indent ::: *.c
|
|
|
|
|
|
|
|
|
|
B<5> find ~ksb/bin -type f ! -perm -111 -print | xapply -f -v 'chmod a+x' -
|
|
|
|
|
5$ find ~ksb/bin -type f ! -perm -111 -print | xapply -f -v 'chmod a+x' -
|
|
|
|
|
|
|
|
|
|
B<5> find ~ksb/bin -type f ! -perm -111 -print | parallel -v chmod a+x
|
|
|
|
|
5$ find ~ksb/bin -type f ! -perm -111 -print | parallel -v chmod a+x
|
|
|
|
|
|
|
|
|
|
B<6> find */ -... | fmt 960 1024 | xapply -f -i /dev/tty 'vi' -
|
|
|
|
|
6$ find */ -... | fmt 960 1024 | xapply -f -i /dev/tty 'vi' -
|
|
|
|
|
|
|
|
|
|
B<6> sh <(find */ -... | parallel -s 1024 echo vi)
|
|
|
|
|
6$ sh <(find */ -... | parallel -s 1024 echo vi)
|
|
|
|
|
|
|
|
|
|
B<6> find */ -... | parallel -s 1024 -Xuj1 vi
|
|
|
|
|
6$ find */ -... | parallel -s 1024 -Xuj1 vi
|
|
|
|
|
|
|
|
|
|
B<7> find ... | xapply -f -5 -i /dev/tty 'vi' - - - - -
|
|
|
|
|
7$ find ... | xapply -f -5 -i /dev/tty 'vi' - - - - -
|
|
|
|
|
|
|
|
|
|
B<7> sh <(find ... |parallel -n5 echo vi)
|
|
|
|
|
7$ sh <(find ... |parallel -n5 echo vi)
|
|
|
|
|
|
|
|
|
|
B<7> find ... |parallel -n5 -uj1 vi
|
|
|
|
|
7$ find ... |parallel -n5 -uj1 vi
|
|
|
|
|
|
|
|
|
|
B<8> xapply -fn "" /etc/passwd
|
|
|
|
|
8$ xapply -fn "" /etc/passwd
|
|
|
|
|
|
|
|
|
|
B<8> parallel -k echo < /etc/passwd
|
|
|
|
|
8$ parallel -k echo < /etc/passwd
|
|
|
|
|
|
|
|
|
|
B<9> tr ':' '\012' < /etc/passwd | xapply -7 -nf 'chown %1 %6' - - - - - - -
|
|
|
|
|
9$ tr ':' '\012' < /etc/passwd | xapply -7 -nf 'chown %1 %6' - - - - - - -
|
|
|
|
|
|
|
|
|
|
B<9> tr ':' '\012' < /etc/passwd | parallel -N7 chown {1} {6}
|
|
|
|
|
9$ tr ':' '\012' < /etc/passwd | parallel -N7 chown {1} {6}
|
|
|
|
|
|
|
|
|
|
B<10> xapply '[ -d %1/RCS ] || echo %1' */
|
|
|
|
|
10$ xapply '[ -d %1/RCS ] || echo %1' */
|
|
|
|
|
|
|
|
|
|
B<10> parallel '[ -d {}/RCS ] || echo {}' ::: */
|
|
|
|
|
10$ parallel '[ -d {}/RCS ] || echo {}' ::: */
|
|
|
|
|
|
|
|
|
|
B<11> xapply -f '[ -f %1 ] && echo %1' List | ...
|
|
|
|
|
11$ xapply -f '[ -f %1 ] && echo %1' List | ...
|
|
|
|
|
|
|
|
|
|
B<11> parallel '[ -f {} ] && echo {}' < List | ...
|
|
|
|
|
11$ parallel '[ -f {} ] && echo {}' < List | ...
|
|
|
|
|
|
|
|
|
|
https://web.archive.org/web/20160702211113/
|
|
|
|
|
http://carrera.databits.net/~ksb/msrc/local/bin/xapply/xapply.html
|
|
|
|
|
@ -554,30 +563,32 @@ very much like GNU B<parallel>. B<apply> does not run jobs in
|
|
|
|
|
parallel. B<apply> does not use an argument separator (like B<:::>);
|
|
|
|
|
instead the template must be the first argument.
|
|
|
|
|
|
|
|
|
|
=head3 EXAMPLES FROM IBM's KNOWLEDGE CENTER
|
|
|
|
|
|
|
|
|
|
Here are the examples from IBM's Knowledge Center and the
|
|
|
|
|
corresponding command using GNU B<parallel>:
|
|
|
|
|
|
|
|
|
|
1. To obtain results similar to those of the B<ls> command, enter:
|
|
|
|
|
=head4 To obtain results similar to those of the B<ls> command, enter:
|
|
|
|
|
|
|
|
|
|
apply echo *
|
|
|
|
|
parallel echo ::: *
|
|
|
|
|
1$ apply echo *
|
|
|
|
|
1$ parallel echo ::: *
|
|
|
|
|
|
|
|
|
|
2. To compare the file named B<a1> to the file named B<b1>, and the
|
|
|
|
|
file named B<a2> to the file named B<b2>, enter:
|
|
|
|
|
=head4 To compare the file named a1 to the file named b1, and
|
|
|
|
|
the file named a2 to the file named b2, enter:
|
|
|
|
|
|
|
|
|
|
apply -2 cmp a1 b1 a2 b2
|
|
|
|
|
parallel -N2 cmp ::: a1 b1 a2 b2
|
|
|
|
|
2$ apply -2 cmp a1 b1 a2 b2
|
|
|
|
|
2$ parallel -N2 cmp ::: a1 b1 a2 b2
|
|
|
|
|
|
|
|
|
|
3. To run the B<who> command five times, enter:
|
|
|
|
|
=head4 To run the B<who> command five times, enter:
|
|
|
|
|
|
|
|
|
|
apply -0 who 1 2 3 4 5
|
|
|
|
|
parallel -N0 who ::: 1 2 3 4 5
|
|
|
|
|
3$ apply -0 who 1 2 3 4 5
|
|
|
|
|
3$ parallel -N0 who ::: 1 2 3 4 5
|
|
|
|
|
|
|
|
|
|
4. To link all files in the current directory to the directory
|
|
|
|
|
B</usr/joe>, enter:
|
|
|
|
|
=head4 To link all files in the current directory to the directory
|
|
|
|
|
/usr/joe, enter:
|
|
|
|
|
|
|
|
|
|
apply 'ln %1 /usr/joe' *
|
|
|
|
|
parallel ln {} /usr/joe ::: *
|
|
|
|
|
4$ apply 'ln %1 /usr/joe' *
|
|
|
|
|
4$ parallel ln {} /usr/joe ::: *
|
|
|
|
|
|
|
|
|
|
https://www-01.ibm.com/support/knowledgecenter/
|
|
|
|
|
ssw_aix_71/com.ibm.aix.cmds1/apply.htm (Last checked: 2019-01)
|
|
|
|
|
@ -593,46 +604,60 @@ output. This means you will have to write a wrapper for most programs.
|
|
|
|
|
B<paexec> has a job dependency facility so a job can depend on another
|
|
|
|
|
job to be executed successfully. Sort of a poor-man's B<make>.
|
|
|
|
|
|
|
|
|
|
=head3 EXAMPLES FROM paexec's EXAMPLE CATALOG
|
|
|
|
|
|
|
|
|
|
Here are the examples from B<paexec>'s example catalog with the equivalent
|
|
|
|
|
using GNU B<parallel>:
|
|
|
|
|
|
|
|
|
|
=over 1
|
|
|
|
|
=head4 1_div_X_run
|
|
|
|
|
|
|
|
|
|
=item 1_div_X_run:
|
|
|
|
|
1$ ../../paexec -s -l -c "`pwd`/1_div_X_cmd" -n +1 <<EOF [...]
|
|
|
|
|
|
|
|
|
|
../../paexec -s -l -c "`pwd`/1_div_X_cmd" -n +1 <<EOF [...]
|
|
|
|
|
parallel echo {} '|' `pwd`/1_div_X_cmd <<EOF [...]
|
|
|
|
|
1$ parallel echo {} '|' `pwd`/1_div_X_cmd <<EOF [...]
|
|
|
|
|
|
|
|
|
|
=item all_substr_run:
|
|
|
|
|
=head4 all_substr_run
|
|
|
|
|
|
|
|
|
|
../../paexec -lp -c "`pwd`/all_substr_cmd" -n +3 <<EOF [...]
|
|
|
|
|
parallel echo {} '|' `pwd`/all_substr_cmd <<EOF [...]
|
|
|
|
|
2$ ../../paexec -lp -c "`pwd`/all_substr_cmd" -n +3 <<EOF [...]
|
|
|
|
|
|
|
|
|
|
=item cc_wrapper_run:
|
|
|
|
|
2$ parallel echo {} '|' `pwd`/all_substr_cmd <<EOF [...]
|
|
|
|
|
|
|
|
|
|
../../paexec -c "env CC=gcc CFLAGS=-O2 `pwd`/cc_wrapper_cmd" \
|
|
|
|
|
=head4 cc_wrapper_run
|
|
|
|
|
|
|
|
|
|
3$ ../../paexec -c "env CC=gcc CFLAGS=-O2 `pwd`/cc_wrapper_cmd" \
|
|
|
|
|
-n 'host1 host2' \
|
|
|
|
|
-t '/usr/bin/ssh -x' <<EOF [...]
|
|
|
|
|
parallel echo {} '|' "env CC=gcc CFLAGS=-O2 `pwd`/cc_wrapper_cmd" \
|
|
|
|
|
-S host1,host2 <<EOF [...]
|
|
|
|
|
# This is not exactly the same, but avoids the wrapper
|
|
|
|
|
parallel gcc -O2 -c -o {.}.o {} \
|
|
|
|
|
|
|
|
|
|
3$ parallel echo {} '|' "env CC=gcc CFLAGS=-O2 `pwd`/cc_wrapper_cmd" \
|
|
|
|
|
-S host1,host2 <<EOF [...]
|
|
|
|
|
|
|
|
|
|
=item toupper_run:
|
|
|
|
|
# This is not exactly the same, but avoids the wrapper
|
|
|
|
|
parallel gcc -O2 -c -o {.}.o {} \
|
|
|
|
|
-S host1,host2 <<EOF [...]
|
|
|
|
|
|
|
|
|
|
../../paexec -lp -c "`pwd`/toupper_cmd" -n +10 <<EOF [...]
|
|
|
|
|
parallel echo {} '|' ./toupper_cmd <<EOF [...]
|
|
|
|
|
# Without the wrapper:
|
|
|
|
|
parallel echo {} '| awk {print\ toupper\(\$0\)}' <<EOF [...]
|
|
|
|
|
=head4 toupper_run
|
|
|
|
|
|
|
|
|
|
=back
|
|
|
|
|
4$ ../../paexec -lp -c "`pwd`/toupper_cmd" -n +10 <<EOF [...]
|
|
|
|
|
|
|
|
|
|
4$ parallel echo {} '|' ./toupper_cmd <<EOF [...]
|
|
|
|
|
|
|
|
|
|
# Without the wrapper:
|
|
|
|
|
parallel echo {} '| awk {print\ toupper\(\$0\)}' <<EOF [...]
|
|
|
|
|
|
|
|
|
|
https://github.com/cheusov/paexec
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
=head2 DIFFERENCES BETWEEN map(sitaramc) AND GNU Parallel
|
|
|
|
|
|
|
|
|
|
Summary table (see legend above):
|
|
|
|
|
I1 - - I4 - - -
|
|
|
|
|
M1 (M2) M3 M4 M5 -
|
|
|
|
|
- O2 O3 - O5 - - N/A N/A O10
|
|
|
|
|
E1 - - - - - -
|
|
|
|
|
- - - - - - - - -
|
|
|
|
|
- -
|
|
|
|
|
|
|
|
|
|
(M2): Only if there is a single replacement string.
|
|
|
|
|
|
|
|
|
|
B<map> sees it as a feature to have less features and in doing so it
|
|
|
|
|
also handles corner cases incorrectly. A lot of GNU B<parallel>'s code
|
|
|
|
|
is to handle corner cases correctly on every platform, so you will not
|
|
|
|
|
@ -707,8 +732,14 @@ context replace:
|
|
|
|
|
|
|
|
|
|
B<map> requires Perl v5.10.0 making it harder to use on old systems.
|
|
|
|
|
|
|
|
|
|
B<map> has no way of using % in the command (GNU B<parallel> has -I to
|
|
|
|
|
specify another replacement string than B<{}>).
|
|
|
|
|
To put a % in the command line B<map> requires you to use %%:
|
|
|
|
|
|
|
|
|
|
seq 10 | map -n1 echo % +10%% {}
|
|
|
|
|
|
|
|
|
|
seq 10 | parallel -I ,, echo ,, +10% {}
|
|
|
|
|
|
|
|
|
|
GNU B<parallel> has -I to specify another replacement string than
|
|
|
|
|
B<{}> if you use B<{}> in the command template.
|
|
|
|
|
|
|
|
|
|
By design B<map> is option incompatible with B<xargs>, it does not
|
|
|
|
|
have remote job execution, a structured way of saving results,
|
|
|
|
|
@ -717,7 +748,7 @@ delimiter (only field delimiter), logging of jobs run with possibility
|
|
|
|
|
to resume, keeping the output in the same order as input, --pipe
|
|
|
|
|
processing, and dynamically timeouts.
|
|
|
|
|
|
|
|
|
|
https://github.com/sitaramc/map
|
|
|
|
|
https://github.com/sitaramc/map (Last checked: 2020-04)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
=head2 DIFFERENCES BETWEEN ladon AND GNU Parallel
|
|
|
|
|
@ -753,21 +784,25 @@ fails for output larger than 200k:
|
|
|
|
|
It is assumed that the '--rpl's above are put in B<~/.parallel/config>
|
|
|
|
|
and that it is run under a shell that supports '**' globbing (such as B<zsh>):
|
|
|
|
|
|
|
|
|
|
B<1> ladon "**/*.txt" -- echo RELPATH
|
|
|
|
|
1$ ladon "**/*.txt" -- echo RELPATH
|
|
|
|
|
|
|
|
|
|
B<1> parallel echo RELPATH ::: **/*.txt
|
|
|
|
|
1$ parallel echo RELPATH ::: **/*.txt
|
|
|
|
|
|
|
|
|
|
B<2> ladon "~/Documents/**/*.pdf" -- shasum FULLPATH >hashes.txt
|
|
|
|
|
2$ ladon "~/Documents/**/*.pdf" -- shasum FULLPATH >hashes.txt
|
|
|
|
|
|
|
|
|
|
B<2> parallel shasum FULLPATH ::: ~/Documents/**/*.pdf >hashes.txt
|
|
|
|
|
2$ parallel shasum FULLPATH ::: ~/Documents/**/*.pdf >hashes.txt
|
|
|
|
|
|
|
|
|
|
B<3> ladon -m thumbs/RELDIR "**/*.jpg" -- convert FULLPATH -thumbnail 100x100^ -gravity center -extent 100x100 thumbs/RELPATH
|
|
|
|
|
3$ ladon -m thumbs/RELDIR "**/*.jpg" -- convert FULLPATH \
|
|
|
|
|
-thumbnail 100x100^ -gravity center -extent 100x100 \
|
|
|
|
|
thumbs/RELPATH
|
|
|
|
|
|
|
|
|
|
B<3> parallel mkdir -p thumbs/RELDIR\; convert FULLPATH -thumbnail 100x100^ -gravity center -extent 100x100 thumbs/RELPATH ::: **/*.jpg
|
|
|
|
|
3$ parallel mkdir -p thumbs/RELDIR\; convert FULLPATH
|
|
|
|
|
-thumbnail 100x100^ -gravity center -extent 100x100 \
|
|
|
|
|
thumbs/RELPATH ::: **/*.jpg
|
|
|
|
|
|
|
|
|
|
B<4> ladon "~/Music/*.wav" -- lame -V 2 FULLPATH DIRNAME/BASENAME.mp3
|
|
|
|
|
4$ ladon "~/Music/*.wav" -- lame -V 2 FULLPATH DIRNAME/BASENAME.mp3
|
|
|
|
|
|
|
|
|
|
B<4> parallel lame -V 2 FULLPATH DIRNAME/BASENAME.mp3 ::: ~/Music/*.wav
|
|
|
|
|
4$ parallel lame -V 2 FULLPATH DIRNAME/BASENAME.mp3 ::: ~/Music/*.wav
|
|
|
|
|
|
|
|
|
|
https://github.com/danielgtaylor/ladon (Last checked: 2019-01)
|
|
|
|
|
|
|
|
|
|
@ -802,21 +837,21 @@ jobs. This can be emulated by GNU B<parallel> using B<bash>'s B<ulimit>:
|
|
|
|
|
|
|
|
|
|
=head3 EXAMPLES FROM jobflow README
|
|
|
|
|
|
|
|
|
|
B<1> cat things.list | jobflow -threads=8 -exec ./mytask {}
|
|
|
|
|
1$ cat things.list | jobflow -threads=8 -exec ./mytask {}
|
|
|
|
|
|
|
|
|
|
B<1> cat things.list | parallel -j8 ./mytask {}
|
|
|
|
|
1$ cat things.list | parallel -j8 ./mytask {}
|
|
|
|
|
|
|
|
|
|
B<2> seq 100 | jobflow -threads=100 -exec echo {}
|
|
|
|
|
2$ seq 100 | jobflow -threads=100 -exec echo {}
|
|
|
|
|
|
|
|
|
|
B<2> seq 100 | parallel -j100 echo {}
|
|
|
|
|
2$ seq 100 | parallel -j100 echo {}
|
|
|
|
|
|
|
|
|
|
B<3> cat urls.txt | jobflow -threads=32 -exec wget {}
|
|
|
|
|
3$ cat urls.txt | jobflow -threads=32 -exec wget {}
|
|
|
|
|
|
|
|
|
|
B<3> cat urls.txt | parallel -j32 wget {}
|
|
|
|
|
3$ cat urls.txt | parallel -j32 wget {}
|
|
|
|
|
|
|
|
|
|
B<4> find . -name '*.bmp' | jobflow -threads=8 -exec bmp2jpeg {.}.bmp {.}.jpg
|
|
|
|
|
4$ find . -name '*.bmp' | jobflow -threads=8 -exec bmp2jpeg {.}.bmp {.}.jpg
|
|
|
|
|
|
|
|
|
|
B<4> find . -name '*.bmp' | parallel -j8 bmp2jpeg {.}.bmp {.}.jpg
|
|
|
|
|
4$ find . -name '*.bmp' | parallel -j8 bmp2jpeg {.}.bmp {.}.jpg
|
|
|
|
|
|
|
|
|
|
https://github.com/rofl0r/jobflow
|
|
|
|
|
|
|
|
|
|
@ -836,15 +871,15 @@ is killed.
|
|
|
|
|
|
|
|
|
|
Output to stderr (standard error) is changed if the command fails.
|
|
|
|
|
|
|
|
|
|
Here are the two examples from B<gargs> website.
|
|
|
|
|
=head3 EXAMPLES FROM gargs WEBSITE
|
|
|
|
|
|
|
|
|
|
B<1> seq 12 -1 1 | gargs -p 4 -n 3 "sleep {0}; echo {1} {2}"
|
|
|
|
|
1$ seq 12 -1 1 | gargs -p 4 -n 3 "sleep {0}; echo {1} {2}"
|
|
|
|
|
|
|
|
|
|
B<1> seq 12 -1 1 | parallel -P 4 -n 3 "sleep {1}; echo {2} {3}"
|
|
|
|
|
1$ seq 12 -1 1 | parallel -P 4 -n 3 "sleep {1}; echo {2} {3}"
|
|
|
|
|
|
|
|
|
|
B<2> cat t.txt | gargs --sep "\s+" -p 2 "echo '{0}:{1}-{2}' full-line: \'{}\'"
|
|
|
|
|
2$ cat t.txt | gargs --sep "\s+" -p 2 "echo '{0}:{1}-{2}' full-line: \'{}\'"
|
|
|
|
|
|
|
|
|
|
B<2> cat t.txt | parallel --colsep "\\s+" -P 2 "echo '{1}:{2}-{3}' full-line: \'{}\'"
|
|
|
|
|
2$ cat t.txt | parallel --colsep "\\s+" -P 2 "echo '{1}:{2}-{3}' full-line: \'{}\'"
|
|
|
|
|
|
|
|
|
|
https://github.com/brentp/gargs
|
|
|
|
|
|
|
|
|
|
@ -966,11 +1001,11 @@ instead of ^ as that is closer to bash's ${var%postfix}):
|
|
|
|
|
--rpl '{/:} s:(.*/)?([^/.]+)(\.[^/]+)*$:$2:'
|
|
|
|
|
--rpl '{@(.*?)} /$$1/ and $_=$1;'
|
|
|
|
|
|
|
|
|
|
=head3 EXAMPLES FROM rush's WEBSITE
|
|
|
|
|
|
|
|
|
|
Here are the examples from B<rush>'s website with the equivalent
|
|
|
|
|
command in GNU B<parallel>.
|
|
|
|
|
|
|
|
|
|
=head3 EXAMPLES
|
|
|
|
|
|
|
|
|
|
B<1. Simple run, quoting is not necessary>
|
|
|
|
|
|
|
|
|
|
$ seq 1 3 | rush echo {}
|
|
|
|
|
@ -1518,6 +1553,8 @@ Make Ctrl-C kill all running processes
|
|
|
|
|
|
|
|
|
|
=back
|
|
|
|
|
|
|
|
|
|
=head3 EXAMPLES FROM maps WEBSITE
|
|
|
|
|
|
|
|
|
|
Here are the 5 examples converted to GNU Parallel:
|
|
|
|
|
|
|
|
|
|
1$ ls *.c | map f 'foo $f'
|
|
|
|
|
@ -1556,6 +1593,8 @@ B<loop> cannot run functions:
|
|
|
|
|
export -f myfunc
|
|
|
|
|
loop 'myfunc this fails'
|
|
|
|
|
|
|
|
|
|
=head3 EXAMPLES FROM loop's WEBSITE
|
|
|
|
|
|
|
|
|
|
Some of the examples from https://github.com/Miserlou/Loop/ can be
|
|
|
|
|
emulated with GNU B<parallel>:
|
|
|
|
|
|
|
|
|
|
@ -1760,48 +1799,50 @@ to fit on a single command line.
|
|
|
|
|
|
|
|
|
|
B<paral> has no support for running jobs remotely.
|
|
|
|
|
|
|
|
|
|
=head3 EXAMPLES FROM README.markdown
|
|
|
|
|
|
|
|
|
|
The examples from B<README.markdown> and the corresponding command run
|
|
|
|
|
with GNU B<parallel> (B<--results
|
|
|
|
|
'out_{#}_{=s/[^\sa-z_0-9]//g;s/\s+/_/g=}.log' --eta> is omitted from
|
|
|
|
|
the GNU B<parallel> command):
|
|
|
|
|
|
|
|
|
|
paral "command 1" "command 2 --flag" "command arg1 arg2"
|
|
|
|
|
parallel ::: "command 1" "command 2 --flag" "command arg1 arg2"
|
|
|
|
|
1$ paral "command 1" "command 2 --flag" "command arg1 arg2"
|
|
|
|
|
1$ parallel ::: "command 1" "command 2 --flag" "command arg1 arg2"
|
|
|
|
|
|
|
|
|
|
paral "sleep 1 && echo c1" "sleep 2 && echo c2" \
|
|
|
|
|
"sleep 3 && echo c3" "sleep 4 && echo c4" "sleep 5 && echo c5"
|
|
|
|
|
parallel ::: "sleep 1 && echo c1" "sleep 2 && echo c2" \
|
|
|
|
|
"sleep 3 && echo c3" "sleep 4 && echo c4" "sleep 5 && echo c5"
|
|
|
|
|
# Or shorter:
|
|
|
|
|
parallel "sleep {} && echo c{}" ::: {1..5}
|
|
|
|
|
2$ paral "sleep 1 && echo c1" "sleep 2 && echo c2" \
|
|
|
|
|
"sleep 3 && echo c3" "sleep 4 && echo c4" "sleep 5 && echo c5"
|
|
|
|
|
2$ parallel ::: "sleep 1 && echo c1" "sleep 2 && echo c2" \
|
|
|
|
|
"sleep 3 && echo c3" "sleep 4 && echo c4" "sleep 5 && echo c5"
|
|
|
|
|
# Or shorter:
|
|
|
|
|
parallel "sleep {} && echo c{}" ::: {1..5}
|
|
|
|
|
|
|
|
|
|
paral -n=0 "sleep 5 && echo c5" "sleep 4 && echo c4" \
|
|
|
|
|
"sleep 3 && echo c3" "sleep 2 && echo c2" "sleep 1 && echo c1"
|
|
|
|
|
parallel ::: "sleep 5 && echo c5" "sleep 4 && echo c4" \
|
|
|
|
|
"sleep 3 && echo c3" "sleep 2 && echo c2" "sleep 1 && echo c1"
|
|
|
|
|
# Or shorter:
|
|
|
|
|
parallel -j0 "sleep {} && echo c{}" ::: 5 4 3 2 1
|
|
|
|
|
3$ paral -n=0 "sleep 5 && echo c5" "sleep 4 && echo c4" \
|
|
|
|
|
"sleep 3 && echo c3" "sleep 2 && echo c2" "sleep 1 && echo c1"
|
|
|
|
|
3$ parallel ::: "sleep 5 && echo c5" "sleep 4 && echo c4" \
|
|
|
|
|
"sleep 3 && echo c3" "sleep 2 && echo c2" "sleep 1 && echo c1"
|
|
|
|
|
# Or shorter:
|
|
|
|
|
parallel -j0 "sleep {} && echo c{}" ::: 5 4 3 2 1
|
|
|
|
|
|
|
|
|
|
paral -n=1 "sleep 5 && echo c5" "sleep 4 && echo c4" \
|
|
|
|
|
"sleep 3 && echo c3" "sleep 2 && echo c2" "sleep 1 && echo c1"
|
|
|
|
|
parallel -j1 "sleep {} && echo c{}" ::: 5 4 3 2 1
|
|
|
|
|
4$ paral -n=1 "sleep 5 && echo c5" "sleep 4 && echo c4" \
|
|
|
|
|
"sleep 3 && echo c3" "sleep 2 && echo c2" "sleep 1 && echo c1"
|
|
|
|
|
4$ parallel -j1 "sleep {} && echo c{}" ::: 5 4 3 2 1
|
|
|
|
|
|
|
|
|
|
paral -n=2 "sleep 5 && echo c5" "sleep 4 && echo c4" \
|
|
|
|
|
"sleep 3 && echo c3" "sleep 2 && echo c2" "sleep 1 && echo c1"
|
|
|
|
|
parallel -j2 "sleep {} && echo c{}" ::: 5 4 3 2 1
|
|
|
|
|
5$ paral -n=2 "sleep 5 && echo c5" "sleep 4 && echo c4" \
|
|
|
|
|
"sleep 3 && echo c3" "sleep 2 && echo c2" "sleep 1 && echo c1"
|
|
|
|
|
5$ parallel -j2 "sleep {} && echo c{}" ::: 5 4 3 2 1
|
|
|
|
|
|
|
|
|
|
paral -n=5 "sleep 5 && echo c5" "sleep 4 && echo c4" \
|
|
|
|
|
"sleep 3 && echo c3" "sleep 2 && echo c2" "sleep 1 && echo c1"
|
|
|
|
|
parallel -j5 "sleep {} && echo c{}" ::: 5 4 3 2 1
|
|
|
|
|
6$ paral -n=5 "sleep 5 && echo c5" "sleep 4 && echo c4" \
|
|
|
|
|
"sleep 3 && echo c3" "sleep 2 && echo c2" "sleep 1 && echo c1"
|
|
|
|
|
6$ parallel -j5 "sleep {} && echo c{}" ::: 5 4 3 2 1
|
|
|
|
|
|
|
|
|
|
paral -n=1 "echo a && sleep 0.5 && echo b && sleep 0.5 && \
|
|
|
|
|
echo c && sleep 0.5 && echo d && sleep 0.5 && \
|
|
|
|
|
echo e && sleep 0.5 && echo f && sleep 0.5 && \
|
|
|
|
|
echo g && sleep 0.5 && echo h"
|
|
|
|
|
parallel ::: "echo a && sleep 0.5 && echo b && sleep 0.5 && \
|
|
|
|
|
echo c && sleep 0.5 && echo d && sleep 0.5 && \
|
|
|
|
|
echo e && sleep 0.5 && echo f && sleep 0.5 && \
|
|
|
|
|
echo g && sleep 0.5 && echo h"
|
|
|
|
|
7$ paral -n=1 "echo a && sleep 0.5 && echo b && sleep 0.5 && \
|
|
|
|
|
echo c && sleep 0.5 && echo d && sleep 0.5 && \
|
|
|
|
|
echo e && sleep 0.5 && echo f && sleep 0.5 && \
|
|
|
|
|
echo g && sleep 0.5 && echo h"
|
|
|
|
|
7$ parallel ::: "echo a && sleep 0.5 && echo b && sleep 0.5 && \
|
|
|
|
|
echo c && sleep 0.5 && echo d && sleep 0.5 && \
|
|
|
|
|
echo e && sleep 0.5 && echo f && sleep 0.5 && \
|
|
|
|
|
echo g && sleep 0.5 && echo h"
|
|
|
|
|
|
|
|
|
|
https://github.com/amattn/paral (Last checked: 2019-01)
|
|
|
|
|
|
|
|
|
|
@ -1811,19 +1852,21 @@ https://github.com/amattn/paral (Last checked: 2019-01)
|
|
|
|
|
B<concurr> is built to run jobs in parallel using a client/server
|
|
|
|
|
model.
|
|
|
|
|
|
|
|
|
|
=head3 EXAMPLES FROM README.md
|
|
|
|
|
|
|
|
|
|
The examples from B<README.md>:
|
|
|
|
|
|
|
|
|
|
concurr 'echo job {#} on slot {%}: {}' : arg1 arg2 arg3 arg4
|
|
|
|
|
parallel 'echo job {#} on slot {%}: {}' ::: arg1 arg2 arg3 arg4
|
|
|
|
|
1$ concurr 'echo job {#} on slot {%}: {}' : arg1 arg2 arg3 arg4
|
|
|
|
|
1$ parallel 'echo job {#} on slot {%}: {}' ::: arg1 arg2 arg3 arg4
|
|
|
|
|
|
|
|
|
|
concurr 'echo job {#} on slot {%}: {}' :: file1 file2 file3
|
|
|
|
|
parallel 'echo job {#} on slot {%}: {}' :::: file1 file2 file3
|
|
|
|
|
2$ concurr 'echo job {#} on slot {%}: {}' :: file1 file2 file3
|
|
|
|
|
2$ parallel 'echo job {#} on slot {%}: {}' :::: file1 file2 file3
|
|
|
|
|
|
|
|
|
|
concurr 'echo {}' < input_file
|
|
|
|
|
parallel 'echo {}' < input_file
|
|
|
|
|
3$ concurr 'echo {}' < input_file
|
|
|
|
|
3$ parallel 'echo {}' < input_file
|
|
|
|
|
|
|
|
|
|
cat file | concurr 'echo {}'
|
|
|
|
|
cat file | parallel 'echo {}'
|
|
|
|
|
4$ cat file | concurr 'echo {}'
|
|
|
|
|
4$ cat file | parallel 'echo {}'
|
|
|
|
|
|
|
|
|
|
B<concurr> deals badly empty input files and with output larger than
|
|
|
|
|
64 KB.
|
|
|
|
|
@ -1857,37 +1900,45 @@ https://github.com/spion/npm-parallel (Last checked: 2019-01)
|
|
|
|
|
=head2 DIFFERENCES BETWEEN machma AND GNU Parallel
|
|
|
|
|
|
|
|
|
|
B<machma> runs tasks in parallel. It gives time stamped
|
|
|
|
|
output. It buffers in RAM. The examples from README.md:
|
|
|
|
|
output. It buffers in RAM.
|
|
|
|
|
|
|
|
|
|
# Put shorthand for timestamp in config for the examples
|
|
|
|
|
echo '--rpl '\
|
|
|
|
|
\''{time} $_=::strftime("%Y-%m-%d %H:%M:%S",localtime())'\' \
|
|
|
|
|
> ~/.parallel/machma
|
|
|
|
|
echo '--line-buffer --tagstring "{#} {time} {}"' >> ~/.parallel/machma
|
|
|
|
|
=head3 EXAMPLES FROM README.md
|
|
|
|
|
|
|
|
|
|
find . -iname '*.jpg' |
|
|
|
|
|
machma -- mogrify -resize 1200x1200 -filter Lanczos {}
|
|
|
|
|
find . -iname '*.jpg' |
|
|
|
|
|
parallel --bar -Jmachma mogrify -resize 1200x1200 -filter Lanczos {}
|
|
|
|
|
The examples from README.md:
|
|
|
|
|
|
|
|
|
|
cat /tmp/ips | machma -p 2 -- ping -c 2 -q {}
|
|
|
|
|
cat /tmp/ips | parallel -j2 -Jmachma ping -c 2 -q {}
|
|
|
|
|
1$ # Put shorthand for timestamp in config for the examples
|
|
|
|
|
echo '--rpl '\
|
|
|
|
|
\''{time} $_=::strftime("%Y-%m-%d %H:%M:%S",localtime())'\' \
|
|
|
|
|
> ~/.parallel/machma
|
|
|
|
|
echo '--line-buffer --tagstring "{#} {time} {}"' \
|
|
|
|
|
>> ~/.parallel/machma
|
|
|
|
|
|
|
|
|
|
cat /tmp/ips |
|
|
|
|
|
machma -- sh -c 'ping -c 2 -q $0 > /dev/null && echo alive' {}
|
|
|
|
|
cat /tmp/ips |
|
|
|
|
|
parallel -Jmachma 'ping -c 2 -q {} > /dev/null && echo alive'
|
|
|
|
|
2$ find . -iname '*.jpg' |
|
|
|
|
|
machma -- mogrify -resize 1200x1200 -filter Lanczos {}
|
|
|
|
|
find . -iname '*.jpg' |
|
|
|
|
|
parallel --bar -Jmachma mogrify -resize 1200x1200 \
|
|
|
|
|
-filter Lanczos {}
|
|
|
|
|
|
|
|
|
|
find . -iname '*.jpg' |
|
|
|
|
|
machma --timeout 5s -- mogrify -resize 1200x1200 -filter Lanczos {}
|
|
|
|
|
find . -iname '*.jpg' |
|
|
|
|
|
parallel --timeout 5s --bar mogrify -resize 1200x1200 \
|
|
|
|
|
-filter Lanczos {}
|
|
|
|
|
3$ cat /tmp/ips | machma -p 2 -- ping -c 2 -q {}
|
|
|
|
|
3$ cat /tmp/ips | parallel -j2 -Jmachma ping -c 2 -q {}
|
|
|
|
|
|
|
|
|
|
find . -iname '*.jpg' -print0 |
|
|
|
|
|
machma --null -- mogrify -resize 1200x1200 -filter Lanczos {}
|
|
|
|
|
find . -iname '*.jpg' -print0 |
|
|
|
|
|
parallel --null --bar mogrify -resize 1200x1200 -filter Lanczos {}
|
|
|
|
|
4$ cat /tmp/ips |
|
|
|
|
|
machma -- sh -c 'ping -c 2 -q $0 > /dev/null && echo alive' {}
|
|
|
|
|
4$ cat /tmp/ips |
|
|
|
|
|
parallel -Jmachma 'ping -c 2 -q {} > /dev/null && echo alive'
|
|
|
|
|
|
|
|
|
|
5$ find . -iname '*.jpg' |
|
|
|
|
|
machma --timeout 5s -- mogrify -resize 1200x1200 \
|
|
|
|
|
-filter Lanczos {}
|
|
|
|
|
5$ find . -iname '*.jpg' |
|
|
|
|
|
parallel --timeout 5s --bar mogrify -resize 1200x1200 \
|
|
|
|
|
-filter Lanczos {}
|
|
|
|
|
|
|
|
|
|
6$ find . -iname '*.jpg' -print0 |
|
|
|
|
|
machma --null -- mogrify -resize 1200x1200 -filter Lanczos {}
|
|
|
|
|
6$ find . -iname '*.jpg' -print0 |
|
|
|
|
|
parallel --null --bar mogrify -resize 1200x1200 \
|
|
|
|
|
-filter Lanczos {}
|
|
|
|
|
|
|
|
|
|
https://github.com/fd0/machma (Last checked: 2019-06)
|
|
|
|
|
|
|
|
|
|
@ -1909,6 +1960,8 @@ B<interface> does not buffer output, so output from different jobs mixes.
|
|
|
|
|
The overhead for each target is O(n*n), so with 1000 targets it
|
|
|
|
|
becomes very slow with an overhead in the order of 500ms/target.
|
|
|
|
|
|
|
|
|
|
=head3 EXAMPLES FROM interlace's WEBSITE
|
|
|
|
|
|
|
|
|
|
Using B<prips> most of the examples from
|
|
|
|
|
https://github.com/codingo/Interlace can be run with GNU B<parallel>:
|
|
|
|
|
|
|
|
|
|
@ -2319,17 +2372,18 @@ For full support of {n..m:s} including negative numbers use a dynamic
|
|
|
|
|
replacement string like this:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
PARALLEL=--rpl\ \''{r((-?\d+)?)\.\.((-?\d+)?)((:([^}]*))?)}
|
|
|
|
|
PARALLEL=--rpl\ \''{r((-?\d+)?)\.\.((-?\d+)?)((:([^}]*))?)}
|
|
|
|
|
$a = defined $$2 ? $$2 < 0 ? 1+$#arg+$$2 : $$2 : 1;
|
|
|
|
|
$b = defined $$4 ? $$4 < 0 ? 1+$#arg+$$4 : $$4 : $#arg+1;
|
|
|
|
|
$s = defined $$6 ? $$7 : " ";
|
|
|
|
|
$_ = join $s,@arg[$a..$b]'\'
|
|
|
|
|
export PARALLEL
|
|
|
|
|
export PARALLEL
|
|
|
|
|
|
|
|
|
|
You can then do:
|
|
|
|
|
|
|
|
|
|
head /etc/passwd | parallel --colsep : echo ..={1r..} ..3={1r..3} 4..={1r4..} 2..4={1r2..4} 3..3={1r3..3} ..3:-={1r..3:-} ..3:/={1r..3:/} -1={-1} -5={-5} -6={-6} -3..={1r-3..}
|
|
|
|
|
|
|
|
|
|
head /etc/passwd | parallel --colsep : echo ..={1r..} ..3={1r..3} \
|
|
|
|
|
4..={1r4..} 2..4={1r2..4} 3..3={1r3..3} ..3:-={1r..3:-} \
|
|
|
|
|
..3:/={1r..3:/} -1={-1} -5={-5} -6={-6} -3..={1r-3..}
|
|
|
|
|
|
|
|
|
|
=head3 EXAMPLES FROM rargs MANUAL
|
|
|
|
|
|
|
|
|
|
@ -2370,6 +2424,7 @@ virtual memory will cause the machine to crash.
|
|
|
|
|
|
|
|
|
|
https://github.com/voodooEntity/threader (Last checked: 2020-04)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
=head2 DIFFERENCES BETWEEN runp AND GNU Parallel
|
|
|
|
|
|
|
|
|
|
Summary table (see legend above):
|
|
|
|
|
@ -2496,9 +2551,53 @@ output to stderr (this can be prevented with -q)
|
|
|
|
|
https://github.com/jreisinger/runp (Last checked: 2020-04)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
=head2 Todo
|
|
|
|
|
=head2 DIFFERENCES BETWEEN papply AND GNU Parallel
|
|
|
|
|
|
|
|
|
|
https://pypi.org/project/papply/
|
|
|
|
|
Summary table (see legend above):
|
|
|
|
|
- - - I4 - - -
|
|
|
|
|
M1 - M3 - - M6
|
|
|
|
|
- - O3 - O5 - - N/A N/A O10
|
|
|
|
|
E1 - - E4 - - -
|
|
|
|
|
- - - - - - - - -
|
|
|
|
|
- -
|
|
|
|
|
|
|
|
|
|
B<papply> does not print the output if the command fails:
|
|
|
|
|
|
|
|
|
|
$ papply 'echo %F; false' foo
|
|
|
|
|
"echo foo; false" did not succeed
|
|
|
|
|
|
|
|
|
|
B<papply>'s replacement strings (%F %d %f %n %e %z) can be simulated in GNU
|
|
|
|
|
B<parallel> by putting this in B<~/.parallel/config>:
|
|
|
|
|
|
|
|
|
|
--rpl '%F'
|
|
|
|
|
--rpl '%d $_=Q(::dirname($_));'
|
|
|
|
|
--rpl '%f s:.*/::;'
|
|
|
|
|
--rpl '%n s:.*/::;s:\.[^/.]+$::;'
|
|
|
|
|
--rpl '%e s:.*\.:.:'
|
|
|
|
|
--rpl '%z $_=""'
|
|
|
|
|
|
|
|
|
|
B<papply> buffers in RAM, and uses twice the amount of output. So
|
|
|
|
|
output of 5 GB takes 10 GB RAM.
|
|
|
|
|
|
|
|
|
|
The buffering is very CPU intensive: Buffering a line of 5 GB takes 40
|
|
|
|
|
seconds (compared to 10 seconds with GNU B<parallel>).
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
=head3 Examples as GNU Parallel
|
|
|
|
|
|
|
|
|
|
1$ papply gzip *.txt
|
|
|
|
|
|
|
|
|
|
1$ parallel gzip ::: *.txt
|
|
|
|
|
|
|
|
|
|
2$ papply "convert %F %n.jpg" *.png
|
|
|
|
|
|
|
|
|
|
2$ parallel convert {} {.}.jpg ::: *.png
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
https://pypi.org/project/papply/ (Last checked: 2020-04)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
=head2 Todo
|
|
|
|
|
|
|
|
|
|
https://github.com/JeiKeiLim/simple_distribute_job
|
|
|
|
|
|
|
|
|
|
|
Ole Tange