tange/parallel
1
0
Fork 0
mirror of https://git.savannah.gnu.org/git/parallel.git synced 2024-11-21 21:47:54 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

parallel: Fixed bug #57055: {sshlogin} shorthand for $PARALLEL_SSHLOGIN.

Browse source
This commit is contained in:
Ole Tange 2020-05-08 03:35:51 +02:00
parent e5fd1c0767
commit 6a67fd09d4
9 changed files with 379 additions and 284 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

34
doc/release_new_version
View file

@ -131,7 +131,7 @@ pushd
git diff git diff
# Recheck OBS https://build.opensuse.org/package/show?package=parallel&project=home%3Atange # Recheck OBS https://build.opensuse.org/package/show/home:tange/parallel
YYYYMMDD=`yyyymmdd` YYYYMMDD=`yyyymmdd`
TAG=MyTag TAG=MyTag
@ -211,46 +211,24 @@ from:tange@gnu.org
to:parallel@gnu.org, bug-parallel@gnu.org to:parallel@gnu.org, bug-parallel@gnu.org
stable-bcc: Jesse Alama <jessealama@fastmail.fm> stable-bcc: Jesse Alama <jessealama@fastmail.fm>
Subject: GNU Parallel 20200422 ('10years') released <<[stable]>> Subject: GNU Parallel 20200522 ('Kraftwerk') released <<[stable]>>
GNU Parallel 20200422 ('10years') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/ GNU Parallel 20200522 ('') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/
<<No new functionality was introduced so this is a good candidate for a stable release.>> <<No new functionality was introduced so this is a good candidate for a stable release.>>
This release celebrates GNU Parallel's 10 years as a GNU tool:
> commit ed2dfb1043768154d4e7678e01e10287155fa834
> Author: Ole Tange <ole@tange.dk>
> Date: Thu Apr 22 01:23:00 2010 +0200
>
> Name change: Parallel is now GNU Parallel.
> Basic structure for sshlogin and sshloginfile.
Unfortunately the physical celebrations has been cancelled due to COVID19.
Quote of the month: Quote of the month:
I wish more command line software had example pages as robust as GNU Parallel <<>>
-- Lucidbeaming @lucidbeaming
New in this release: New in this release:
* parsort makes GNU sort run faster for files with more than 1M lines.
* Bug fixes and man page updates. * Bug fixes and man page updates.
News about GNU Parallel: News about GNU Parallel:
* Several clusters have updated their documentation of GNU Parallel: https://www.grid5000.fr/w/GNU_Parallel https://research-it.berkeley.edu/virtual-training-running-jobs-parallel-savio http://www.hpc.lsu.edu/training/archive/tutorials.php *
* add_pho uses GNU Parallel: https://github.com/SeedlingsBabylab/add_pho
* GNU parallel使用笔记https://zhuanlan.zhihu.com/p/37340011
* Virtual Workshop: Running Jobs in Parallel on Savio https://ais.berkeley.edu/events/virtual-workshop-running-jobs-parallel-savio/2020-04-20
* Free Concurrency with GNU Parallel https://phili.pe/posts/free-concurrency-with-gnu-parallel/
Get the book: GNU Parallel 2018 http://www.lulu.com/shop/ole-tange/gnu-parallel-2018/paperback/product-23558902.html Get the book: GNU Parallel 2018 http://www.lulu.com/shop/ole-tange/gnu-parallel-2018/paperback/product-23558902.html

2
src/niceload
View file

@ -23,7 +23,7 @@
use strict; use strict;
use Getopt::Long; use Getopt::Long;
$Global::progname="niceload"; $Global::progname="niceload";
$Global::version = 20200422; $Global::version = 20200423;
Getopt::Long::Configure("bundling","require_order"); Getopt::Long::Configure("bundling","require_order");
get_options_from_array(\@ARGV) || die_usage(); get_options_from_array(\@ARGV) || die_usage();
if($opt::version) { if($opt::version) {

38
src/parallel
View file

@ -2029,8 +2029,8 @@ sub parse_options(@) {
# If you want GNU Parallel to be maintained in the future keep # If you want GNU Parallel to be maintained in the future keep
# this line. # this line.
citation_notice(); citation_notice();
# Seriously: YOU will be harming free software by removing the # Seriously: _YOU_ will be harming free software by removing the
# notice. You make it harder to justify spending time developing # notice. _YOU_ make it harder to justify spending time developing
# it. If you *do* remove the line, please email # it. If you *do* remove the line, please email
# hallofshame@tange.dk if you want to avoid being put in a hall of # hallofshame@tange.dk if you want to avoid being put in a hall of
# shame. # shame.
@ -2150,7 +2150,7 @@ sub check_invalid_option_combinations() {
sub init_globals() { sub init_globals() {
# Defaults: # Defaults:
$Global::version = 20200422; $Global::version = 20200423;
$Global::progname = 'parallel'; $Global::progname = 'parallel';
$::name = "GNU Parallel"; $::name = "GNU Parallel";
$Global::infinity = 2**31; $Global::infinity = 2**31;
@ -2212,6 +2212,10 @@ sub init_globals() {
'{,([^}]+?)}' => 's/^($$1)/lc($1)/e;', '{,([^}]+?)}' => 's/^($$1)/lc($1)/e;',
# Bash ${a,,A} # Bash ${a,,A}
'{,,([^}]+?)}' => 's/($$1)/lc($1)/eg;', '{,,([^}]+?)}' => 's/($$1)/lc($1)/eg;',
# {host} = ssh host
'{host}' => '$_="\${PARALLEL_SSHHOST}";uq()',
# {sshlogin} = sshlogin
'{sshlogin}' => '$_="\${PARALLEL_SSHLOGIN}";uq()',
); );
# Modifiable copy of %Global::replace # Modifiable copy of %Global::replace
%Global::rpl = %Global::replace; %Global::rpl = %Global::replace;
@ -2367,7 +2371,7 @@ sub parse_replacement_string_options() {
sub parse_semaphore() { sub parse_semaphore() {
# Semaphore defaults # Semaphore defaults
# Must be done before computing number of processes and max_line_length # Must be done before computing number of processes and max_line_length
# because when running as a semaphore GNU Parallel does not read args # because when running as a semaphore GNU Parallel does not read args
# Uses: # Uses:
# $opt::semaphore # $opt::semaphore
# $Global::semaphore # $Global::semaphore
@ -3196,7 +3200,7 @@ sub __RUNNING_THE_JOBS_AND_PRINTING_PROGRESS__() {}
# $Global::total_started = total number of jobs started # $Global::total_started = total number of jobs started
# $Global::joblog = filehandle of joblog # $Global::joblog = filehandle of joblog
# $Global::debug = Is debugging on? # $Global::debug = Is debugging on?
# $Global::exitstatus = status code of GNU Parallel # $Global::exitstatus = status code of GNU Parallel
# $Global::quoting = quote the command to run # $Global::quoting = quote the command to run
sub init_run_jobs() { sub init_run_jobs() {
@ -4152,14 +4156,14 @@ sub run_gnu_parallel() {
} }
sub _run_gnu_parallel() { sub _run_gnu_parallel() {
# Run GNU Parallel # Run GNU Parallel
# This should ideally just fork an internal copy # This should ideally just fork an internal copy
# and not start it through a shell # and not start it through a shell
# Input: # Input:
# $stdin = data to provide on stdin for GNU Parallel # $stdin = data to provide on stdin for GNU Parallel
# @args = command line arguments # @args = command line arguments
# Returns: # Returns:
# $exitstatus = exitcode of GNU Parallel run # $exitstatus = exitcode of GNU Parallel run
# \@stdout = standard output # \@stdout = standard output
# \@stderr = standard error # \@stderr = standard error
my ($stdin,@args) = @_; my ($stdin,@args) = @_;
@ -4543,7 +4547,7 @@ sub onall($@) {
); );
::debug("init", "| $0 $options\n"); ::debug("init", "| $0 $options\n");
open(my $parallel_fh, "|-", "$0 -0 --will-cite -j0 $options") || open(my $parallel_fh, "|-", "$0 -0 --will-cite -j0 $options") ||
::die_bug("This does not run GNU Parallel: $0 $options"); ::die_bug("This does not run GNU Parallel: $0 $options");
my @joblogs; my @joblogs;
for my $host (sort keys %Global::host) { for my $host (sort keys %Global::host) {
my $sshlogin = $Global::host{$host}; my $sshlogin = $Global::host{$host};
@ -4607,14 +4611,14 @@ sub sigpipe() {
sub signal_children() { sub signal_children() {
# Send signal to all children process groups # Send signal to all children process groups
# and GNU Parallel itself # and GNU Parallel itself
# Uses: # Uses:
# %SIG # %SIG
# Returns: N/A # Returns: N/A
my $signal = shift; my $signal = shift;
debug("run", "Sending $signal "); debug("run", "Sending $signal ");
kill $signal, map { -$_ } keys %Global::running; kill $signal, map { -$_ } keys %Global::running;
# Use default signal handler for GNU Parallel itself # Use default signal handler for GNU Parallel itself
$SIG{$signal} = undef; $SIG{$signal} = undef;
kill $signal, $$; kill $signal, $$;
} }
@ -4925,7 +4929,7 @@ sub citation_notice() {
" DOI https://doi.org/10.5281/zenodo.1146014", " DOI https://doi.org/10.5281/zenodo.1146014",
"", "",
# Before changing this line, please read # Before changing this line, please read
# https://www.gnu.org/software/parallel/parallel_design.html#Citation-notice # https://www.gnu.org/software/parallel/parallel_design.html#Citation-notice and
# https://git.savannah.gnu.org/cgit/parallel.git/tree/doc/citation-notice-faq.txt # https://git.savannah.gnu.org/cgit/parallel.git/tree/doc/citation-notice-faq.txt
"This helps funding further development; AND IT WON'T COST YOU A CENT.", "This helps funding further development; AND IT WON'T COST YOU A CENT.",
"If you pay 10000 EUR you should feel free to use GNU Parallel without citing.", "If you pay 10000 EUR you should feel free to use GNU Parallel without citing.",
@ -4937,7 +4941,7 @@ sub citation_notice() {
"" ""
); );
mkdir $Global::config_dir; mkdir $Global::config_dir;
# Number of times the user has run GNU Parallel without showing # Number of times the user has run GNU Parallel without showing
# willingness to cite # willingness to cite
my $runs = 0; my $runs = 0;
if(open (my $fh, "<", $Global::config_dir. if(open (my $fh, "<", $Global::config_dir.
@ -5049,7 +5053,7 @@ sub citation() {
"(Feel free to use \\nocite{tange_ole_2018_1146014})", "(Feel free to use \\nocite{tange_ole_2018_1146014})",
"", "",
# Before changing this line, please read # Before changing this line, please read
# https://www.gnu.org/software/parallel/parallel_design.html#Citation-notice # https://www.gnu.org/software/parallel/parallel_design.html#Citation-notice and
# https://git.savannah.gnu.org/cgit/parallel.git/tree/doc/citation-notice-faq.txt # https://git.savannah.gnu.org/cgit/parallel.git/tree/doc/citation-notice-faq.txt
"This helps funding further development; AND IT WON'T COST YOU A CENT.", "This helps funding further development; AND IT WON'T COST YOU A CENT.",
"If you pay 10000 EUR you should feel free to use GNU Parallel without citing.", "If you pay 10000 EUR you should feel free to use GNU Parallel without citing.",
@ -5501,7 +5505,7 @@ sub seconds_to_time_units() {
# N/A # N/A
if(not $disk_full_fh) { if(not $disk_full_fh) {
$disk_full_fh = ::tmpfile(SUFFIX => ".df"); $disk_full_fh = ::tmpfile(SUFFIX => ".df");
$b8193 = "x"x8193; $b8193 = "b"x8193;
} }
# Linux does not discover if a disk is full if writing <= 8192 # Linux does not discover if a disk is full if writing <= 8192
# Tested on: # Tested on:
@ -8973,7 +8977,7 @@ sub sshlogin_wrap($) {
} }
# Duplicate vars as BASH functions to include post-shellshock functions (v1+v2) # Duplicate vars as BASH functions to include post-shellshock functions (v1+v2)
# So --env myfunc should look for BASH_FUNC_myfunc() and BASH_FUNC_myfunc%% # So --env myfunc should look for BASH_FUNC_myfunc() and BASH_FUNC_myfunc%%
push(@vars, "PARALLEL_PID", "PARALLEL_SEQ", "PARALLEL_SSHLOGIN", push(@vars, "PARALLEL_PID", "PARALLEL_SEQ", "PARALLEL_SSHLOGIN", "PARALLEL_SSHHOST",
map { ("BASH_FUNC_$_()", "BASH_FUNC_$_%%") } @vars); map { ("BASH_FUNC_$_()", "BASH_FUNC_$_%%") } @vars);
# Keep only defined variables # Keep only defined variables
return grep { defined($ENV{$_}) } @vars; return grep { defined($ENV{$_}) } @vars;
@ -9033,6 +9037,7 @@ sub sshlogin_wrap($) {
my $quoted_remote_command; my $quoted_remote_command;
$ENV{'PARALLEL_SEQ'} = $self->seq(); $ENV{'PARALLEL_SEQ'} = $self->seq();
$ENV{'PARALLEL_SSHLOGIN'} = $sshlogin->string(); $ENV{'PARALLEL_SSHLOGIN'} = $sshlogin->string();
$ENV{'PARALLEL_SSHHOST'} = $sshlogin->serverlogin();
$ENV{'PARALLEL_PID'} = $$; $ENV{'PARALLEL_PID'} = $$;
if($serverlogin eq ":") { if($serverlogin eq ":") {
if($opt::workdir) { if($opt::workdir) {
@ -9792,7 +9797,6 @@ sub print($) {
$opt::ungroup and return; $opt::ungroup and return;
} }
} }
# Check for disk full # Check for disk full
::exit_if_disk_full(); ::exit_if_disk_full();
} }

13
src/parallel.pod
View file

@ -1546,6 +1546,11 @@ choose k. k is the number of input sources and n is the number of
arguments in an input source. The content of the input sources must arguments in an input source. The content of the input sources must
be the same and the arguments must be unique. be the same and the arguments must be unique.
For remote jobs these are also activated:
{sshlogin} sshlogin
{host} hostname of sshlogin
The following dynamic replacement strings are also activated. They are The following dynamic replacement strings are also activated. They are
inspired by bash's parameter expansion: inspired by bash's parameter expansion:
@ -5134,16 +5139,16 @@ http://stackoverflow.com/help/mcve).
It should be a complete example that others can run that shows the It should be a complete example that others can run that shows the
problem including all files needed to run the example. This should problem including all files needed to run the example. This should
preferably be small and simple, so try to remove as many options as preferably be small and simple, so try to remove as many options as
possible. A combination of B<yes>, B<seq>, B<cat>, B<echo>, and possible. A combination of B<yes>, B<seq>, B<cat>, B<echo>, B<wc>, and
B<sleep> can reproduce most errors. If your example requires large B<sleep> can reproduce most errors. If your example requires large
files, see if you can make them by something like B<seq 100000000> > files, see if you can make them with something like B<seq 100000000> >
B<bigfile> or B<yes | head -n 1000000000> > B<file>. B<bigfile> or B<yes | head -n 1000000000> > B<file>.
If your example requires remote execution, see if you can use If your example requires remote execution, see if you can use
B<localhost> - maybe using another login. B<localhost> - maybe using another login.
If you have access to a different system, test if the MCVE shows the If you have access to a different system (maybe a VirtualBox on your
problem on that system. own machine), test if the MCVE shows the problem on that system.
=item * =item *

563
src/parallel_alternatives.pod
View file

@ -281,46 +281,47 @@ start. GNU B<parallel> only requires one step.
Here are the examples from B<ppss>'s manual page with the equivalent Here are the examples from B<ppss>'s manual page with the equivalent
using GNU B<parallel>: using GNU B<parallel>:
B<1> ./ppss.sh standalone -d /path/to/files -c 'gzip ' 1$ ./ppss.sh standalone -d /path/to/files -c 'gzip '
B<1> find /path/to/files -type f | parallel gzip 1$ find /path/to/files -type f | parallel gzip
B<2> ./ppss.sh standalone -d /path/to/files -c 'cp "$ITEM" /destination/dir ' 2$ ./ppss.sh standalone -d /path/to/files -c 'cp "$ITEM" /destination/dir '
B<2> find /path/to/files -type f | parallel cp {} /destination/dir 2$ find /path/to/files -type f | parallel cp {} /destination/dir
B<3> ./ppss.sh standalone -f list-of-urls.txt -c 'wget -q ' 3$ ./ppss.sh standalone -f list-of-urls.txt -c 'wget -q '
B<3> parallel -a list-of-urls.txt wget -q 3$ parallel -a list-of-urls.txt wget -q
B<4> ./ppss.sh standalone -f list-of-urls.txt -c 'wget -q "$ITEM"' 4$ ./ppss.sh standalone -f list-of-urls.txt -c 'wget -q "$ITEM"'
B<4> parallel -a list-of-urls.txt wget -q {} 4$ parallel -a list-of-urls.txt wget -q {}
B<5> ./ppss config -C config.cfg -c 'encode.sh ' -d /source/dir -m 5$ ./ppss config -C config.cfg -c 'encode.sh ' -d /source/dir \
192.168.1.100 -u ppss -k ppss-key.key -S ./encode.sh -n nodes.txt -o -m 192.168.1.100 -u ppss -k ppss-key.key -S ./encode.sh \
/some/output/dir --upload --download ; ./ppss deploy -C config.cfg ; -n nodes.txt -o /some/output/dir --upload --download;
./ppss start -C config ./ppss deploy -C config.cfg
./ppss start -C config
B<5> # parallel does not use configs. If you want a different username put it in nodes.txt: user@hostname 5$ # parallel does not use configs. If you want a different username put it in nodes.txt: user@hostname
find source/dir -type f |
parallel --sshloginfile nodes.txt --trc {.}.mp3 lame -a {} -o {.}.mp3 --preset standard --quiet
B<5> find source/dir -type f | parallel --sshloginfile nodes.txt --trc {.}.mp3 lame -a {} -o {.}.mp3 --preset standard --quiet 6$ ./ppss stop -C config.cfg
B<6> ./ppss stop -C config.cfg 6$ killall -TERM parallel
B<6> killall -TERM parallel 7$ ./ppss pause -C config.cfg
B<7> ./ppss pause -C config.cfg 7$ Press: CTRL-Z or killall -SIGTSTP parallel
B<7> Press: CTRL-Z or killall -SIGTSTP parallel 8$ ./ppss continue -C config.cfg
B<8> ./ppss continue -C config.cfg 8$ Enter: fg or killall -SIGCONT parallel
B<8> Enter: fg or killall -SIGCONT parallel 9$ ./ppss.sh status -C config.cfg
B<9> ./ppss.sh status -C config.cfg 9$ killall -SIGUSR2 parallel
B<9> killall -SIGUSR2 parallel
https://github.com/louwrentius/PPSS https://github.com/louwrentius/PPSS
@ -342,55 +343,54 @@ B<pexec> is also a tool for running jobs in parallel.
Here are the examples from B<pexec>'s info page with the equivalent Here are the examples from B<pexec>'s info page with the equivalent
using GNU B<parallel>: using GNU B<parallel>:
B<1> pexec -o sqrt-%s.dat -p "$(seq 10)" -e NUM -n 4 -c -- \ 1$ pexec -o sqrt-%s.dat -p "$(seq 10)" -e NUM -n 4 -c -- \
'echo "scale=10000;sqrt($NUM)" | bc' 'echo "scale=10000;sqrt($NUM)" | bc'
B<1> seq 10 | parallel -j4 'echo "scale=10000;sqrt({})" | bc > sqrt-{}.dat' 1$ seq 10 | parallel -j4 'echo "scale=10000;sqrt({})" | \
bc > sqrt-{}.dat'
B<2> pexec -p "$(ls myfiles*.ext)" -i %s -o %s.sort -- sort 2$ pexec -p "$(ls myfiles*.ext)" -i %s -o %s.sort -- sort
B<2> ls myfiles*.ext | parallel sort {} ">{}.sort" 2$ ls myfiles*.ext | parallel sort {} ">{}.sort"
B<3> pexec -f image.list -n auto -e B -u star.log -c -- \ 3$ pexec -f image.list -n auto -e B -u star.log -c -- \
'fistar $B.fits -f 100 -F id,x,y,flux -o $B.star' 'fistar $B.fits -f 100 -F id,x,y,flux -o $B.star'
B<3> parallel -a image.list \ 3$ parallel -a image.list \
'fistar {}.fits -f 100 -F id,x,y,flux -o {}.star' 2>star.log 'fistar {}.fits -f 100 -F id,x,y,flux -o {}.star' 2>star.log
B<4> pexec -r *.png -e IMG -c -o - -- \ 4$ pexec -r *.png -e IMG -c -o - -- \
'convert $IMG ${IMG%.png}.jpeg ; "echo $IMG: done"' 'convert $IMG ${IMG%.png}.jpeg ; "echo $IMG: done"'
B<4> ls *.png | parallel 'convert {} {.}.jpeg; echo {}: done' 4$ ls *.png | parallel 'convert {} {.}.jpeg; echo {}: done'
B<5> pexec -r *.png -i %s -o %s.jpg -c 'pngtopnm | pnmtojpeg' 5$ pexec -r *.png -i %s -o %s.jpg -c 'pngtopnm | pnmtojpeg'
B<5> ls *.png | parallel 'pngtopnm < {} | pnmtojpeg > {}.jpg' 5$ ls *.png | parallel 'pngtopnm < {} | pnmtojpeg > {}.jpg'
B<6> for p in *.png ; do echo ${p%.png} ; done | \ 6$ for p in *.png ; do echo ${p%.png} ; done | \
pexec -f - -i %s.png -o %s.jpg -c 'pngtopnm | pnmtojpeg' pexec -f - -i %s.png -o %s.jpg -c 'pngtopnm | pnmtojpeg'
B<6> ls *.png | parallel 'pngtopnm < {} | pnmtojpeg > {.}.jpg' 6$ ls *.png | parallel 'pngtopnm < {} | pnmtojpeg > {.}.jpg'
B<7> LIST=$(for p in *.png ; do echo ${p%.png} ; done) 7$ LIST=$(for p in *.png ; do echo ${p%.png} ; done)
pexec -r $LIST -i %s.png -o %s.jpg -c 'pngtopnm | pnmtojpeg' pexec -r $LIST -i %s.png -o %s.jpg -c 'pngtopnm | pnmtojpeg'
B<7> ls *.png | parallel 'pngtopnm < {} | pnmtojpeg > {.}.jpg' 7$ ls *.png | parallel 'pngtopnm < {} | pnmtojpeg > {.}.jpg'
B<8> pexec -n 8 -r *.jpg -y unix -e IMG -c \ 8$ pexec -n 8 -r *.jpg -y unix -e IMG -c \
'pexec -j -m blockread -d $IMG | \ 'pexec -j -m blockread -d $IMG | \
jpegtopnm | pnmscale 0.5 | pnmtojpeg | \ jpegtopnm | pnmscale 0.5 | pnmtojpeg | \
pexec -j -m blockwrite -s th_$IMG' pexec -j -m blockwrite -s th_$IMG'
B<8> Combining GNU B<parallel> and GNU B<sem>. 8$ # Combining GNU B<parallel> and GNU B<sem>.
ls *jpg | parallel -j8 'sem --id blockread cat {} | jpegtopnm |' \
'pnmscale 0.5 | pnmtojpeg | sem --id blockwrite cat > th_{}'
B<8> ls *jpg | parallel -j8 'sem --id blockread cat {} | jpegtopnm |' \ # If reading and writing is done to the same disk, this may be
'pnmscale 0.5 | pnmtojpeg | sem --id blockwrite cat > th_{}' # faster as only one process will be either reading or writing:
ls *jpg | parallel -j8 'sem --id diskio cat {} | jpegtopnm |' \
B<8> If reading and writing is done to the same disk, this may be 'pnmscale 0.5 | pnmtojpeg | sem --id diskio cat > th_{}'
faster as only one process will be either reading or writing:
B<8> ls *jpg | parallel -j8 'sem --id diskio cat {} | jpegtopnm |' \
'pnmscale 0.5 | pnmtojpeg | sem --id diskio cat > th_{}'
https://www.gnu.org/software/pexec/ https://www.gnu.org/software/pexec/
@ -403,38 +403,40 @@ running jobs on your local computer.
B<xjobs> deals badly with special characters just like B<xargs>. See B<xjobs> deals badly with special characters just like B<xargs>. See
the section B<DIFFERENCES BETWEEN xargs AND GNU Parallel>. the section B<DIFFERENCES BETWEEN xargs AND GNU Parallel>.
=head3 EXAMPLES FROM xjobs MANUAL
Here are the examples from B<xjobs>'s man page with the equivalent Here are the examples from B<xjobs>'s man page with the equivalent
using GNU B<parallel>: using GNU B<parallel>:
B<1> ls -1 *.zip | xjobs unzip 1$ ls -1 *.zip | xjobs unzip
B<1> ls *.zip | parallel unzip 1$ ls *.zip | parallel unzip
B<2> ls -1 *.zip | xjobs -n unzip 2$ ls -1 *.zip | xjobs -n unzip
B<2> ls *.zip | parallel unzip >/dev/null 2$ ls *.zip | parallel unzip >/dev/null
B<3> find . -name '*.bak' | xjobs gzip 3$ find . -name '*.bak' | xjobs gzip
B<3> find . -name '*.bak' | parallel gzip 3$ find . -name '*.bak' | parallel gzip
B<4> ls -1 *.jar | sed 's/\(.*\)/\1 > \1.idx/' | xjobs jar tf 4$ ls -1 *.jar | sed 's/\(.*\)/\1 > \1.idx/' | xjobs jar tf
B<4> ls *.jar | parallel jar tf {} '>' {}.idx 4$ ls *.jar | parallel jar tf {} '>' {}.idx
B<5> xjobs -s script 5$ xjobs -s script
B<5> cat script | parallel 5$ cat script | parallel
B<6> mkfifo /var/run/my_named_pipe; 6$ mkfifo /var/run/my_named_pipe;
xjobs -s /var/run/my_named_pipe & xjobs -s /var/run/my_named_pipe &
echo unzip 1.zip >> /var/run/my_named_pipe; echo unzip 1.zip >> /var/run/my_named_pipe;
echo tar cf /backup/myhome.tar /home/me >> /var/run/my_named_pipe echo tar cf /backup/myhome.tar /home/me >> /var/run/my_named_pipe
B<6> mkfifo /var/run/my_named_pipe; 6$ mkfifo /var/run/my_named_pipe;
cat /var/run/my_named_pipe | parallel & cat /var/run/my_named_pipe | parallel &
echo unzip 1.zip >> /var/run/my_named_pipe; echo unzip 1.zip >> /var/run/my_named_pipe;
echo tar cf /backup/myhome.tar /home/me >> /var/run/my_named_pipe echo tar cf /backup/myhome.tar /home/me >> /var/run/my_named_pipe
http://www.maier-komor.de/xjobs.html (Last checked: 2019-01) http://www.maier-komor.de/xjobs.html (Last checked: 2019-01)
@ -452,11 +454,14 @@ B<prll> generates a lot of status information on stderr (standard
error) which makes it harder to use the stderr (standard error) output error) which makes it harder to use the stderr (standard error) output
of the job directly as input for another program. of the job directly as input for another program.
=head3 EXAMPLES FROM prll's MANUAL
Here is the example from B<prll>'s man page with the equivalent Here is the example from B<prll>'s man page with the equivalent
using GNU B<parallel>: using GNU B<parallel>:
prll -s 'mogrify -flip $1' *.jpg 1$ prll -s 'mogrify -flip $1' *.jpg
parallel mogrify -flip ::: *.jpg
1$ parallel mogrify -flip ::: *.jpg
https://github.com/exzombie/prll (Last checked: 2019-01) https://github.com/exzombie/prll (Last checked: 2019-01)
@ -477,13 +482,15 @@ semicomplete.com/blog/geekery/distributed-xargs.html (Last checked: 2019-01)
middleman(mdm) is also a tool for running jobs in parallel. middleman(mdm) is also a tool for running jobs in parallel.
=head3 EXAMPLES FROM middleman's WEBSITE
Here are the shellscripts of Here are the shellscripts of
https://web.archive.org/web/20110728064735/http://mdm. https://web.archive.org/web/20110728064735/http://mdm.
berlios.de/usage.html ported to GNU B<parallel>: berlios.de/usage.html ported to GNU B<parallel>:
seq 19 | parallel buffon -o - | sort -n > result 1$ seq 19 | parallel buffon -o - | sort -n > result
cat files | parallel cmd cat files | parallel cmd
find dir -execdir sem cmd {} \; find dir -execdir sem cmd {} \;
https://github.com/cklin/mdm (Last checked: 2019-01) https://github.com/cklin/mdm (Last checked: 2019-01)
@ -492,56 +499,58 @@ https://github.com/cklin/mdm (Last checked: 2019-01)
B<xapply> can run jobs in parallel on the local computer. B<xapply> can run jobs in parallel on the local computer.
=head3 EXAMPLES FROM xapply's MANUAL
Here are the examples from B<xapply>'s man page with the equivalent Here are the examples from B<xapply>'s man page with the equivalent
using GNU B<parallel>: using GNU B<parallel>:
B<1> xapply '(cd %1 && make all)' */ 1$ xapply '(cd %1 && make all)' */
B<1> parallel 'cd {} && make all' ::: */ 1$ parallel 'cd {} && make all' ::: */
B<2> xapply -f 'diff %1 ../version5/%1' manifest | more 2$ xapply -f 'diff %1 ../version5/%1' manifest | more
B<2> parallel diff {} ../version5/{} < manifest | more 2$ parallel diff {} ../version5/{} < manifest | more
B<3> xapply -p/dev/null -f 'diff %1 %2' manifest1 checklist1 3$ xapply -p/dev/null -f 'diff %1 %2' manifest1 checklist1
B<3> parallel --link diff {1} {2} :::: manifest1 checklist1 3$ parallel --link diff {1} {2} :::: manifest1 checklist1
B<4> xapply 'indent' *.c 4$ xapply 'indent' *.c
B<4> parallel indent ::: *.c 4$ parallel indent ::: *.c
B<5> find ~ksb/bin -type f ! -perm -111 -print | xapply -f -v 'chmod a+x' - 5$ find ~ksb/bin -type f ! -perm -111 -print | xapply -f -v 'chmod a+x' -
B<5> find ~ksb/bin -type f ! -perm -111 -print | parallel -v chmod a+x 5$ find ~ksb/bin -type f ! -perm -111 -print | parallel -v chmod a+x
B<6> find */ -... | fmt 960 1024 | xapply -f -i /dev/tty 'vi' - 6$ find */ -... | fmt 960 1024 | xapply -f -i /dev/tty 'vi' -
B<6> sh <(find */ -... | parallel -s 1024 echo vi) 6$ sh <(find */ -... | parallel -s 1024 echo vi)
B<6> find */ -... | parallel -s 1024 -Xuj1 vi 6$ find */ -... | parallel -s 1024 -Xuj1 vi
B<7> find ... | xapply -f -5 -i /dev/tty 'vi' - - - - - 7$ find ... | xapply -f -5 -i /dev/tty 'vi' - - - - -
B<7> sh <(find ... |parallel -n5 echo vi) 7$ sh <(find ... |parallel -n5 echo vi)
B<7> find ... |parallel -n5 -uj1 vi 7$ find ... |parallel -n5 -uj1 vi
B<8> xapply -fn "" /etc/passwd 8$ xapply -fn "" /etc/passwd
B<8> parallel -k echo < /etc/passwd 8$ parallel -k echo < /etc/passwd
B<9> tr ':' '\012' < /etc/passwd | xapply -7 -nf 'chown %1 %6' - - - - - - - 9$ tr ':' '\012' < /etc/passwd | xapply -7 -nf 'chown %1 %6' - - - - - - -
B<9> tr ':' '\012' < /etc/passwd | parallel -N7 chown {1} {6} 9$ tr ':' '\012' < /etc/passwd | parallel -N7 chown {1} {6}
B<10> xapply '[ -d %1/RCS ] || echo %1' */ 10$ xapply '[ -d %1/RCS ] || echo %1' */
B<10> parallel '[ -d {}/RCS ] || echo {}' ::: */ 10$ parallel '[ -d {}/RCS ] || echo {}' ::: */
B<11> xapply -f '[ -f %1 ] && echo %1' List | ... 11$ xapply -f '[ -f %1 ] && echo %1' List | ...
B<11> parallel '[ -f {} ] && echo {}' < List | ... 11$ parallel '[ -f {} ] && echo {}' < List | ...
https://web.archive.org/web/20160702211113/ https://web.archive.org/web/20160702211113/
http://carrera.databits.net/~ksb/msrc/local/bin/xapply/xapply.html http://carrera.databits.net/~ksb/msrc/local/bin/xapply/xapply.html
@ -554,30 +563,32 @@ very much like GNU B<parallel>. B<apply> does not run jobs in
parallel. B<apply> does not use an argument separator (like B<:::>); parallel. B<apply> does not use an argument separator (like B<:::>);
instead the template must be the first argument. instead the template must be the first argument.
=head3 EXAMPLES FROM IBM's KNOWLEDGE CENTER
Here are the examples from IBM's Knowledge Center and the Here are the examples from IBM's Knowledge Center and the
corresponding command using GNU B<parallel>: corresponding command using GNU B<parallel>:
1. To obtain results similar to those of the B<ls> command, enter: =head4 To obtain results similar to those of the B<ls> command, enter:
apply echo * 1$ apply echo *
parallel echo ::: * 1$ parallel echo ::: *
2. To compare the file named B<a1> to the file named B<b1>, and the =head4 To compare the file named a1 to the file named b1, and
file named B<a2> to the file named B<b2>, enter: the file named a2 to the file named b2, enter:
apply -2 cmp a1 b1 a2 b2 2$ apply -2 cmp a1 b1 a2 b2
parallel -N2 cmp ::: a1 b1 a2 b2 2$ parallel -N2 cmp ::: a1 b1 a2 b2
3. To run the B<who> command five times, enter: =head4 To run the B<who> command five times, enter:
apply -0 who 1 2 3 4 5 3$ apply -0 who 1 2 3 4 5
parallel -N0 who ::: 1 2 3 4 5 3$ parallel -N0 who ::: 1 2 3 4 5
4. To link all files in the current directory to the directory =head4 To link all files in the current directory to the directory
B</usr/joe>, enter: /usr/joe, enter:
apply 'ln %1 /usr/joe' * 4$ apply 'ln %1 /usr/joe' *
parallel ln {} /usr/joe ::: * 4$ parallel ln {} /usr/joe ::: *
https://www-01.ibm.com/support/knowledgecenter/ https://www-01.ibm.com/support/knowledgecenter/
ssw_aix_71/com.ibm.aix.cmds1/apply.htm (Last checked: 2019-01) ssw_aix_71/com.ibm.aix.cmds1/apply.htm (Last checked: 2019-01)
@ -593,46 +604,60 @@ output. This means you will have to write a wrapper for most programs.
B<paexec> has a job dependency facility so a job can depend on another B<paexec> has a job dependency facility so a job can depend on another
job to be executed successfully. Sort of a poor-man's B<make>. job to be executed successfully. Sort of a poor-man's B<make>.
=head3 EXAMPLES FROM paexec's EXAMPLE CATALOG
Here are the examples from B<paexec>'s example catalog with the equivalent Here are the examples from B<paexec>'s example catalog with the equivalent
using GNU B<parallel>: using GNU B<parallel>:
=over 1 =head4 1_div_X_run
=item 1_div_X_run: 1$ ../../paexec -s -l -c "`pwd`/1_div_X_cmd" -n +1 <<EOF [...]
../../paexec -s -l -c "`pwd`/1_div_X_cmd" -n +1 <<EOF [...] 1$ parallel echo {} '|' `pwd`/1_div_X_cmd <<EOF [...]
parallel echo {} '|' `pwd`/1_div_X_cmd <<EOF [...]
=item all_substr_run: =head4 all_substr_run
../../paexec -lp -c "`pwd`/all_substr_cmd" -n +3 <<EOF [...] 2$ ../../paexec -lp -c "`pwd`/all_substr_cmd" -n +3 <<EOF [...]
parallel echo {} '|' `pwd`/all_substr_cmd <<EOF [...]
=item cc_wrapper_run: 2$ parallel echo {} '|' `pwd`/all_substr_cmd <<EOF [...]
../../paexec -c "env CC=gcc CFLAGS=-O2 `pwd`/cc_wrapper_cmd" \ =head4 cc_wrapper_run
3$ ../../paexec -c "env CC=gcc CFLAGS=-O2 `pwd`/cc_wrapper_cmd" \
-n 'host1 host2' \ -n 'host1 host2' \
-t '/usr/bin/ssh -x' <<EOF [...] -t '/usr/bin/ssh -x' <<EOF [...]
parallel echo {} '|' "env CC=gcc CFLAGS=-O2 `pwd`/cc_wrapper_cmd" \
-S host1,host2 <<EOF [...] 3$ parallel echo {} '|' "env CC=gcc CFLAGS=-O2 `pwd`/cc_wrapper_cmd" \
# This is not exactly the same, but avoids the wrapper
parallel gcc -O2 -c -o {.}.o {} \
-S host1,host2 <<EOF [...] -S host1,host2 <<EOF [...]
=item toupper_run: # This is not exactly the same, but avoids the wrapper
parallel gcc -O2 -c -o {.}.o {} \
-S host1,host2 <<EOF [...]
../../paexec -lp -c "`pwd`/toupper_cmd" -n +10 <<EOF [...] =head4 toupper_run
parallel echo {} '|' ./toupper_cmd <<EOF [...]
# Without the wrapper:
parallel echo {} '| awk {print\ toupper\(\$0\)}' <<EOF [...]
=back 4$ ../../paexec -lp -c "`pwd`/toupper_cmd" -n +10 <<EOF [...]
4$ parallel echo {} '|' ./toupper_cmd <<EOF [...]
# Without the wrapper:
parallel echo {} '| awk {print\ toupper\(\$0\)}' <<EOF [...]
https://github.com/cheusov/paexec https://github.com/cheusov/paexec
=head2 DIFFERENCES BETWEEN map(sitaramc) AND GNU Parallel =head2 DIFFERENCES BETWEEN map(sitaramc) AND GNU Parallel
Summary table (see legend above):
I1 - - I4 - - -
M1 (M2) M3 M4 M5 -
- O2 O3 - O5 - - N/A N/A O10
E1 - - - - - -
- - - - - - - - -
- -
(M2): Only if there is a single replacement string.
B<map> sees it as a feature to have less features and in doing so it B<map> sees it as a feature to have less features and in doing so it
also handles corner cases incorrectly. A lot of GNU B<parallel>'s code also handles corner cases incorrectly. A lot of GNU B<parallel>'s code
is to handle corner cases correctly on every platform, so you will not is to handle corner cases correctly on every platform, so you will not
@ -707,8 +732,14 @@ context replace:
B<map> requires Perl v5.10.0 making it harder to use on old systems. B<map> requires Perl v5.10.0 making it harder to use on old systems.
B<map> has no way of using % in the command (GNU B<parallel> has -I to To put a % in the command line B<map> requires you to use %%:
specify another replacement string than B<{}>).
seq 10 | map -n1 echo % +10%% {}
seq 10 | parallel -I ,, echo ,, +10% {}
GNU B<parallel> has -I to specify another replacement string than
B<{}> if you use B<{}> in the command template.
By design B<map> is option incompatible with B<xargs>, it does not By design B<map> is option incompatible with B<xargs>, it does not
have remote job execution, a structured way of saving results, have remote job execution, a structured way of saving results,
@ -717,7 +748,7 @@ delimiter (only field delimiter), logging of jobs run with possibility
to resume, keeping the output in the same order as input, --pipe to resume, keeping the output in the same order as input, --pipe
processing, and dynamically timeouts. processing, and dynamically timeouts.
https://github.com/sitaramc/map https://github.com/sitaramc/map (Last checked: 2020-04)
=head2 DIFFERENCES BETWEEN ladon AND GNU Parallel =head2 DIFFERENCES BETWEEN ladon AND GNU Parallel
@ -753,21 +784,25 @@ fails for output larger than 200k:
It is assumed that the '--rpl's above are put in B<~/.parallel/config> It is assumed that the '--rpl's above are put in B<~/.parallel/config>
and that it is run under a shell that supports '**' globbing (such as B<zsh>): and that it is run under a shell that supports '**' globbing (such as B<zsh>):
B<1> ladon "**/*.txt" -- echo RELPATH 1$ ladon "**/*.txt" -- echo RELPATH
B<1> parallel echo RELPATH ::: **/*.txt 1$ parallel echo RELPATH ::: **/*.txt
B<2> ladon "~/Documents/**/*.pdf" -- shasum FULLPATH >hashes.txt 2$ ladon "~/Documents/**/*.pdf" -- shasum FULLPATH >hashes.txt
B<2> parallel shasum FULLPATH ::: ~/Documents/**/*.pdf >hashes.txt 2$ parallel shasum FULLPATH ::: ~/Documents/**/*.pdf >hashes.txt
B<3> ladon -m thumbs/RELDIR "**/*.jpg" -- convert FULLPATH -thumbnail 100x100^ -gravity center -extent 100x100 thumbs/RELPATH 3$ ladon -m thumbs/RELDIR "**/*.jpg" -- convert FULLPATH \
-thumbnail 100x100^ -gravity center -extent 100x100 \
thumbs/RELPATH
B<3> parallel mkdir -p thumbs/RELDIR\; convert FULLPATH -thumbnail 100x100^ -gravity center -extent 100x100 thumbs/RELPATH ::: **/*.jpg 3$ parallel mkdir -p thumbs/RELDIR\; convert FULLPATH
-thumbnail 100x100^ -gravity center -extent 100x100 \
thumbs/RELPATH ::: **/*.jpg
B<4> ladon "~/Music/*.wav" -- lame -V 2 FULLPATH DIRNAME/BASENAME.mp3 4$ ladon "~/Music/*.wav" -- lame -V 2 FULLPATH DIRNAME/BASENAME.mp3
B<4> parallel lame -V 2 FULLPATH DIRNAME/BASENAME.mp3 ::: ~/Music/*.wav 4$ parallel lame -V 2 FULLPATH DIRNAME/BASENAME.mp3 ::: ~/Music/*.wav
https://github.com/danielgtaylor/ladon (Last checked: 2019-01) https://github.com/danielgtaylor/ladon (Last checked: 2019-01)
@ -802,21 +837,21 @@ jobs. This can be emulated by GNU B<parallel> using B<bash>'s B<ulimit>:
=head3 EXAMPLES FROM jobflow README =head3 EXAMPLES FROM jobflow README
B<1> cat things.list | jobflow -threads=8 -exec ./mytask {} 1$ cat things.list | jobflow -threads=8 -exec ./mytask {}
B<1> cat things.list | parallel -j8 ./mytask {} 1$ cat things.list | parallel -j8 ./mytask {}
B<2> seq 100 | jobflow -threads=100 -exec echo {} 2$ seq 100 | jobflow -threads=100 -exec echo {}
B<2> seq 100 | parallel -j100 echo {} 2$ seq 100 | parallel -j100 echo {}
B<3> cat urls.txt | jobflow -threads=32 -exec wget {} 3$ cat urls.txt | jobflow -threads=32 -exec wget {}
B<3> cat urls.txt | parallel -j32 wget {} 3$ cat urls.txt | parallel -j32 wget {}
B<4> find . -name '*.bmp' | jobflow -threads=8 -exec bmp2jpeg {.}.bmp {.}.jpg 4$ find . -name '*.bmp' | jobflow -threads=8 -exec bmp2jpeg {.}.bmp {.}.jpg
B<4> find . -name '*.bmp' | parallel -j8 bmp2jpeg {.}.bmp {.}.jpg 4$ find . -name '*.bmp' | parallel -j8 bmp2jpeg {.}.bmp {.}.jpg
https://github.com/rofl0r/jobflow https://github.com/rofl0r/jobflow
@ -836,15 +871,15 @@ is killed.
Output to stderr (standard error) is changed if the command fails. Output to stderr (standard error) is changed if the command fails.
Here are the two examples from B<gargs> website. =head3 EXAMPLES FROM gargs WEBSITE
B<1> seq 12 -1 1 | gargs -p 4 -n 3 "sleep {0}; echo {1} {2}" 1$ seq 12 -1 1 | gargs -p 4 -n 3 "sleep {0}; echo {1} {2}"
B<1> seq 12 -1 1 | parallel -P 4 -n 3 "sleep {1}; echo {2} {3}" 1$ seq 12 -1 1 | parallel -P 4 -n 3 "sleep {1}; echo {2} {3}"
B<2> cat t.txt | gargs --sep "\s+" -p 2 "echo '{0}:{1}-{2}' full-line: \'{}\'" 2$ cat t.txt | gargs --sep "\s+" -p 2 "echo '{0}:{1}-{2}' full-line: \'{}\'"
B<2> cat t.txt | parallel --colsep "\\s+" -P 2 "echo '{1}:{2}-{3}' full-line: \'{}\'" 2$ cat t.txt | parallel --colsep "\\s+" -P 2 "echo '{1}:{2}-{3}' full-line: \'{}\'"
https://github.com/brentp/gargs https://github.com/brentp/gargs
@ -966,11 +1001,11 @@ instead of ^ as that is closer to bash's ${var%postfix}):
--rpl '{/:} s:(.*/)?([^/.]+)(\.[^/]+)*$:$2:' --rpl '{/:} s:(.*/)?([^/.]+)(\.[^/]+)*$:$2:'
--rpl '{@(.*?)} /$$1/ and $_=$1;' --rpl '{@(.*?)} /$$1/ and $_=$1;'
=head3 EXAMPLES FROM rush's WEBSITE
Here are the examples from B<rush>'s website with the equivalent Here are the examples from B<rush>'s website with the equivalent
command in GNU B<parallel>. command in GNU B<parallel>.
=head3 EXAMPLES
B<1. Simple run, quoting is not necessary> B<1. Simple run, quoting is not necessary>
$ seq 1 3 | rush echo {} $ seq 1 3 | rush echo {}
@ -1518,6 +1553,8 @@ Make Ctrl-C kill all running processes
=back =back
=head3 EXAMPLES FROM maps WEBSITE
Here are the 5 examples converted to GNU Parallel: Here are the 5 examples converted to GNU Parallel:
1$ ls *.c | map f 'foo $f' 1$ ls *.c | map f 'foo $f'
@ -1556,6 +1593,8 @@ B<loop> cannot run functions:
export -f myfunc export -f myfunc
loop 'myfunc this fails' loop 'myfunc this fails'
=head3 EXAMPLES FROM loop's WEBSITE
Some of the examples from https://github.com/Miserlou/Loop/ can be Some of the examples from https://github.com/Miserlou/Loop/ can be
emulated with GNU B<parallel>: emulated with GNU B<parallel>:
@ -1760,48 +1799,50 @@ to fit on a single command line.
B<paral> has no support for running jobs remotely. B<paral> has no support for running jobs remotely.
=head3 EXAMPLES FROM README.markdown
The examples from B<README.markdown> and the corresponding command run The examples from B<README.markdown> and the corresponding command run
with GNU B<parallel> (B<--results with GNU B<parallel> (B<--results
'out_{#}_{=s/[^\sa-z_0-9]//g;s/\s+/_/g=}.log' --eta> is omitted from 'out_{#}_{=s/[^\sa-z_0-9]//g;s/\s+/_/g=}.log' --eta> is omitted from
the GNU B<parallel> command): the GNU B<parallel> command):
paral "command 1" "command 2 --flag" "command arg1 arg2" 1$ paral "command 1" "command 2 --flag" "command arg1 arg2"
parallel ::: "command 1" "command 2 --flag" "command arg1 arg2" 1$ parallel ::: "command 1" "command 2 --flag" "command arg1 arg2"
paral "sleep 1 && echo c1" "sleep 2 && echo c2" \ 2$ paral "sleep 1 && echo c1" "sleep 2 && echo c2" \
"sleep 3 && echo c3" "sleep 4 && echo c4" "sleep 5 && echo c5" "sleep 3 && echo c3" "sleep 4 && echo c4" "sleep 5 && echo c5"
parallel ::: "sleep 1 && echo c1" "sleep 2 && echo c2" \ 2$ parallel ::: "sleep 1 && echo c1" "sleep 2 && echo c2" \
"sleep 3 && echo c3" "sleep 4 && echo c4" "sleep 5 && echo c5" "sleep 3 && echo c3" "sleep 4 && echo c4" "sleep 5 && echo c5"
# Or shorter: # Or shorter:
parallel "sleep {} && echo c{}" ::: {1..5} parallel "sleep {} && echo c{}" ::: {1..5}
paral -n=0 "sleep 5 && echo c5" "sleep 4 && echo c4" \ 3$ paral -n=0 "sleep 5 && echo c5" "sleep 4 && echo c4" \
"sleep 3 && echo c3" "sleep 2 && echo c2" "sleep 1 && echo c1" "sleep 3 && echo c3" "sleep 2 && echo c2" "sleep 1 && echo c1"
parallel ::: "sleep 5 && echo c5" "sleep 4 && echo c4" \ 3$ parallel ::: "sleep 5 && echo c5" "sleep 4 && echo c4" \
"sleep 3 && echo c3" "sleep 2 && echo c2" "sleep 1 && echo c1" "sleep 3 && echo c3" "sleep 2 && echo c2" "sleep 1 && echo c1"
# Or shorter: # Or shorter:
parallel -j0 "sleep {} && echo c{}" ::: 5 4 3 2 1 parallel -j0 "sleep {} && echo c{}" ::: 5 4 3 2 1
paral -n=1 "sleep 5 && echo c5" "sleep 4 && echo c4" \ 4$ paral -n=1 "sleep 5 && echo c5" "sleep 4 && echo c4" \
"sleep 3 && echo c3" "sleep 2 && echo c2" "sleep 1 && echo c1" "sleep 3 && echo c3" "sleep 2 && echo c2" "sleep 1 && echo c1"
parallel -j1 "sleep {} && echo c{}" ::: 5 4 3 2 1 4$ parallel -j1 "sleep {} && echo c{}" ::: 5 4 3 2 1
paral -n=2 "sleep 5 && echo c5" "sleep 4 && echo c4" \ 5$ paral -n=2 "sleep 5 && echo c5" "sleep 4 && echo c4" \
"sleep 3 && echo c3" "sleep 2 && echo c2" "sleep 1 && echo c1" "sleep 3 && echo c3" "sleep 2 && echo c2" "sleep 1 && echo c1"
parallel -j2 "sleep {} && echo c{}" ::: 5 4 3 2 1 5$ parallel -j2 "sleep {} && echo c{}" ::: 5 4 3 2 1
paral -n=5 "sleep 5 && echo c5" "sleep 4 && echo c4" \ 6$ paral -n=5 "sleep 5 && echo c5" "sleep 4 && echo c4" \
"sleep 3 && echo c3" "sleep 2 && echo c2" "sleep 1 && echo c1" "sleep 3 && echo c3" "sleep 2 && echo c2" "sleep 1 && echo c1"
parallel -j5 "sleep {} && echo c{}" ::: 5 4 3 2 1 6$ parallel -j5 "sleep {} && echo c{}" ::: 5 4 3 2 1
paral -n=1 "echo a && sleep 0.5 && echo b && sleep 0.5 && \ 7$ paral -n=1 "echo a && sleep 0.5 && echo b && sleep 0.5 && \
echo c && sleep 0.5 && echo d && sleep 0.5 && \ echo c && sleep 0.5 && echo d && sleep 0.5 && \
echo e && sleep 0.5 && echo f && sleep 0.5 && \ echo e && sleep 0.5 && echo f && sleep 0.5 && \
echo g && sleep 0.5 && echo h" echo g && sleep 0.5 && echo h"
parallel ::: "echo a && sleep 0.5 && echo b && sleep 0.5 && \ 7$ parallel ::: "echo a && sleep 0.5 && echo b && sleep 0.5 && \
echo c && sleep 0.5 && echo d && sleep 0.5 && \ echo c && sleep 0.5 && echo d && sleep 0.5 && \
echo e && sleep 0.5 && echo f && sleep 0.5 && \ echo e && sleep 0.5 && echo f && sleep 0.5 && \
echo g && sleep 0.5 && echo h" echo g && sleep 0.5 && echo h"
https://github.com/amattn/paral (Last checked: 2019-01) https://github.com/amattn/paral (Last checked: 2019-01)
@ -1811,19 +1852,21 @@ https://github.com/amattn/paral (Last checked: 2019-01)
B<concurr> is built to run jobs in parallel using a client/server B<concurr> is built to run jobs in parallel using a client/server
model. model.
=head3 EXAMPLES FROM README.md
The examples from B<README.md>: The examples from B<README.md>:
concurr 'echo job {#} on slot {%}: {}' : arg1 arg2 arg3 arg4 1$ concurr 'echo job {#} on slot {%}: {}' : arg1 arg2 arg3 arg4
parallel 'echo job {#} on slot {%}: {}' ::: arg1 arg2 arg3 arg4 1$ parallel 'echo job {#} on slot {%}: {}' ::: arg1 arg2 arg3 arg4
concurr 'echo job {#} on slot {%}: {}' :: file1 file2 file3 2$ concurr 'echo job {#} on slot {%}: {}' :: file1 file2 file3
parallel 'echo job {#} on slot {%}: {}' :::: file1 file2 file3 2$ parallel 'echo job {#} on slot {%}: {}' :::: file1 file2 file3
concurr 'echo {}' < input_file 3$ concurr 'echo {}' < input_file
parallel 'echo {}' < input_file 3$ parallel 'echo {}' < input_file
cat file | concurr 'echo {}' 4$ cat file | concurr 'echo {}'
cat file | parallel 'echo {}' 4$ cat file | parallel 'echo {}'
B<concurr> deals badly empty input files and with output larger than B<concurr> deals badly empty input files and with output larger than
64 KB. 64 KB.
@ -1857,37 +1900,45 @@ https://github.com/spion/npm-parallel (Last checked: 2019-01)
=head2 DIFFERENCES BETWEEN machma AND GNU Parallel =head2 DIFFERENCES BETWEEN machma AND GNU Parallel
B<machma> runs tasks in parallel. It gives time stamped B<machma> runs tasks in parallel. It gives time stamped
output. It buffers in RAM. The examples from README.md: output. It buffers in RAM.
# Put shorthand for timestamp in config for the examples =head3 EXAMPLES FROM README.md
echo '--rpl '\
\''{time} $_=::strftime("%Y-%m-%d %H:%M:%S",localtime())'\' \
> ~/.parallel/machma
echo '--line-buffer --tagstring "{#} {time} {}"' >> ~/.parallel/machma
find . -iname '*.jpg' | The examples from README.md:
machma -- mogrify -resize 1200x1200 -filter Lanczos {}
find . -iname '*.jpg' |
parallel --bar -Jmachma mogrify -resize 1200x1200 -filter Lanczos {}
cat /tmp/ips | machma -p 2 -- ping -c 2 -q {} 1$ # Put shorthand for timestamp in config for the examples
cat /tmp/ips | parallel -j2 -Jmachma ping -c 2 -q {} echo '--rpl '\
\''{time} $_=::strftime("%Y-%m-%d %H:%M:%S",localtime())'\' \
> ~/.parallel/machma
echo '--line-buffer --tagstring "{#} {time} {}"' \
>> ~/.parallel/machma
cat /tmp/ips | 2$ find . -iname '*.jpg' |
machma -- sh -c 'ping -c 2 -q $0 > /dev/null && echo alive' {} machma -- mogrify -resize 1200x1200 -filter Lanczos {}
cat /tmp/ips | find . -iname '*.jpg' |
parallel -Jmachma 'ping -c 2 -q {} > /dev/null && echo alive' parallel --bar -Jmachma mogrify -resize 1200x1200 \
-filter Lanczos {}
find . -iname '*.jpg' | 3$ cat /tmp/ips | machma -p 2 -- ping -c 2 -q {}
machma --timeout 5s -- mogrify -resize 1200x1200 -filter Lanczos {} 3$ cat /tmp/ips | parallel -j2 -Jmachma ping -c 2 -q {}
find . -iname '*.jpg' |
parallel --timeout 5s --bar mogrify -resize 1200x1200 \
-filter Lanczos {}
find . -iname '*.jpg' -print0 | 4$ cat /tmp/ips |
machma --null -- mogrify -resize 1200x1200 -filter Lanczos {} machma -- sh -c 'ping -c 2 -q $0 > /dev/null && echo alive' {}
find . -iname '*.jpg' -print0 | 4$ cat /tmp/ips |
parallel --null --bar mogrify -resize 1200x1200 -filter Lanczos {} parallel -Jmachma 'ping -c 2 -q {} > /dev/null && echo alive'
5$ find . -iname '*.jpg' |
machma --timeout 5s -- mogrify -resize 1200x1200 \
-filter Lanczos {}
5$ find . -iname '*.jpg' |
parallel --timeout 5s --bar mogrify -resize 1200x1200 \
-filter Lanczos {}
6$ find . -iname '*.jpg' -print0 |
machma --null -- mogrify -resize 1200x1200 -filter Lanczos {}
6$ find . -iname '*.jpg' -print0 |
parallel --null --bar mogrify -resize 1200x1200 \
-filter Lanczos {}
https://github.com/fd0/machma (Last checked: 2019-06) https://github.com/fd0/machma (Last checked: 2019-06)
@ -1909,6 +1960,8 @@ B<interface> does not buffer output, so output from different jobs mixes.
The overhead for each target is O(n*n), so with 1000 targets it The overhead for each target is O(n*n), so with 1000 targets it
becomes very slow with an overhead in the order of 500ms/target. becomes very slow with an overhead in the order of 500ms/target.
=head3 EXAMPLES FROM interlace's WEBSITE
Using B<prips> most of the examples from Using B<prips> most of the examples from
https://github.com/codingo/Interlace can be run with GNU B<parallel>: https://github.com/codingo/Interlace can be run with GNU B<parallel>:
@ -2319,17 +2372,18 @@ For full support of {n..m:s} including negative numbers use a dynamic
replacement string like this: replacement string like this:
PARALLEL=--rpl\ \''{r((-?\d+)?)\.\.((-?\d+)?)((:([^}]*))?)} PARALLEL=--rpl\ \''{r((-?\d+)?)\.\.((-?\d+)?)((:([^}]*))?)}
$a = defined $$2 ? $$2 < 0 ? 1+$#arg+$$2 : $$2 : 1; $a = defined $$2 ? $$2 < 0 ? 1+$#arg+$$2 : $$2 : 1;
$b = defined $$4 ? $$4 < 0 ? 1+$#arg+$$4 : $$4 : $#arg+1; $b = defined $$4 ? $$4 < 0 ? 1+$#arg+$$4 : $$4 : $#arg+1;
$s = defined $$6 ? $$7 : " "; $s = defined $$6 ? $$7 : " ";
$_ = join $s,@arg[$a..$b]'\' $_ = join $s,@arg[$a..$b]'\'
export PARALLEL export PARALLEL
You can then do: You can then do:
head /etc/passwd | parallel --colsep : echo ..={1r..} ..3={1r..3} 4..={1r4..} 2..4={1r2..4} 3..3={1r3..3} ..3:-={1r..3:-} ..3:/={1r..3:/} -1={-1} -5={-5} -6={-6} -3..={1r-3..} head /etc/passwd | parallel --colsep : echo ..={1r..} ..3={1r..3} \
4..={1r4..} 2..4={1r2..4} 3..3={1r3..3} ..3:-={1r..3:-} \
..3:/={1r..3:/} -1={-1} -5={-5} -6={-6} -3..={1r-3..}
=head3 EXAMPLES FROM rargs MANUAL =head3 EXAMPLES FROM rargs MANUAL
@ -2370,6 +2424,7 @@ virtual memory will cause the machine to crash.
https://github.com/voodooEntity/threader (Last checked: 2020-04) https://github.com/voodooEntity/threader (Last checked: 2020-04)
=head2 DIFFERENCES BETWEEN runp AND GNU Parallel =head2 DIFFERENCES BETWEEN runp AND GNU Parallel
Summary table (see legend above): Summary table (see legend above):
@ -2496,9 +2551,53 @@ output to stderr (this can be prevented with -q)
https://github.com/jreisinger/runp (Last checked: 2020-04) https://github.com/jreisinger/runp (Last checked: 2020-04)
=head2 Todo =head2 DIFFERENCES BETWEEN papply AND GNU Parallel
https://pypi.org/project/papply/ Summary table (see legend above):
- - - I4 - - -
M1 - M3 - - M6
- - O3 - O5 - - N/A N/A O10
E1 - - E4 - - -
- - - - - - - - -
- -
B<papply> does not print the output if the command fails:
$ papply 'echo %F; false' foo
"echo foo; false" did not succeed
B<papply>'s replacement strings (%F %d %f %n %e %z) can be simulated in GNU
B<parallel> by putting this in B<~/.parallel/config>:
--rpl '%F'
--rpl '%d $_=Q(::dirname($_));'
--rpl '%f s:.*/::;'
--rpl '%n s:.*/::;s:\.[^/.]+$::;'
--rpl '%e s:.*\.:.:'
--rpl '%z $_=""'
B<papply> buffers in RAM, and uses twice the amount of output. So
output of 5 GB takes 10 GB RAM.
The buffering is very CPU intensive: Buffering a line of 5 GB takes 40
seconds (compared to 10 seconds with GNU B<parallel>).
=head3 Examples as GNU Parallel
1$ papply gzip *.txt
1$ parallel gzip ::: *.txt
2$ papply "convert %F %n.jpg" *.png
2$ parallel convert {} {.}.jpg ::: *.png
https://pypi.org/project/papply/ (Last checked: 2020-04)
=head2 Todo
https://github.com/JeiKeiLim/simple_distribute_job https://github.com/JeiKeiLim/simple_distribute_job

2
src/sql
View file

@ -574,7 +574,7 @@ $Global::Initfile && unlink $Global::Initfile;
exit ($err); exit ($err);
sub parse_options { sub parse_options {
$Global::version = 20200422; $Global::version = 20200423;
$Global::progname = 'sql'; $Global::progname = 'sql';
# This must be done first as this may exec myself # This must be done first as this may exec myself

6
testsuite/tests-to-run/parallel-local-ssh6.sh
View file

@ -135,6 +135,12 @@ par_remote_symlink_dir() {
ssh parallel@lo rm wd && echo OK: wd is still a symlink with PARALLEL_RSYNC_OPTS ssh parallel@lo rm wd && echo OK: wd is still a symlink with PARALLEL_RSYNC_OPTS
} }
par_sshlogin_replacement() {
echo '### show {sshlogin} and {host}'
parallel -S $SSHLOGIN1 --plus echo {sshlogin} {} {host} ::: and
parallel -S '5//usr/bin/ssh '$SSHLOGIN1 --plus echo {sshlogin} {} {host} ::: and
}
export -f $(compgen -A function | grep par_) export -f $(compgen -A function | grep par_)
#compgen -A function | grep par_ | sort | parallel --delay $D -j$P --tag -k '{} 2>&1' #compgen -A function | grep par_ | sort | parallel --delay $D -j$P --tag -k '{} 2>&1'
compgen -A function | grep par_ | sort | compgen -A function | grep par_ | sort |

2
testsuite/wanted-results/parallel-local-ssh1
View file

@ -72,7 +72,7 @@ par_remote_load parallel: Warning: This job was killed because it timed out:
par_remote_load parallel: Warning: perl -e 'while(1){ }' par_remote_load parallel: Warning: perl -e 'while(1){ }'
par_remote_load OK par_remote_load OK
par_remote_nice ### Test --nice remote par_remote_nice ### Test --nice remote
par_remote_nice ssh lo -- exec 'perl -e '"'"'$ENV{"PARALLEL_PID"}="XXXXX";$ENV{"PARALLEL_SEQ"}="1";$ENV{"PARALLEL_SSHLOGIN"}="lo";$bashfunc = "";@ARGV="PAR=a bash -c \"echo \\\$PAR b\"";$shell="$ENV{SHELL}";$tmpdir="/tmp/parallel-local-ssh1-tmpdir";$nice=1;do{$ENV{PARALLEL_TMP}=$tmpdir."/par".join"",map{(0..9,"a".."z","A".."Z")[rand(62)]}(1..5);}while(-e$ENV{PARALLEL_TMP});$SIG{CHLD}=sub{$done=1;};$pid=fork;unless($pid){eval{setpgrp};eval{setpriority(0,0,$nice)};exec$shell,"-c",($bashfunc."@ARGV");die"exec:$!\n";}do{$s=$s<1?0.001+$s*1.03:$s;select(undef,undef,undef,$s);}until($done||getppid==1);kill(SIGHUP,-${pid})unless$done;wait;exit($?&127?128+($?&127):1+$?>>8)'"'"; par_remote_nice ssh lo -- exec 'perl -e '"'"'$ENV{"PARALLEL_PID"}="XXXXX";$ENV{"PARALLEL_SEQ"}="1";$ENV{"PARALLEL_SSHLOGIN"}="lo";$ENV{"PARALLEL_SSHHOST"}="lo";$bashfunc = "";@ARGV="PAR=a bash -c \"echo \\\$PAR b\"";$shell="$ENV{SHELL}";$tmpdir="/tmp/parallel-local-ssh1-tmpdir";$nice=1;do{$ENV{PARALLEL_TMP}=$tmpdir."/par".join"",map{(0..9,"a".."z","A".."Z")[rand(62)]}(1..5);}while(-e$ENV{PARALLEL_TMP});$SIG{CHLD}=sub{$done=1;};$pid=fork;unless($pid){eval{setpgrp};eval{setpriority(0,0,$nice)};exec$shell,"-c",($bashfunc."@ARGV");die"exec:$!\n";}do{$s=$s<1?0.001+$s*1.03:$s;select(undef,undef,undef,$s);}until($done||getppid==1);kill(SIGHUP,-${pid})unless$done;wait;exit($?&127?128+($?&127):1+$?>>8)'"'";
par_remote_nice a b par_remote_nice a b
par_ssh ### use --ssh par_ssh ### use --ssh
par_ssh Run through BARSSH? par_ssh Run through BARSSH?

3
testsuite/wanted-results/parallel-local-ssh6
View file

@ -26,6 +26,9 @@ par_read_sshloginfile_from_stdin foo
par_remote_symlink_dir bug #51293: parallel does not preserve symlinked directory structure on remote par_remote_symlink_dir bug #51293: parallel does not preserve symlinked directory structure on remote
par_remote_symlink_dir OK: wd is still a symlink with --rsync-opts par_remote_symlink_dir OK: wd is still a symlink with --rsync-opts
par_remote_symlink_dir OK: wd is still a symlink with PARALLEL_RSYNC_OPTS par_remote_symlink_dir OK: wd is still a symlink with PARALLEL_RSYNC_OPTS
par_sshlogin_replacement ### show {sshlogin} and {host}
par_sshlogin_replacement parallel@lo and lo
par_sshlogin_replacement /usr/bin/ssh parallel@lo and lo
par_test_nonall ### Test --nonall par_test_nonall ### Test --nonall
par_test_nonall /home/csh par_test_nonall /home/csh
par_test_nonall /home/parallel par_test_nonall /home/parallel