parallel: Fixed bug #57055: {sshlogin} shorthand for $PARALLEL_SSHLOGIN.

This commit is contained in:
Ole Tange 2020-05-08 03:35:51 +02:00
parent e5fd1c0767
commit 6a67fd09d4
9 changed files with 379 additions and 284 deletions

View file

@ -131,7 +131,7 @@ pushd
git diff
# Recheck OBS https://build.opensuse.org/package/show?package=parallel&project=home%3Atange
# Recheck OBS https://build.opensuse.org/package/show/home:tange/parallel
YYYYMMDD=`yyyymmdd`
TAG=MyTag
@ -211,46 +211,24 @@ from:tange@gnu.org
to:parallel@gnu.org, bug-parallel@gnu.org
stable-bcc: Jesse Alama <jessealama@fastmail.fm>
Subject: GNU Parallel 20200422 ('10years') released <<[stable]>>
Subject: GNU Parallel 20200522 ('Kraftwerk') released <<[stable]>>
GNU Parallel 20200422 ('10years') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/
GNU Parallel 20200522 ('') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/
<<No new functionality was introduced so this is a good candidate for a stable release.>>
This release celebrates GNU Parallel's 10 years as a GNU tool:
> commit ed2dfb1043768154d4e7678e01e10287155fa834
> Author: Ole Tange <ole@tange.dk>
> Date: Thu Apr 22 01:23:00 2010 +0200
>
> Name change: Parallel is now GNU Parallel.
> Basic structure for sshlogin and sshloginfile.
Unfortunately the physical celebrations has been cancelled due to COVID19.
Quote of the month:
I wish more command line software had example pages as robust as GNU Parallel
-- Lucidbeaming @lucidbeaming
<<>>
New in this release:
* parsort makes GNU sort run faster for files with more than 1M lines.
* Bug fixes and man page updates.
News about GNU Parallel:
* Several clusters have updated their documentation of GNU Parallel: https://www.grid5000.fr/w/GNU_Parallel https://research-it.berkeley.edu/virtual-training-running-jobs-parallel-savio http://www.hpc.lsu.edu/training/archive/tutorials.php
* add_pho uses GNU Parallel: https://github.com/SeedlingsBabylab/add_pho
* GNU parallel使用笔记https://zhuanlan.zhihu.com/p/37340011
* Virtual Workshop: Running Jobs in Parallel on Savio https://ais.berkeley.edu/events/virtual-workshop-running-jobs-parallel-savio/2020-04-20
* Free Concurrency with GNU Parallel https://phili.pe/posts/free-concurrency-with-gnu-parallel/
*
Get the book: GNU Parallel 2018 http://www.lulu.com/shop/ole-tange/gnu-parallel-2018/paperback/product-23558902.html

View file

@ -23,7 +23,7 @@
use strict;
use Getopt::Long;
$Global::progname="niceload";
$Global::version = 20200422;
$Global::version = 20200423;
Getopt::Long::Configure("bundling","require_order");
get_options_from_array(\@ARGV) || die_usage();
if($opt::version) {

View file

@ -2029,8 +2029,8 @@ sub parse_options(@) {
# If you want GNU Parallel to be maintained in the future keep
# this line.
citation_notice();
# Seriously: YOU will be harming free software by removing the
# notice. You make it harder to justify spending time developing
# Seriously: _YOU_ will be harming free software by removing the
# notice. _YOU_ make it harder to justify spending time developing
# it. If you *do* remove the line, please email
# hallofshame@tange.dk if you want to avoid being put in a hall of
# shame.
@ -2150,7 +2150,7 @@ sub check_invalid_option_combinations() {
sub init_globals() {
# Defaults:
$Global::version = 20200422;
$Global::version = 20200423;
$Global::progname = 'parallel';
$::name = "GNU Parallel";
$Global::infinity = 2**31;
@ -2212,6 +2212,10 @@ sub init_globals() {
'{,([^}]+?)}' => 's/^($$1)/lc($1)/e;',
# Bash ${a,,A}
'{,,([^}]+?)}' => 's/($$1)/lc($1)/eg;',
# {host} = ssh host
'{host}' => '$_="\${PARALLEL_SSHHOST}";uq()',
# {sshlogin} = sshlogin
'{sshlogin}' => '$_="\${PARALLEL_SSHLOGIN}";uq()',
);
# Modifiable copy of %Global::replace
%Global::rpl = %Global::replace;
@ -2367,7 +2371,7 @@ sub parse_replacement_string_options() {
sub parse_semaphore() {
# Semaphore defaults
# Must be done before computing number of processes and max_line_length
# because when running as a semaphore GNU Parallel does not read args
# because when running as a semaphore GNU Parallel does not read args
# Uses:
# $opt::semaphore
# $Global::semaphore
@ -3196,7 +3200,7 @@ sub __RUNNING_THE_JOBS_AND_PRINTING_PROGRESS__() {}
# $Global::total_started = total number of jobs started
# $Global::joblog = filehandle of joblog
# $Global::debug = Is debugging on?
# $Global::exitstatus = status code of GNU Parallel
# $Global::exitstatus = status code of GNU Parallel
# $Global::quoting = quote the command to run
sub init_run_jobs() {
@ -4152,14 +4156,14 @@ sub run_gnu_parallel() {
}
sub _run_gnu_parallel() {
# Run GNU Parallel
# Run GNU Parallel
# This should ideally just fork an internal copy
# and not start it through a shell
# Input:
# $stdin = data to provide on stdin for GNU Parallel
# $stdin = data to provide on stdin for GNU Parallel
# @args = command line arguments
# Returns:
# $exitstatus = exitcode of GNU Parallel run
# $exitstatus = exitcode of GNU Parallel run
# \@stdout = standard output
# \@stderr = standard error
my ($stdin,@args) = @_;
@ -4543,7 +4547,7 @@ sub onall($@) {
);
::debug("init", "| $0 $options\n");
open(my $parallel_fh, "|-", "$0 -0 --will-cite -j0 $options") ||
::die_bug("This does not run GNU Parallel: $0 $options");
::die_bug("This does not run GNU Parallel: $0 $options");
my @joblogs;
for my $host (sort keys %Global::host) {
my $sshlogin = $Global::host{$host};
@ -4607,14 +4611,14 @@ sub sigpipe() {
sub signal_children() {
# Send signal to all children process groups
# and GNU Parallel itself
# and GNU Parallel itself
# Uses:
# %SIG
# Returns: N/A
my $signal = shift;
debug("run", "Sending $signal ");
kill $signal, map { -$_ } keys %Global::running;
# Use default signal handler for GNU Parallel itself
# Use default signal handler for GNU Parallel itself
$SIG{$signal} = undef;
kill $signal, $$;
}
@ -4925,7 +4929,7 @@ sub citation_notice() {
" DOI https://doi.org/10.5281/zenodo.1146014",
"",
# Before changing this line, please read
# https://www.gnu.org/software/parallel/parallel_design.html#Citation-notice
# https://www.gnu.org/software/parallel/parallel_design.html#Citation-notice and
# https://git.savannah.gnu.org/cgit/parallel.git/tree/doc/citation-notice-faq.txt
"This helps funding further development; AND IT WON'T COST YOU A CENT.",
"If you pay 10000 EUR you should feel free to use GNU Parallel without citing.",
@ -4937,7 +4941,7 @@ sub citation_notice() {
""
);
mkdir $Global::config_dir;
# Number of times the user has run GNU Parallel without showing
# Number of times the user has run GNU Parallel without showing
# willingness to cite
my $runs = 0;
if(open (my $fh, "<", $Global::config_dir.
@ -5049,7 +5053,7 @@ sub citation() {
"(Feel free to use \\nocite{tange_ole_2018_1146014})",
"",
# Before changing this line, please read
# https://www.gnu.org/software/parallel/parallel_design.html#Citation-notice
# https://www.gnu.org/software/parallel/parallel_design.html#Citation-notice and
# https://git.savannah.gnu.org/cgit/parallel.git/tree/doc/citation-notice-faq.txt
"This helps funding further development; AND IT WON'T COST YOU A CENT.",
"If you pay 10000 EUR you should feel free to use GNU Parallel without citing.",
@ -5501,7 +5505,7 @@ sub seconds_to_time_units() {
# N/A
if(not $disk_full_fh) {
$disk_full_fh = ::tmpfile(SUFFIX => ".df");
$b8193 = "x"x8193;
$b8193 = "b"x8193;
}
# Linux does not discover if a disk is full if writing <= 8192
# Tested on:
@ -8973,7 +8977,7 @@ sub sshlogin_wrap($) {
}
# Duplicate vars as BASH functions to include post-shellshock functions (v1+v2)
# So --env myfunc should look for BASH_FUNC_myfunc() and BASH_FUNC_myfunc%%
push(@vars, "PARALLEL_PID", "PARALLEL_SEQ", "PARALLEL_SSHLOGIN",
push(@vars, "PARALLEL_PID", "PARALLEL_SEQ", "PARALLEL_SSHLOGIN", "PARALLEL_SSHHOST",
map { ("BASH_FUNC_$_()", "BASH_FUNC_$_%%") } @vars);
# Keep only defined variables
return grep { defined($ENV{$_}) } @vars;
@ -9033,6 +9037,7 @@ sub sshlogin_wrap($) {
my $quoted_remote_command;
$ENV{'PARALLEL_SEQ'} = $self->seq();
$ENV{'PARALLEL_SSHLOGIN'} = $sshlogin->string();
$ENV{'PARALLEL_SSHHOST'} = $sshlogin->serverlogin();
$ENV{'PARALLEL_PID'} = $$;
if($serverlogin eq ":") {
if($opt::workdir) {
@ -9792,7 +9797,6 @@ sub print($) {
$opt::ungroup and return;
}
}
# Check for disk full
::exit_if_disk_full();
}

View file

@ -1546,6 +1546,11 @@ choose k. k is the number of input sources and n is the number of
arguments in an input source. The content of the input sources must
be the same and the arguments must be unique.
For remote jobs these are also activated:
{sshlogin} sshlogin
{host} hostname of sshlogin
The following dynamic replacement strings are also activated. They are
inspired by bash's parameter expansion:
@ -5134,16 +5139,16 @@ http://stackoverflow.com/help/mcve).
It should be a complete example that others can run that shows the
problem including all files needed to run the example. This should
preferably be small and simple, so try to remove as many options as
possible. A combination of B<yes>, B<seq>, B<cat>, B<echo>, and
possible. A combination of B<yes>, B<seq>, B<cat>, B<echo>, B<wc>, and
B<sleep> can reproduce most errors. If your example requires large
files, see if you can make them by something like B<seq 100000000> >
files, see if you can make them with something like B<seq 100000000> >
B<bigfile> or B<yes | head -n 1000000000> > B<file>.
If your example requires remote execution, see if you can use
B<localhost> - maybe using another login.
If you have access to a different system, test if the MCVE shows the
problem on that system.
If you have access to a different system (maybe a VirtualBox on your
own machine), test if the MCVE shows the problem on that system.
=item *

View file

@ -281,46 +281,47 @@ start. GNU B<parallel> only requires one step.
Here are the examples from B<ppss>'s manual page with the equivalent
using GNU B<parallel>:
B<1> ./ppss.sh standalone -d /path/to/files -c 'gzip '
1$ ./ppss.sh standalone -d /path/to/files -c 'gzip '
B<1> find /path/to/files -type f | parallel gzip
1$ find /path/to/files -type f | parallel gzip
B<2> ./ppss.sh standalone -d /path/to/files -c 'cp "$ITEM" /destination/dir '
2$ ./ppss.sh standalone -d /path/to/files -c 'cp "$ITEM" /destination/dir '
B<2> find /path/to/files -type f | parallel cp {} /destination/dir
2$ find /path/to/files -type f | parallel cp {} /destination/dir
B<3> ./ppss.sh standalone -f list-of-urls.txt -c 'wget -q '
3$ ./ppss.sh standalone -f list-of-urls.txt -c 'wget -q '
B<3> parallel -a list-of-urls.txt wget -q
3$ parallel -a list-of-urls.txt wget -q
B<4> ./ppss.sh standalone -f list-of-urls.txt -c 'wget -q "$ITEM"'
4$ ./ppss.sh standalone -f list-of-urls.txt -c 'wget -q "$ITEM"'
B<4> parallel -a list-of-urls.txt wget -q {}
4$ parallel -a list-of-urls.txt wget -q {}
B<5> ./ppss config -C config.cfg -c 'encode.sh ' -d /source/dir -m
192.168.1.100 -u ppss -k ppss-key.key -S ./encode.sh -n nodes.txt -o
/some/output/dir --upload --download ; ./ppss deploy -C config.cfg ;
./ppss start -C config
5$ ./ppss config -C config.cfg -c 'encode.sh ' -d /source/dir \
-m 192.168.1.100 -u ppss -k ppss-key.key -S ./encode.sh \
-n nodes.txt -o /some/output/dir --upload --download;
./ppss deploy -C config.cfg
./ppss start -C config
B<5> # parallel does not use configs. If you want a different username put it in nodes.txt: user@hostname
5$ # parallel does not use configs. If you want a different username put it in nodes.txt: user@hostname
find source/dir -type f |
parallel --sshloginfile nodes.txt --trc {.}.mp3 lame -a {} -o {.}.mp3 --preset standard --quiet
B<5> find source/dir -type f | parallel --sshloginfile nodes.txt --trc {.}.mp3 lame -a {} -o {.}.mp3 --preset standard --quiet
6$ ./ppss stop -C config.cfg
B<6> ./ppss stop -C config.cfg
6$ killall -TERM parallel
B<6> killall -TERM parallel
7$ ./ppss pause -C config.cfg
B<7> ./ppss pause -C config.cfg
7$ Press: CTRL-Z or killall -SIGTSTP parallel
B<7> Press: CTRL-Z or killall -SIGTSTP parallel
8$ ./ppss continue -C config.cfg
B<8> ./ppss continue -C config.cfg
8$ Enter: fg or killall -SIGCONT parallel
B<8> Enter: fg or killall -SIGCONT parallel
9$ ./ppss.sh status -C config.cfg
B<9> ./ppss.sh status -C config.cfg
B<9> killall -SIGUSR2 parallel
9$ killall -SIGUSR2 parallel
https://github.com/louwrentius/PPSS
@ -342,55 +343,54 @@ B<pexec> is also a tool for running jobs in parallel.
Here are the examples from B<pexec>'s info page with the equivalent
using GNU B<parallel>:
B<1> pexec -o sqrt-%s.dat -p "$(seq 10)" -e NUM -n 4 -c -- \
'echo "scale=10000;sqrt($NUM)" | bc'
1$ pexec -o sqrt-%s.dat -p "$(seq 10)" -e NUM -n 4 -c -- \
'echo "scale=10000;sqrt($NUM)" | bc'
B<1> seq 10 | parallel -j4 'echo "scale=10000;sqrt({})" | bc > sqrt-{}.dat'
1$ seq 10 | parallel -j4 'echo "scale=10000;sqrt({})" | \
bc > sqrt-{}.dat'
B<2> pexec -p "$(ls myfiles*.ext)" -i %s -o %s.sort -- sort
2$ pexec -p "$(ls myfiles*.ext)" -i %s -o %s.sort -- sort
B<2> ls myfiles*.ext | parallel sort {} ">{}.sort"
2$ ls myfiles*.ext | parallel sort {} ">{}.sort"
B<3> pexec -f image.list -n auto -e B -u star.log -c -- \
'fistar $B.fits -f 100 -F id,x,y,flux -o $B.star'
3$ pexec -f image.list -n auto -e B -u star.log -c -- \
'fistar $B.fits -f 100 -F id,x,y,flux -o $B.star'
B<3> parallel -a image.list \
'fistar {}.fits -f 100 -F id,x,y,flux -o {}.star' 2>star.log
3$ parallel -a image.list \
'fistar {}.fits -f 100 -F id,x,y,flux -o {}.star' 2>star.log
B<4> pexec -r *.png -e IMG -c -o - -- \
'convert $IMG ${IMG%.png}.jpeg ; "echo $IMG: done"'
4$ pexec -r *.png -e IMG -c -o - -- \
'convert $IMG ${IMG%.png}.jpeg ; "echo $IMG: done"'
B<4> ls *.png | parallel 'convert {} {.}.jpeg; echo {}: done'
4$ ls *.png | parallel 'convert {} {.}.jpeg; echo {}: done'
B<5> pexec -r *.png -i %s -o %s.jpg -c 'pngtopnm | pnmtojpeg'
5$ pexec -r *.png -i %s -o %s.jpg -c 'pngtopnm | pnmtojpeg'
B<5> ls *.png | parallel 'pngtopnm < {} | pnmtojpeg > {}.jpg'
5$ ls *.png | parallel 'pngtopnm < {} | pnmtojpeg > {}.jpg'
B<6> for p in *.png ; do echo ${p%.png} ; done | \
pexec -f - -i %s.png -o %s.jpg -c 'pngtopnm | pnmtojpeg'
6$ for p in *.png ; do echo ${p%.png} ; done | \
pexec -f - -i %s.png -o %s.jpg -c 'pngtopnm | pnmtojpeg'
B<6> ls *.png | parallel 'pngtopnm < {} | pnmtojpeg > {.}.jpg'
6$ ls *.png | parallel 'pngtopnm < {} | pnmtojpeg > {.}.jpg'
B<7> LIST=$(for p in *.png ; do echo ${p%.png} ; done)
pexec -r $LIST -i %s.png -o %s.jpg -c 'pngtopnm | pnmtojpeg'
7$ LIST=$(for p in *.png ; do echo ${p%.png} ; done)
pexec -r $LIST -i %s.png -o %s.jpg -c 'pngtopnm | pnmtojpeg'
B<7> ls *.png | parallel 'pngtopnm < {} | pnmtojpeg > {.}.jpg'
7$ ls *.png | parallel 'pngtopnm < {} | pnmtojpeg > {.}.jpg'
B<8> pexec -n 8 -r *.jpg -y unix -e IMG -c \
'pexec -j -m blockread -d $IMG | \
jpegtopnm | pnmscale 0.5 | pnmtojpeg | \
pexec -j -m blockwrite -s th_$IMG'
8$ pexec -n 8 -r *.jpg -y unix -e IMG -c \
'pexec -j -m blockread -d $IMG | \
jpegtopnm | pnmscale 0.5 | pnmtojpeg | \
pexec -j -m blockwrite -s th_$IMG'
B<8> Combining GNU B<parallel> and GNU B<sem>.
8$ # Combining GNU B<parallel> and GNU B<sem>.
ls *jpg | parallel -j8 'sem --id blockread cat {} | jpegtopnm |' \
'pnmscale 0.5 | pnmtojpeg | sem --id blockwrite cat > th_{}'
B<8> ls *jpg | parallel -j8 'sem --id blockread cat {} | jpegtopnm |' \
'pnmscale 0.5 | pnmtojpeg | sem --id blockwrite cat > th_{}'
B<8> If reading and writing is done to the same disk, this may be
faster as only one process will be either reading or writing:
B<8> ls *jpg | parallel -j8 'sem --id diskio cat {} | jpegtopnm |' \
'pnmscale 0.5 | pnmtojpeg | sem --id diskio cat > th_{}'
# If reading and writing is done to the same disk, this may be
# faster as only one process will be either reading or writing:
ls *jpg | parallel -j8 'sem --id diskio cat {} | jpegtopnm |' \
'pnmscale 0.5 | pnmtojpeg | sem --id diskio cat > th_{}'
https://www.gnu.org/software/pexec/
@ -403,38 +403,40 @@ running jobs on your local computer.
B<xjobs> deals badly with special characters just like B<xargs>. See
the section B<DIFFERENCES BETWEEN xargs AND GNU Parallel>.
=head3 EXAMPLES FROM xjobs MANUAL
Here are the examples from B<xjobs>'s man page with the equivalent
using GNU B<parallel>:
B<1> ls -1 *.zip | xjobs unzip
1$ ls -1 *.zip | xjobs unzip
B<1> ls *.zip | parallel unzip
1$ ls *.zip | parallel unzip
B<2> ls -1 *.zip | xjobs -n unzip
2$ ls -1 *.zip | xjobs -n unzip
B<2> ls *.zip | parallel unzip >/dev/null
2$ ls *.zip | parallel unzip >/dev/null
B<3> find . -name '*.bak' | xjobs gzip
3$ find . -name '*.bak' | xjobs gzip
B<3> find . -name '*.bak' | parallel gzip
3$ find . -name '*.bak' | parallel gzip
B<4> ls -1 *.jar | sed 's/\(.*\)/\1 > \1.idx/' | xjobs jar tf
4$ ls -1 *.jar | sed 's/\(.*\)/\1 > \1.idx/' | xjobs jar tf
B<4> ls *.jar | parallel jar tf {} '>' {}.idx
4$ ls *.jar | parallel jar tf {} '>' {}.idx
B<5> xjobs -s script
5$ xjobs -s script
B<5> cat script | parallel
5$ cat script | parallel
B<6> mkfifo /var/run/my_named_pipe;
xjobs -s /var/run/my_named_pipe &
echo unzip 1.zip >> /var/run/my_named_pipe;
echo tar cf /backup/myhome.tar /home/me >> /var/run/my_named_pipe
6$ mkfifo /var/run/my_named_pipe;
xjobs -s /var/run/my_named_pipe &
echo unzip 1.zip >> /var/run/my_named_pipe;
echo tar cf /backup/myhome.tar /home/me >> /var/run/my_named_pipe
B<6> mkfifo /var/run/my_named_pipe;
cat /var/run/my_named_pipe | parallel &
echo unzip 1.zip >> /var/run/my_named_pipe;
echo tar cf /backup/myhome.tar /home/me >> /var/run/my_named_pipe
6$ mkfifo /var/run/my_named_pipe;
cat /var/run/my_named_pipe | parallel &
echo unzip 1.zip >> /var/run/my_named_pipe;
echo tar cf /backup/myhome.tar /home/me >> /var/run/my_named_pipe
http://www.maier-komor.de/xjobs.html (Last checked: 2019-01)
@ -452,11 +454,14 @@ B<prll> generates a lot of status information on stderr (standard
error) which makes it harder to use the stderr (standard error) output
of the job directly as input for another program.
=head3 EXAMPLES FROM prll's MANUAL
Here is the example from B<prll>'s man page with the equivalent
using GNU B<parallel>:
prll -s 'mogrify -flip $1' *.jpg
parallel mogrify -flip ::: *.jpg
1$ prll -s 'mogrify -flip $1' *.jpg
1$ parallel mogrify -flip ::: *.jpg
https://github.com/exzombie/prll (Last checked: 2019-01)
@ -477,13 +482,15 @@ semicomplete.com/blog/geekery/distributed-xargs.html (Last checked: 2019-01)
middleman(mdm) is also a tool for running jobs in parallel.
=head3 EXAMPLES FROM middleman's WEBSITE
Here are the shellscripts of
https://web.archive.org/web/20110728064735/http://mdm.
berlios.de/usage.html ported to GNU B<parallel>:
seq 19 | parallel buffon -o - | sort -n > result
cat files | parallel cmd
find dir -execdir sem cmd {} \;
1$ seq 19 | parallel buffon -o - | sort -n > result
cat files | parallel cmd
find dir -execdir sem cmd {} \;
https://github.com/cklin/mdm (Last checked: 2019-01)
@ -492,56 +499,58 @@ https://github.com/cklin/mdm (Last checked: 2019-01)
B<xapply> can run jobs in parallel on the local computer.
=head3 EXAMPLES FROM xapply's MANUAL
Here are the examples from B<xapply>'s man page with the equivalent
using GNU B<parallel>:
B<1> xapply '(cd %1 && make all)' */
1$ xapply '(cd %1 && make all)' */
B<1> parallel 'cd {} && make all' ::: */
1$ parallel 'cd {} && make all' ::: */
B<2> xapply -f 'diff %1 ../version5/%1' manifest | more
2$ xapply -f 'diff %1 ../version5/%1' manifest | more
B<2> parallel diff {} ../version5/{} < manifest | more
2$ parallel diff {} ../version5/{} < manifest | more
B<3> xapply -p/dev/null -f 'diff %1 %2' manifest1 checklist1
3$ xapply -p/dev/null -f 'diff %1 %2' manifest1 checklist1
B<3> parallel --link diff {1} {2} :::: manifest1 checklist1
3$ parallel --link diff {1} {2} :::: manifest1 checklist1
B<4> xapply 'indent' *.c
4$ xapply 'indent' *.c
B<4> parallel indent ::: *.c
4$ parallel indent ::: *.c
B<5> find ~ksb/bin -type f ! -perm -111 -print | xapply -f -v 'chmod a+x' -
5$ find ~ksb/bin -type f ! -perm -111 -print | xapply -f -v 'chmod a+x' -
B<5> find ~ksb/bin -type f ! -perm -111 -print | parallel -v chmod a+x
5$ find ~ksb/bin -type f ! -perm -111 -print | parallel -v chmod a+x
B<6> find */ -... | fmt 960 1024 | xapply -f -i /dev/tty 'vi' -
6$ find */ -... | fmt 960 1024 | xapply -f -i /dev/tty 'vi' -
B<6> sh <(find */ -... | parallel -s 1024 echo vi)
6$ sh <(find */ -... | parallel -s 1024 echo vi)
B<6> find */ -... | parallel -s 1024 -Xuj1 vi
6$ find */ -... | parallel -s 1024 -Xuj1 vi
B<7> find ... | xapply -f -5 -i /dev/tty 'vi' - - - - -
7$ find ... | xapply -f -5 -i /dev/tty 'vi' - - - - -
B<7> sh <(find ... |parallel -n5 echo vi)
7$ sh <(find ... |parallel -n5 echo vi)
B<7> find ... |parallel -n5 -uj1 vi
7$ find ... |parallel -n5 -uj1 vi
B<8> xapply -fn "" /etc/passwd
8$ xapply -fn "" /etc/passwd
B<8> parallel -k echo < /etc/passwd
8$ parallel -k echo < /etc/passwd
B<9> tr ':' '\012' < /etc/passwd | xapply -7 -nf 'chown %1 %6' - - - - - - -
9$ tr ':' '\012' < /etc/passwd | xapply -7 -nf 'chown %1 %6' - - - - - - -
B<9> tr ':' '\012' < /etc/passwd | parallel -N7 chown {1} {6}
9$ tr ':' '\012' < /etc/passwd | parallel -N7 chown {1} {6}
B<10> xapply '[ -d %1/RCS ] || echo %1' */
10$ xapply '[ -d %1/RCS ] || echo %1' */
B<10> parallel '[ -d {}/RCS ] || echo {}' ::: */
10$ parallel '[ -d {}/RCS ] || echo {}' ::: */
B<11> xapply -f '[ -f %1 ] && echo %1' List | ...
11$ xapply -f '[ -f %1 ] && echo %1' List | ...
B<11> parallel '[ -f {} ] && echo {}' < List | ...
11$ parallel '[ -f {} ] && echo {}' < List | ...
https://web.archive.org/web/20160702211113/
http://carrera.databits.net/~ksb/msrc/local/bin/xapply/xapply.html
@ -554,30 +563,32 @@ very much like GNU B<parallel>. B<apply> does not run jobs in
parallel. B<apply> does not use an argument separator (like B<:::>);
instead the template must be the first argument.
=head3 EXAMPLES FROM IBM's KNOWLEDGE CENTER
Here are the examples from IBM's Knowledge Center and the
corresponding command using GNU B<parallel>:
1. To obtain results similar to those of the B<ls> command, enter:
=head4 To obtain results similar to those of the B<ls> command, enter:
apply echo *
parallel echo ::: *
1$ apply echo *
1$ parallel echo ::: *
2. To compare the file named B<a1> to the file named B<b1>, and the
file named B<a2> to the file named B<b2>, enter:
=head4 To compare the file named a1 to the file named b1, and
the file named a2 to the file named b2, enter:
apply -2 cmp a1 b1 a2 b2
parallel -N2 cmp ::: a1 b1 a2 b2
2$ apply -2 cmp a1 b1 a2 b2
2$ parallel -N2 cmp ::: a1 b1 a2 b2
3. To run the B<who> command five times, enter:
=head4 To run the B<who> command five times, enter:
apply -0 who 1 2 3 4 5
parallel -N0 who ::: 1 2 3 4 5
3$ apply -0 who 1 2 3 4 5
3$ parallel -N0 who ::: 1 2 3 4 5
4. To link all files in the current directory to the directory
B</usr/joe>, enter:
=head4 To link all files in the current directory to the directory
/usr/joe, enter:
apply 'ln %1 /usr/joe' *
parallel ln {} /usr/joe ::: *
4$ apply 'ln %1 /usr/joe' *
4$ parallel ln {} /usr/joe ::: *
https://www-01.ibm.com/support/knowledgecenter/
ssw_aix_71/com.ibm.aix.cmds1/apply.htm (Last checked: 2019-01)
@ -593,46 +604,60 @@ output. This means you will have to write a wrapper for most programs.
B<paexec> has a job dependency facility so a job can depend on another
job to be executed successfully. Sort of a poor-man's B<make>.
=head3 EXAMPLES FROM paexec's EXAMPLE CATALOG
Here are the examples from B<paexec>'s example catalog with the equivalent
using GNU B<parallel>:
=over 1
=head4 1_div_X_run
=item 1_div_X_run:
1$ ../../paexec -s -l -c "`pwd`/1_div_X_cmd" -n +1 <<EOF [...]
../../paexec -s -l -c "`pwd`/1_div_X_cmd" -n +1 <<EOF [...]
parallel echo {} '|' `pwd`/1_div_X_cmd <<EOF [...]
1$ parallel echo {} '|' `pwd`/1_div_X_cmd <<EOF [...]
=item all_substr_run:
=head4 all_substr_run
../../paexec -lp -c "`pwd`/all_substr_cmd" -n +3 <<EOF [...]
parallel echo {} '|' `pwd`/all_substr_cmd <<EOF [...]
2$ ../../paexec -lp -c "`pwd`/all_substr_cmd" -n +3 <<EOF [...]
=item cc_wrapper_run:
2$ parallel echo {} '|' `pwd`/all_substr_cmd <<EOF [...]
../../paexec -c "env CC=gcc CFLAGS=-O2 `pwd`/cc_wrapper_cmd" \
=head4 cc_wrapper_run
3$ ../../paexec -c "env CC=gcc CFLAGS=-O2 `pwd`/cc_wrapper_cmd" \
-n 'host1 host2' \
-t '/usr/bin/ssh -x' <<EOF [...]
parallel echo {} '|' "env CC=gcc CFLAGS=-O2 `pwd`/cc_wrapper_cmd" \
-S host1,host2 <<EOF [...]
# This is not exactly the same, but avoids the wrapper
parallel gcc -O2 -c -o {.}.o {} \
3$ parallel echo {} '|' "env CC=gcc CFLAGS=-O2 `pwd`/cc_wrapper_cmd" \
-S host1,host2 <<EOF [...]
=item toupper_run:
# This is not exactly the same, but avoids the wrapper
parallel gcc -O2 -c -o {.}.o {} \
-S host1,host2 <<EOF [...]
../../paexec -lp -c "`pwd`/toupper_cmd" -n +10 <<EOF [...]
parallel echo {} '|' ./toupper_cmd <<EOF [...]
# Without the wrapper:
parallel echo {} '| awk {print\ toupper\(\$0\)}' <<EOF [...]
=head4 toupper_run
=back
4$ ../../paexec -lp -c "`pwd`/toupper_cmd" -n +10 <<EOF [...]
4$ parallel echo {} '|' ./toupper_cmd <<EOF [...]
# Without the wrapper:
parallel echo {} '| awk {print\ toupper\(\$0\)}' <<EOF [...]
https://github.com/cheusov/paexec
=head2 DIFFERENCES BETWEEN map(sitaramc) AND GNU Parallel
Summary table (see legend above):
I1 - - I4 - - -
M1 (M2) M3 M4 M5 -
- O2 O3 - O5 - - N/A N/A O10
E1 - - - - - -
- - - - - - - - -
- -
(M2): Only if there is a single replacement string.
B<map> sees it as a feature to have less features and in doing so it
also handles corner cases incorrectly. A lot of GNU B<parallel>'s code
is to handle corner cases correctly on every platform, so you will not
@ -707,8 +732,14 @@ context replace:
B<map> requires Perl v5.10.0 making it harder to use on old systems.
B<map> has no way of using % in the command (GNU B<parallel> has -I to
specify another replacement string than B<{}>).
To put a % in the command line B<map> requires you to use %%:
seq 10 | map -n1 echo % +10%% {}
seq 10 | parallel -I ,, echo ,, +10% {}
GNU B<parallel> has -I to specify another replacement string than
B<{}> if you use B<{}> in the command template.
By design B<map> is option incompatible with B<xargs>, it does not
have remote job execution, a structured way of saving results,
@ -717,7 +748,7 @@ delimiter (only field delimiter), logging of jobs run with possibility
to resume, keeping the output in the same order as input, --pipe
processing, and dynamically timeouts.
https://github.com/sitaramc/map
https://github.com/sitaramc/map (Last checked: 2020-04)
=head2 DIFFERENCES BETWEEN ladon AND GNU Parallel
@ -753,21 +784,25 @@ fails for output larger than 200k:
It is assumed that the '--rpl's above are put in B<~/.parallel/config>
and that it is run under a shell that supports '**' globbing (such as B<zsh>):
B<1> ladon "**/*.txt" -- echo RELPATH
1$ ladon "**/*.txt" -- echo RELPATH
B<1> parallel echo RELPATH ::: **/*.txt
1$ parallel echo RELPATH ::: **/*.txt
B<2> ladon "~/Documents/**/*.pdf" -- shasum FULLPATH >hashes.txt
2$ ladon "~/Documents/**/*.pdf" -- shasum FULLPATH >hashes.txt
B<2> parallel shasum FULLPATH ::: ~/Documents/**/*.pdf >hashes.txt
2$ parallel shasum FULLPATH ::: ~/Documents/**/*.pdf >hashes.txt
B<3> ladon -m thumbs/RELDIR "**/*.jpg" -- convert FULLPATH -thumbnail 100x100^ -gravity center -extent 100x100 thumbs/RELPATH
3$ ladon -m thumbs/RELDIR "**/*.jpg" -- convert FULLPATH \
-thumbnail 100x100^ -gravity center -extent 100x100 \
thumbs/RELPATH
B<3> parallel mkdir -p thumbs/RELDIR\; convert FULLPATH -thumbnail 100x100^ -gravity center -extent 100x100 thumbs/RELPATH ::: **/*.jpg
3$ parallel mkdir -p thumbs/RELDIR\; convert FULLPATH
-thumbnail 100x100^ -gravity center -extent 100x100 \
thumbs/RELPATH ::: **/*.jpg
B<4> ladon "~/Music/*.wav" -- lame -V 2 FULLPATH DIRNAME/BASENAME.mp3
4$ ladon "~/Music/*.wav" -- lame -V 2 FULLPATH DIRNAME/BASENAME.mp3
B<4> parallel lame -V 2 FULLPATH DIRNAME/BASENAME.mp3 ::: ~/Music/*.wav
4$ parallel lame -V 2 FULLPATH DIRNAME/BASENAME.mp3 ::: ~/Music/*.wav
https://github.com/danielgtaylor/ladon (Last checked: 2019-01)
@ -802,21 +837,21 @@ jobs. This can be emulated by GNU B<parallel> using B<bash>'s B<ulimit>:
=head3 EXAMPLES FROM jobflow README
B<1> cat things.list | jobflow -threads=8 -exec ./mytask {}
1$ cat things.list | jobflow -threads=8 -exec ./mytask {}
B<1> cat things.list | parallel -j8 ./mytask {}
1$ cat things.list | parallel -j8 ./mytask {}
B<2> seq 100 | jobflow -threads=100 -exec echo {}
2$ seq 100 | jobflow -threads=100 -exec echo {}
B<2> seq 100 | parallel -j100 echo {}
2$ seq 100 | parallel -j100 echo {}
B<3> cat urls.txt | jobflow -threads=32 -exec wget {}
3$ cat urls.txt | jobflow -threads=32 -exec wget {}
B<3> cat urls.txt | parallel -j32 wget {}
3$ cat urls.txt | parallel -j32 wget {}
B<4> find . -name '*.bmp' | jobflow -threads=8 -exec bmp2jpeg {.}.bmp {.}.jpg
4$ find . -name '*.bmp' | jobflow -threads=8 -exec bmp2jpeg {.}.bmp {.}.jpg
B<4> find . -name '*.bmp' | parallel -j8 bmp2jpeg {.}.bmp {.}.jpg
4$ find . -name '*.bmp' | parallel -j8 bmp2jpeg {.}.bmp {.}.jpg
https://github.com/rofl0r/jobflow
@ -836,15 +871,15 @@ is killed.
Output to stderr (standard error) is changed if the command fails.
Here are the two examples from B<gargs> website.
=head3 EXAMPLES FROM gargs WEBSITE
B<1> seq 12 -1 1 | gargs -p 4 -n 3 "sleep {0}; echo {1} {2}"
1$ seq 12 -1 1 | gargs -p 4 -n 3 "sleep {0}; echo {1} {2}"
B<1> seq 12 -1 1 | parallel -P 4 -n 3 "sleep {1}; echo {2} {3}"
1$ seq 12 -1 1 | parallel -P 4 -n 3 "sleep {1}; echo {2} {3}"
B<2> cat t.txt | gargs --sep "\s+" -p 2 "echo '{0}:{1}-{2}' full-line: \'{}\'"
2$ cat t.txt | gargs --sep "\s+" -p 2 "echo '{0}:{1}-{2}' full-line: \'{}\'"
B<2> cat t.txt | parallel --colsep "\\s+" -P 2 "echo '{1}:{2}-{3}' full-line: \'{}\'"
2$ cat t.txt | parallel --colsep "\\s+" -P 2 "echo '{1}:{2}-{3}' full-line: \'{}\'"
https://github.com/brentp/gargs
@ -966,11 +1001,11 @@ instead of ^ as that is closer to bash's ${var%postfix}):
--rpl '{/:} s:(.*/)?([^/.]+)(\.[^/]+)*$:$2:'
--rpl '{@(.*?)} /$$1/ and $_=$1;'
=head3 EXAMPLES FROM rush's WEBSITE
Here are the examples from B<rush>'s website with the equivalent
command in GNU B<parallel>.
=head3 EXAMPLES
B<1. Simple run, quoting is not necessary>
$ seq 1 3 | rush echo {}
@ -1518,6 +1553,8 @@ Make Ctrl-C kill all running processes
=back
=head3 EXAMPLES FROM maps WEBSITE
Here are the 5 examples converted to GNU Parallel:
1$ ls *.c | map f 'foo $f'
@ -1556,6 +1593,8 @@ B<loop> cannot run functions:
export -f myfunc
loop 'myfunc this fails'
=head3 EXAMPLES FROM loop's WEBSITE
Some of the examples from https://github.com/Miserlou/Loop/ can be
emulated with GNU B<parallel>:
@ -1760,48 +1799,50 @@ to fit on a single command line.
B<paral> has no support for running jobs remotely.
=head3 EXAMPLES FROM README.markdown
The examples from B<README.markdown> and the corresponding command run
with GNU B<parallel> (B<--results
'out_{#}_{=s/[^\sa-z_0-9]//g;s/\s+/_/g=}.log' --eta> is omitted from
the GNU B<parallel> command):
paral "command 1" "command 2 --flag" "command arg1 arg2"
parallel ::: "command 1" "command 2 --flag" "command arg1 arg2"
1$ paral "command 1" "command 2 --flag" "command arg1 arg2"
1$ parallel ::: "command 1" "command 2 --flag" "command arg1 arg2"
paral "sleep 1 && echo c1" "sleep 2 && echo c2" \
"sleep 3 && echo c3" "sleep 4 && echo c4" "sleep 5 && echo c5"
parallel ::: "sleep 1 && echo c1" "sleep 2 && echo c2" \
"sleep 3 && echo c3" "sleep 4 && echo c4" "sleep 5 && echo c5"
# Or shorter:
parallel "sleep {} && echo c{}" ::: {1..5}
2$ paral "sleep 1 && echo c1" "sleep 2 && echo c2" \
"sleep 3 && echo c3" "sleep 4 && echo c4" "sleep 5 && echo c5"
2$ parallel ::: "sleep 1 && echo c1" "sleep 2 && echo c2" \
"sleep 3 && echo c3" "sleep 4 && echo c4" "sleep 5 && echo c5"
# Or shorter:
parallel "sleep {} && echo c{}" ::: {1..5}
paral -n=0 "sleep 5 && echo c5" "sleep 4 && echo c4" \
"sleep 3 && echo c3" "sleep 2 && echo c2" "sleep 1 && echo c1"
parallel ::: "sleep 5 && echo c5" "sleep 4 && echo c4" \
"sleep 3 && echo c3" "sleep 2 && echo c2" "sleep 1 && echo c1"
# Or shorter:
parallel -j0 "sleep {} && echo c{}" ::: 5 4 3 2 1
3$ paral -n=0 "sleep 5 && echo c5" "sleep 4 && echo c4" \
"sleep 3 && echo c3" "sleep 2 && echo c2" "sleep 1 && echo c1"
3$ parallel ::: "sleep 5 && echo c5" "sleep 4 && echo c4" \
"sleep 3 && echo c3" "sleep 2 && echo c2" "sleep 1 && echo c1"
# Or shorter:
parallel -j0 "sleep {} && echo c{}" ::: 5 4 3 2 1
paral -n=1 "sleep 5 && echo c5" "sleep 4 && echo c4" \
"sleep 3 && echo c3" "sleep 2 && echo c2" "sleep 1 && echo c1"
parallel -j1 "sleep {} && echo c{}" ::: 5 4 3 2 1
4$ paral -n=1 "sleep 5 && echo c5" "sleep 4 && echo c4" \
"sleep 3 && echo c3" "sleep 2 && echo c2" "sleep 1 && echo c1"
4$ parallel -j1 "sleep {} && echo c{}" ::: 5 4 3 2 1
paral -n=2 "sleep 5 && echo c5" "sleep 4 && echo c4" \
"sleep 3 && echo c3" "sleep 2 && echo c2" "sleep 1 && echo c1"
parallel -j2 "sleep {} && echo c{}" ::: 5 4 3 2 1
5$ paral -n=2 "sleep 5 && echo c5" "sleep 4 && echo c4" \
"sleep 3 && echo c3" "sleep 2 && echo c2" "sleep 1 && echo c1"
5$ parallel -j2 "sleep {} && echo c{}" ::: 5 4 3 2 1
paral -n=5 "sleep 5 && echo c5" "sleep 4 && echo c4" \
"sleep 3 && echo c3" "sleep 2 && echo c2" "sleep 1 && echo c1"
parallel -j5 "sleep {} && echo c{}" ::: 5 4 3 2 1
6$ paral -n=5 "sleep 5 && echo c5" "sleep 4 && echo c4" \
"sleep 3 && echo c3" "sleep 2 && echo c2" "sleep 1 && echo c1"
6$ parallel -j5 "sleep {} && echo c{}" ::: 5 4 3 2 1
paral -n=1 "echo a && sleep 0.5 && echo b && sleep 0.5 && \
echo c && sleep 0.5 && echo d && sleep 0.5 && \
echo e && sleep 0.5 && echo f && sleep 0.5 && \
echo g && sleep 0.5 && echo h"
parallel ::: "echo a && sleep 0.5 && echo b && sleep 0.5 && \
echo c && sleep 0.5 && echo d && sleep 0.5 && \
echo e && sleep 0.5 && echo f && sleep 0.5 && \
echo g && sleep 0.5 && echo h"
7$ paral -n=1 "echo a && sleep 0.5 && echo b && sleep 0.5 && \
echo c && sleep 0.5 && echo d && sleep 0.5 && \
echo e && sleep 0.5 && echo f && sleep 0.5 && \
echo g && sleep 0.5 && echo h"
7$ parallel ::: "echo a && sleep 0.5 && echo b && sleep 0.5 && \
echo c && sleep 0.5 && echo d && sleep 0.5 && \
echo e && sleep 0.5 && echo f && sleep 0.5 && \
echo g && sleep 0.5 && echo h"
https://github.com/amattn/paral (Last checked: 2019-01)
@ -1811,19 +1852,21 @@ https://github.com/amattn/paral (Last checked: 2019-01)
B<concurr> is built to run jobs in parallel using a client/server
model.
=head3 EXAMPLES FROM README.md
The examples from B<README.md>:
concurr 'echo job {#} on slot {%}: {}' : arg1 arg2 arg3 arg4
parallel 'echo job {#} on slot {%}: {}' ::: arg1 arg2 arg3 arg4
1$ concurr 'echo job {#} on slot {%}: {}' : arg1 arg2 arg3 arg4
1$ parallel 'echo job {#} on slot {%}: {}' ::: arg1 arg2 arg3 arg4
concurr 'echo job {#} on slot {%}: {}' :: file1 file2 file3
parallel 'echo job {#} on slot {%}: {}' :::: file1 file2 file3
2$ concurr 'echo job {#} on slot {%}: {}' :: file1 file2 file3
2$ parallel 'echo job {#} on slot {%}: {}' :::: file1 file2 file3
concurr 'echo {}' < input_file
parallel 'echo {}' < input_file
3$ concurr 'echo {}' < input_file
3$ parallel 'echo {}' < input_file
cat file | concurr 'echo {}'
cat file | parallel 'echo {}'
4$ cat file | concurr 'echo {}'
4$ cat file | parallel 'echo {}'
B<concurr> deals badly empty input files and with output larger than
64 KB.
@ -1857,37 +1900,45 @@ https://github.com/spion/npm-parallel (Last checked: 2019-01)
=head2 DIFFERENCES BETWEEN machma AND GNU Parallel
B<machma> runs tasks in parallel. It gives time stamped
output. It buffers in RAM. The examples from README.md:
output. It buffers in RAM.
# Put shorthand for timestamp in config for the examples
echo '--rpl '\
\''{time} $_=::strftime("%Y-%m-%d %H:%M:%S",localtime())'\' \
> ~/.parallel/machma
echo '--line-buffer --tagstring "{#} {time} {}"' >> ~/.parallel/machma
=head3 EXAMPLES FROM README.md
find . -iname '*.jpg' |
machma -- mogrify -resize 1200x1200 -filter Lanczos {}
find . -iname '*.jpg' |
parallel --bar -Jmachma mogrify -resize 1200x1200 -filter Lanczos {}
The examples from README.md:
cat /tmp/ips | machma -p 2 -- ping -c 2 -q {}
cat /tmp/ips | parallel -j2 -Jmachma ping -c 2 -q {}
1$ # Put shorthand for timestamp in config for the examples
echo '--rpl '\
\''{time} $_=::strftime("%Y-%m-%d %H:%M:%S",localtime())'\' \
> ~/.parallel/machma
echo '--line-buffer --tagstring "{#} {time} {}"' \
>> ~/.parallel/machma
cat /tmp/ips |
machma -- sh -c 'ping -c 2 -q $0 > /dev/null && echo alive' {}
cat /tmp/ips |
parallel -Jmachma 'ping -c 2 -q {} > /dev/null && echo alive'
2$ find . -iname '*.jpg' |
machma -- mogrify -resize 1200x1200 -filter Lanczos {}
find . -iname '*.jpg' |
parallel --bar -Jmachma mogrify -resize 1200x1200 \
-filter Lanczos {}
find . -iname '*.jpg' |
machma --timeout 5s -- mogrify -resize 1200x1200 -filter Lanczos {}
find . -iname '*.jpg' |
parallel --timeout 5s --bar mogrify -resize 1200x1200 \
-filter Lanczos {}
3$ cat /tmp/ips | machma -p 2 -- ping -c 2 -q {}
3$ cat /tmp/ips | parallel -j2 -Jmachma ping -c 2 -q {}
find . -iname '*.jpg' -print0 |
machma --null -- mogrify -resize 1200x1200 -filter Lanczos {}
find . -iname '*.jpg' -print0 |
parallel --null --bar mogrify -resize 1200x1200 -filter Lanczos {}
4$ cat /tmp/ips |
machma -- sh -c 'ping -c 2 -q $0 > /dev/null && echo alive' {}
4$ cat /tmp/ips |
parallel -Jmachma 'ping -c 2 -q {} > /dev/null && echo alive'
5$ find . -iname '*.jpg' |
machma --timeout 5s -- mogrify -resize 1200x1200 \
-filter Lanczos {}
5$ find . -iname '*.jpg' |
parallel --timeout 5s --bar mogrify -resize 1200x1200 \
-filter Lanczos {}
6$ find . -iname '*.jpg' -print0 |
machma --null -- mogrify -resize 1200x1200 -filter Lanczos {}
6$ find . -iname '*.jpg' -print0 |
parallel --null --bar mogrify -resize 1200x1200 \
-filter Lanczos {}
https://github.com/fd0/machma (Last checked: 2019-06)
@ -1909,6 +1960,8 @@ B<interface> does not buffer output, so output from different jobs mixes.
The overhead for each target is O(n*n), so with 1000 targets it
becomes very slow with an overhead in the order of 500ms/target.
=head3 EXAMPLES FROM interlace's WEBSITE
Using B<prips> most of the examples from
https://github.com/codingo/Interlace can be run with GNU B<parallel>:
@ -2319,17 +2372,18 @@ For full support of {n..m:s} including negative numbers use a dynamic
replacement string like this:
PARALLEL=--rpl\ \''{r((-?\d+)?)\.\.((-?\d+)?)((:([^}]*))?)}
PARALLEL=--rpl\ \''{r((-?\d+)?)\.\.((-?\d+)?)((:([^}]*))?)}
$a = defined $$2 ? $$2 < 0 ? 1+$#arg+$$2 : $$2 : 1;
$b = defined $$4 ? $$4 < 0 ? 1+$#arg+$$4 : $$4 : $#arg+1;
$s = defined $$6 ? $$7 : " ";
$_ = join $s,@arg[$a..$b]'\'
export PARALLEL
export PARALLEL
You can then do:
head /etc/passwd | parallel --colsep : echo ..={1r..} ..3={1r..3} 4..={1r4..} 2..4={1r2..4} 3..3={1r3..3} ..3:-={1r..3:-} ..3:/={1r..3:/} -1={-1} -5={-5} -6={-6} -3..={1r-3..}
head /etc/passwd | parallel --colsep : echo ..={1r..} ..3={1r..3} \
4..={1r4..} 2..4={1r2..4} 3..3={1r3..3} ..3:-={1r..3:-} \
..3:/={1r..3:/} -1={-1} -5={-5} -6={-6} -3..={1r-3..}
=head3 EXAMPLES FROM rargs MANUAL
@ -2370,6 +2424,7 @@ virtual memory will cause the machine to crash.
https://github.com/voodooEntity/threader (Last checked: 2020-04)
=head2 DIFFERENCES BETWEEN runp AND GNU Parallel
Summary table (see legend above):
@ -2496,9 +2551,53 @@ output to stderr (this can be prevented with -q)
https://github.com/jreisinger/runp (Last checked: 2020-04)
=head2 Todo
=head2 DIFFERENCES BETWEEN papply AND GNU Parallel
https://pypi.org/project/papply/
Summary table (see legend above):
- - - I4 - - -
M1 - M3 - - M6
- - O3 - O5 - - N/A N/A O10
E1 - - E4 - - -
- - - - - - - - -
- -
B<papply> does not print the output if the command fails:
$ papply 'echo %F; false' foo
"echo foo; false" did not succeed
B<papply>'s replacement strings (%F %d %f %n %e %z) can be simulated in GNU
B<parallel> by putting this in B<~/.parallel/config>:
--rpl '%F'
--rpl '%d $_=Q(::dirname($_));'
--rpl '%f s:.*/::;'
--rpl '%n s:.*/::;s:\.[^/.]+$::;'
--rpl '%e s:.*\.:.:'
--rpl '%z $_=""'
B<papply> buffers in RAM, and uses twice the amount of output. So
output of 5 GB takes 10 GB RAM.
The buffering is very CPU intensive: Buffering a line of 5 GB takes 40
seconds (compared to 10 seconds with GNU B<parallel>).
=head3 Examples as GNU Parallel
1$ papply gzip *.txt
1$ parallel gzip ::: *.txt
2$ papply "convert %F %n.jpg" *.png
2$ parallel convert {} {.}.jpg ::: *.png
https://pypi.org/project/papply/ (Last checked: 2020-04)
=head2 Todo
https://github.com/JeiKeiLim/simple_distribute_job

View file

@ -574,7 +574,7 @@ $Global::Initfile && unlink $Global::Initfile;
exit ($err);
sub parse_options {
$Global::version = 20200422;
$Global::version = 20200423;
$Global::progname = 'sql';
# This must be done first as this may exec myself

View file

@ -135,6 +135,12 @@ par_remote_symlink_dir() {
ssh parallel@lo rm wd && echo OK: wd is still a symlink with PARALLEL_RSYNC_OPTS
}
par_sshlogin_replacement() {
echo '### show {sshlogin} and {host}'
parallel -S $SSHLOGIN1 --plus echo {sshlogin} {} {host} ::: and
parallel -S '5//usr/bin/ssh '$SSHLOGIN1 --plus echo {sshlogin} {} {host} ::: and
}
export -f $(compgen -A function | grep par_)
#compgen -A function | grep par_ | sort | parallel --delay $D -j$P --tag -k '{} 2>&1'
compgen -A function | grep par_ | sort |

View file

@ -72,7 +72,7 @@ par_remote_load parallel: Warning: This job was killed because it timed out:
par_remote_load parallel: Warning: perl -e 'while(1){ }'
par_remote_load OK
par_remote_nice ### Test --nice remote
par_remote_nice ssh lo -- exec 'perl -e '"'"'$ENV{"PARALLEL_PID"}="XXXXX";$ENV{"PARALLEL_SEQ"}="1";$ENV{"PARALLEL_SSHLOGIN"}="lo";$bashfunc = "";@ARGV="PAR=a bash -c \"echo \\\$PAR b\"";$shell="$ENV{SHELL}";$tmpdir="/tmp/parallel-local-ssh1-tmpdir";$nice=1;do{$ENV{PARALLEL_TMP}=$tmpdir."/par".join"",map{(0..9,"a".."z","A".."Z")[rand(62)]}(1..5);}while(-e$ENV{PARALLEL_TMP});$SIG{CHLD}=sub{$done=1;};$pid=fork;unless($pid){eval{setpgrp};eval{setpriority(0,0,$nice)};exec$shell,"-c",($bashfunc."@ARGV");die"exec:$!\n";}do{$s=$s<1?0.001+$s*1.03:$s;select(undef,undef,undef,$s);}until($done||getppid==1);kill(SIGHUP,-${pid})unless$done;wait;exit($?&127?128+($?&127):1+$?>>8)'"'";
par_remote_nice ssh lo -- exec 'perl -e '"'"'$ENV{"PARALLEL_PID"}="XXXXX";$ENV{"PARALLEL_SEQ"}="1";$ENV{"PARALLEL_SSHLOGIN"}="lo";$ENV{"PARALLEL_SSHHOST"}="lo";$bashfunc = "";@ARGV="PAR=a bash -c \"echo \\\$PAR b\"";$shell="$ENV{SHELL}";$tmpdir="/tmp/parallel-local-ssh1-tmpdir";$nice=1;do{$ENV{PARALLEL_TMP}=$tmpdir."/par".join"",map{(0..9,"a".."z","A".."Z")[rand(62)]}(1..5);}while(-e$ENV{PARALLEL_TMP});$SIG{CHLD}=sub{$done=1;};$pid=fork;unless($pid){eval{setpgrp};eval{setpriority(0,0,$nice)};exec$shell,"-c",($bashfunc."@ARGV");die"exec:$!\n";}do{$s=$s<1?0.001+$s*1.03:$s;select(undef,undef,undef,$s);}until($done||getppid==1);kill(SIGHUP,-${pid})unless$done;wait;exit($?&127?128+($?&127):1+$?>>8)'"'";
par_remote_nice a b
par_ssh ### use --ssh
par_ssh Run through BARSSH?

View file

@ -26,6 +26,9 @@ par_read_sshloginfile_from_stdin foo
par_remote_symlink_dir bug #51293: parallel does not preserve symlinked directory structure on remote
par_remote_symlink_dir OK: wd is still a symlink with --rsync-opts
par_remote_symlink_dir OK: wd is still a symlink with PARALLEL_RSYNC_OPTS
par_sshlogin_replacement ### show {sshlogin} and {host}
par_sshlogin_replacement parallel@lo and lo
par_sshlogin_replacement /usr/bin/ssh parallel@lo and lo
par_test_nonall ### Test --nonall
par_test_nonall /home/csh
par_test_nonall /home/parallel