tange/parallel
1
0
Fork 0
mirror of https://git.savannah.gnu.org/git/parallel.git synced 2024-12-22 20:57:53 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

parallel: Fixed: --null conflicts with --resume --joblog.

Browse source
This commit is contained in:
Ole Tange 2019-03-17 23:07:32 +01:00
parent 8ee7777ee9
commit fe4173927f
24 changed files with 396 additions and 177 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
configure vendored
View file

@ -1,6 +1,6 @@
#! /bin/sh
# Guess values for system-dependent variables and create Makefiles.
# Generated by GNU Autoconf 2.69 for parallel 20190222.
# Generated by GNU Autoconf 2.69 for parallel 20190310.
#
# Report bugs to <bug-parallel@gnu.org>.
#
@ -579,8 +579,8 @@ MAKEFLAGS=
# Identity of this package.
PACKAGE_NAME='parallel'
PACKAGE_TARNAME='parallel'
PACKAGE_VERSION='20190222'
PACKAGE_STRING='parallel 20190222'
PACKAGE_VERSION='20190310'
PACKAGE_STRING='parallel 20190310'
PACKAGE_BUGREPORT='bug-parallel@gnu.org'
PACKAGE_URL=''
@ -1214,7 +1214,7 @@ if test "$ac_init_help" = "long"; then
# Omit some internal or obsolete options to make the list less imposing.
# This message is too long to be a string in the A/UX 3.1 sh.
cat <<_ACEOF
\`configure' configures parallel 20190222 to adapt to many kinds of systems.
\`configure' configures parallel 20190310 to adapt to many kinds of systems.
Usage: $0 [OPTION]... [VAR=VALUE]...
@ -1281,7 +1281,7 @@ fi
if test -n "$ac_init_help"; then
case $ac_init_help in
short | recursive ) echo "Configuration of parallel 20190222:";;
short | recursive ) echo "Configuration of parallel 20190310:";;
esac
cat <<\_ACEOF
@ -1357,7 +1357,7 @@ fi
test -n "$ac_init_help" && exit $ac_status
if $ac_init_version; then
cat <<\_ACEOF
parallel configure 20190222
parallel configure 20190310
generated by GNU Autoconf 2.69
Copyright (C) 2012 Free Software Foundation, Inc.
@ -1374,7 +1374,7 @@ cat >config.log <<_ACEOF
This file contains any messages produced by compilers while
running configure, to aid debugging if configure makes a mistake.
It was created by parallel $as_me 20190222, which was
It was created by parallel $as_me 20190310, which was
generated by GNU Autoconf 2.69. Invocation command line was
$ $0 $@
@ -2237,7 +2237,7 @@ fi
# Define the identity of the package.
PACKAGE='parallel'
VERSION='20190222'
VERSION='20190310'
cat >>confdefs.h <<_ACEOF
@ -2880,7 +2880,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
# report actual input values of CONFIG_FILES etc. instead of their
# values after options handling.
ac_log="
This file was extended by parallel $as_me 20190222, which was
This file was extended by parallel $as_me 20190310, which was
generated by GNU Autoconf 2.69. Invocation command line was
CONFIG_FILES = $CONFIG_FILES
@ -2942,7 +2942,7 @@ _ACEOF
cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
ac_cs_version="\\
parallel config.status 20190222
parallel config.status 20190310
configured by $0, generated by GNU Autoconf 2.69,
with options \\"\$ac_cs_config\\"

2
configure.ac
View file

@ -1,4 +1,4 @@
AC_INIT([parallel], [20190222], [bug-parallel@gnu.org])
AC_INIT([parallel], [20190310], [bug-parallel@gnu.org])
AM_INIT_AUTOMAKE([-Wall -Werror foreign])
AC_CONFIG_HEADERS([config.h])
AC_CONFIG_FILES([

7
doc/haikus
View file

@ -1,9 +1,10 @@
Quote of the month:
There are so many things to love about GNU parallel. You could honestly teach a whole parallel computing course with it and never have to leave it for a real language.
-- Aubrey Bailey @DNAvinci@twitter
Parallel is a life send, what an awesome piece of software.
-- Blaze9@reddit.com
Y'all need some GNU parallel in your lives
-- ChaKu @ChaiLovesChai@twitter

7
doc/release_new_version
View file

@ -207,9 +207,10 @@ from:tange@gnu.org
to:parallel@gnu.org, bug-parallel@gnu.org
stable-bcc: Jesse Alama <jessealama@fastmail.fm>
Subject: GNU Parallel 20190222 ('indien pakistan Kashmir') released <<[stable]>>
Subject: GNU Parallel 20190322 (climatestrike FridayforFuture
Christchurch 'Max 8' fly i etiopien 737Max 8 indien pakistan Kashmir') released <<[stable]>>
GNU Parallel 20190222 ('') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/
GNU Parallel 20190322 ('') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/
<<No new functionality was introduced so this is a good candidate for a stable release.>>
@ -220,7 +221,7 @@ Quote of the month:
New in this release:
* GNU Parallel now includes a cheat sheet:
* GNU Parallel now includes a cheat sheet: parallel_cheat.pdf
https://calendar.colorado.edu/event/high_throughput_computing_on_rmacc_summit_and_beyond#.XH2NBhB7mV4

2
src/Makefile.am
View file

@ -288,4 +288,4 @@ EXTRA_DIST = parallel sem sql niceload parcat parset env_parallel \
env_parallel.tcsh env_parallel.zsh parcat.pod parset.pod \
sem.pod parallel.pod env_parallel.pod niceload.pod \
parallel_tutorial.pod parallel_book.pod parallel_design.pod \
parallel_alternatives.pod $(DISTCLEANFILES)
parallel_alternatives.pod parallel_cheat.fodt $(DISTCLEANFILES)

2
src/Makefile.in
View file

@ -275,7 +275,7 @@ EXTRA_DIST = parallel sem sql niceload parcat parset env_parallel \
env_parallel.tcsh env_parallel.zsh parcat.pod parset.pod \
sem.pod parallel.pod env_parallel.pod niceload.pod \
parallel_tutorial.pod parallel_book.pod parallel_design.pod \
parallel_alternatives.pod $(DISTCLEANFILES)
parallel_alternatives.pod parallel_cheat.fodt $(DISTCLEANFILES)
all: all-am

1
src/env_parallel.ash
View file

@ -186,6 +186,7 @@ env_parallel() {
# myfunc is a shell function (in zsh)
# which is /usr/bin/which (in sh, bash)
# which is hashed (/usr/bin/which)
# gi is aliased to `grep -i' (in bash)
# aliased to `alias | /usr/bin/which --tty-only --read-alias --show-dot --show-tilde'
# Return 0 if found, 1 otherwise
LANG=C type "$@" |

5
src/env_parallel.bash
View file

@ -173,11 +173,12 @@ env_parallel() {
# type returns:
# ll is an alias for ls -l (in ash)
# bash is a tracked alias for /bin/bash
# true is a shell builtin
# true is a shell builtin (in bash)
# myfunc is a function (in bash)
# myfunc is a shell function (in zsh)
# which is /usr/bin/which
# which is /usr/bin/which (in sh, bash)
# which is hashed (/usr/bin/which)
# gi is aliased to `grep -i' (in bash)
# aliased to `alias | /usr/bin/which --tty-only --read-alias --show-dot --show-tilde'
# Return 0 if found, 1 otherwise
LANG=C type "$@" |

1
src/env_parallel.dash
View file

@ -186,6 +186,7 @@ env_parallel() {
# myfunc is a shell function (in zsh)
# which is /usr/bin/which (in sh, bash)
# which is hashed (/usr/bin/which)
# gi is aliased to `grep -i' (in bash)
# aliased to `alias | /usr/bin/which --tty-only --read-alias --show-dot --show-tilde'
# Return 0 if found, 1 otherwise
LANG=C type "$@" |

55
src/parallel
View file

@ -2009,6 +2009,8 @@ sub open_joblog() {
if($opt::resume || $opt::resume_failed || $opt::retry_failed) {
if(open(my $joblog_fh, "<", $opt::joblog)) {
# Read the joblog
# Override $/ with \n because -d might be set
local $/ = "\n";
# If there is a header: Open as append later
$append = <$joblog_fh>;
my $joblog_regexp;
@ -2017,22 +2019,19 @@ sub open_joblog() {
# 4 host 1360490623.067 3.445 1023 1222 0 0 command
$joblog_regexp='^(\d+)(?:\t[^\t]+){5}\t0\t0\t';
my @group;
{
local $/ = "\n";
while(<$joblog_fh>) {
if(/$joblog_regexp/o) {
# This is 30% faster than set_job_already_run($1);
vec($Global::job_already_run,($1||0),1) = 1;
$Global::total_completed++;
$group[$1-1] = "true";
} elsif(/(\d+)\s+\S+(\s+[-0-9.]+){6}\s+(.*)$/) {
# Grab out the command
$group[$1-1] = $3;
} else {
chomp;
::error("Format of '$opt::joblog' is wrong: $_");
::wait_and_exit(255);
}
while(<$joblog_fh>) {
if(/$joblog_regexp/o) {
# This is 30% faster than set_job_already_run($1);
vec($Global::job_already_run,($1||0),1) = 1;
$Global::total_completed++;
$group[$1-1] = "true";
} elsif(/(\d+)\s+\S+(\s+[-0-9.]+){6}\s+(.*)$/) {
# Grab out the command
$group[$1-1] = $3;
} else {
chomp;
::error("Format of '$opt::joblog' is wrong: $_");
::wait_and_exit(255);
}
}
if(@group) {
@ -2077,6 +2076,8 @@ sub open_joblog() {
}
close $joblog_fh;
}
# $opt::null may be set if the commands contain \n
if($opt::null) { $/ = "\0"; }
}
if($opt::dryrun) {
# Do not write to joblog in a dry-run
@ -2426,9 +2427,7 @@ sub shell_quote(@) {
# @strings = strings to be quoted
# Returns:
# @shell_quoted_strings = string quoted as needed by the shell
return wantarray ?
(map { Q($_) } @_)
: (join" ",map { Q($_) } @_);
return wantarray ? (map { Q($_) } @_) : (join" ",map { Q($_) } @_);
}
sub shell_quote_scalar_rc($) {
@ -4190,7 +4189,7 @@ sub start_no_new_jobs() {
::status
("$Global::progname: SIGTERM received. No new jobs will be started.",
"$Global::progname: Waiting for these ".(keys %Global::running).
" jobs to finish. Send SIGTERM again to stop now.");
" jobs to finish. Send SIGTERM to stop now.");
list_running_jobs();
$Global::start_no_new_jobs ||= 1;
}
@ -4569,11 +4568,14 @@ sub citation() {
"",
# Before changing this line, please read
# https://www.gnu.org/software/parallel/parallel_design.html#Citation-notice
# https://git.savannah.gnu.org/cgit/parallel.git/tree/doc/citation-notice-faq.txt
"This helps funding further development; AND IT WON'T COST YOU A CENT.",
"If you pay 10000 EUR you should feel free to use GNU Parallel without citing.",
"",
"More about funding GNU Parallel and the citation notice:",
"https://lists.gnu.org/archive/html/parallel/2013-11/msg00006.html",
"https://www.gnu.org/software/parallel/parallel_design.html#Citation-notice",
"https://git.savannah.gnu.org/cgit/parallel.git/tree/doc/citation-notice-faq.txt",
"",
"If you send a copy of your published article to tange\@gnu.org, it will be",
"mentioned in the release notes of next version of GNU Parallel.",
@ -4591,8 +4593,13 @@ sub citation() {
close $fh;
::status(
"",
"Thank you for your support. It is much appreciated. The citation",
"notice is now silenced.",
"Thank you for your support: You are the reason why there is funding to",
"continue maintaining GNU Parallel. On behalf of future versions of",
"GNU Parallel, which would not exist without your support:",
"",
" THANK YOU SO MUCH",
"",
"It is really appreciated. The citation notice is now silenced.",
"");
} else {
::status(
@ -11166,8 +11173,8 @@ sub new($$) {
my $fhs = shift;
for my $fh (@$fhs) {
if(-t $fh and -t ($Global::status_fd || *STDERR)) {
::warning("Input is read from the terminal. You either know what you",
"are doing (in which case: YOU ARE AWESOME!) or you forgot",
::warning("Input is read from the terminal. You are either an expert",
"(in which case: YOU ARE AWESOME!) or maybe you forgot",
"::: or :::: or -a or to pipe data into parallel. If so",
"consider going through the tutorial: man parallel_tutorial",
"Press CTRL-D to exit.");

96
src/parallel.pod
View file

@ -30,22 +30,23 @@ STOP!
Read the B<Reader's guide> below if you are new to GNU B<parallel>.
GNU B<parallel> is a shell tool for executing jobs in parallel using
one or more computers. A job can be a single command or a small
script that has to be run for each of the lines in the input. The
typical input is a list of files, a list of hosts, a list of users, a
list of URLs, or a list of tables. A job can also be a command that
reads from a pipe. GNU B<parallel> can then split the input into
blocks and pipe a block into each command in parallel.
one or more computers. A job can be a single command or a small script
that has to be run for each of the lines in the input. The typical
input is a list of files, a list of hosts, a list of users, a list of
URLs, or a list of tables. A job can also be a command that reads from
a pipe. GNU B<parallel> can then split the input into blocks and pipe
a block into each command in parallel.
If you use xargs and tee today you will find GNU B<parallel> very easy to
use as GNU B<parallel> is written to have the same options as xargs. If
you write loops in shell, you will find GNU B<parallel> may be able to
replace most of the loops and make them run faster by running several
jobs in parallel.
If you use xargs and tee today you will find GNU B<parallel> very easy
to use as GNU B<parallel> is written to have the same options as
xargs. If you write loops in shell, you will find GNU B<parallel> may
be able to replace most of the loops and make them run faster by
running several jobs in parallel.
GNU B<parallel> makes sure output from the commands is the same output as
you would get had you run the commands sequentially. This makes it
possible to use output from GNU B<parallel> as input for other programs.
GNU B<parallel> makes sure output from the commands is the same output
as you would get had you run the commands sequentially. This makes it
possible to use output from GNU B<parallel> as input for other
programs.
For each line of input GNU B<parallel> will execute I<command> with
the line as arguments. If no I<command> is given, the line of input is
@ -121,8 +122,8 @@ replacement string; then the string is not quoted.
Input line without extension. This replacement string will be replaced
by the input with the extension removed. If the input line contains
B<.> after the last B</>, the last B<.> until the end of the string will
be removed and B<{.}> will be replaced with the
B<.> after the last B</>, the last B<.> until the end of the string
will be removed and B<{.}> will be replaced with the
remaining. E.g. I<foo.jpg> becomes I<foo>, I<subdir/foo.jpg> becomes
I<subdir/foo>, I<sub.dir/foo.jpg> becomes I<sub.dir/foo>,
I<sub.dir/bar> remains I<sub.dir/bar>. If the input line does not
@ -522,7 +523,7 @@ this will run 10*5 = 50 jobs in total:
parallel --pipepart -a myfile --block -10 -j5 wc
This is an efficient alternative to B<--round-robin> because data is
This is an efficient alternative to B<--roundrobin> because data is
never read by GNU B<parallel>, but you can still have very few
jobslots process a large amount of data.
@ -1892,14 +1893,14 @@ with B<--sshlogin>.
=item B<--round>
Normally B<--pipe> will give a single block to each instance of the
command. With B<--round-robin> all blocks will at random be written to
command. With B<--roundrobin> all blocks will at random be written to
commands already running. This is useful if the command takes a long
time to initialize.
B<--keep-order> will not work with B<--round-robin> as it is
B<--keep-order> will not work with B<--roundrobin> as it is
impossible to track which input block corresponds to which output.
B<--round-robin> implies B<--pipe>, except if B<--pipepart> is given.
B<--roundrobin> implies B<--pipe>, except if B<--pipepart> is given.
=item B<--rpl> 'I<tag> I<perl expression>'
@ -2741,6 +2742,17 @@ FUBAR in all files in this dir and subdirs:
Note B<-q> is needed because of the space in 'FOO BAR'.
=head1 EXAMPLE: Simple network scanner
B<prips> can generate IP-addresses from CIDR notation. With GNU
B<parallel> you can build a simple network scanner to see which
addresses respond to B<ping>:
prips 130.229.16.0/20 | \
parallel --timeout 2 -j0 \
'ping -c 1 {} >/dev/null && echo {}' 2>/dev/null
=head1 EXAMPLE: Reading arguments from command line
GNU B<parallel> can take the arguments from command line instead of
@ -3069,11 +3081,11 @@ B<youtube-dl> is an excellent tool to download videos. It can,
however, not download videos in parallel. This takes a playlist and
downloads 10 videos in parallel.
url='https://youtu.be/watch?v=0wOf2Fgi3DE&list=UU_cznB5YZZmvAmeq7Y3EriQ'
url='youtu.be/watch?v=0wOf2Fgi3DE&list=UU_cznB5YZZmvAmeq7Y3EriQ'
export url
youtube-dl --flat-playlist "$url" |
youtube-dl --flat-playlist "https://$url" |
parallel --tagstring {#} --lb -j10 \
youtube-dl --playlist-start {#} --playlist-end {#} '"$url"'
youtube-dl --playlist-start {#} --playlist-end {#} '"https://$url"'
=head1 EXAMPLE: Prepend last modified date (ISO8601) to file name
@ -3312,7 +3324,7 @@ repeat the list and use replacement string B<{choose_k}>:
parallel --plus echo {choose_k} ::: A B C D ::: A B C D
parallel --plus echo arg2 {2choose_k} arg1 {1choose_k} ::: A B C D ::: A B C D
parallel --plus echo 2{2choose_k} 1{1choose_k} ::: A B C D ::: A B C D
B<{choose_k}> works for any number of input sources:
@ -3342,7 +3354,8 @@ If the input is in the file in.txt:
If the input is in the array $a here are two solutions:
seq $((${#a[@]}-1)) | env_parallel --env a echo '${a[{=$_--=}]} - ${a[{}]}'
seq $((${#a[@]}-1)) | \
env_parallel --env a echo '${a[{=$_--=}]} - ${a[{}]}'
parallel echo {1} - {2} ::: "${a[@]::${#a[@]}-1}" :::+ "${a[@]:1}"
@ -3370,7 +3383,7 @@ significant. Compare the speed of these:
If your program cannot take multiple arguments, then you can use GNU
B<parallel> to spawn multiple GNU B<parallel>s:
seq -w 0 9999999 |
seq -w 0 9999999 | \
parallel -j10 -q -I,, --pipe parallel -j0 touch pict{}.jpg
If B<-j0> normally spawns 252 jobs, then the above will try to spawn
@ -3465,7 +3478,8 @@ Check the uptime of the servers in I<~/.parallel/sshloginfile>:
Give each job a new color. Most terminals support ANSI colors with the
escape code "\033[30;3Xm" where 0 <= X <= 7:
parallel --tagstring '\033[30;3{=$_=++$::color%8=}m' seq {} ::: {1..10}
seq 10 | \
parallel --tagstring '\033[30;3{=$_=++$::color%8=}m' seq {}
parallel --rpl '{color} $_="\033[30;3".(++$::color%8)."m"' \
--tagstring {color} seq {} ::: {1..10}
@ -3587,7 +3601,7 @@ is much faster.
If it still does not fit in memory you can do this:
parallel --pipepart -a regexps.txt --block 1M grep -Ff - -n bigfile |
parallel --pipepart -a regexps.txt --block 1M grep -Ff - -n bigfile | \
sort -un | perl -pe 's/^\d+://'
The 1M should be your free memory divided by the number of CPU threads and
@ -3599,7 +3613,7 @@ GNU/Linux you can do:
percpu=$((free / 200 / $(parallel --number-of-threads)))k
parallel --pipepart -a regexps.txt --block $percpu --compress \
grep -F -f - -n bigfile |
grep -F -f - -n bigfile | \
sort -un | perl -pe 's/^\d+://'
If you can live with duplicated lines and wrong order, it is faster to do:
@ -3612,8 +3626,8 @@ If you can live with duplicated lines and wrong order, it is faster to do:
If the CPU is the limiting factor parallelization should be done on
the regexps:
cat regexp.txt | parallel --pipe -L1000 --round-robin --compress \
grep -f - -n bigfile |
cat regexp.txt | parallel --pipe -L1000 --roundrobin --compress \
grep -f - -n bigfile | \
sort -un | perl -pe 's/^\d+://'
The command will start one B<grep> per CPU and read I<bigfile> one
@ -3633,7 +3647,7 @@ these chunks. To parallelize both reading of I<bigfile> and I<regexp.txt>
combine the two using B<--fifo>:
parallel --pipepart --block 100M -a bigfile --fifo cat regexp.txt \
\| parallel --pipe -L1000 --round-robin grep -f - {}
\| parallel --pipe -L1000 --roundrobin grep -f - {}
If a line matches multiple regexps, the line may be duplicated.
@ -3672,7 +3686,7 @@ To run B<echo> on B<server.example.com>:
To run commands on more than one remote computer run:
seq 10 | parallel --sshlogin server.example.com,server2.example.net echo
seq 10 | parallel --sshlogin s1.example.com,s2.example.net echo
Or:
@ -3985,7 +3999,8 @@ is even faster:
DBURL=pg://user:pass@host/mydb
DBTABLEURL=$DBURL/mytable
parallel --sqlandworker $DBTABLEURL seq ::: {1..10}
sql $DBURL "COPY (SELECT * FROM mytable) TO stdout DELIMITER ',' CSV HEADER;"
sql $DBURL \
"COPY (SELECT * FROM mytable) TO stdout DELIMITER ',' CSV HEADER;"
Or MySQL:
@ -4015,11 +4030,11 @@ can read into R or LibreCalc, then you can use B<--results>:
The show Aflyttet on Radio 24syv publishes an RSS feed with their audio
podcasts on: http://arkiv.radio24syv.dk/audiopodcast/channel/4466232
Using B<xpath> you can extract the URLs for 2016 and download them
Using B<xpath> you can extract the URLs for 2019 and download them
using GNU B<parallel>:
wget -O - http://arkiv.radio24syv.dk/audiopodcast/channel/4466232 |
xpath -e "//ancestor::pubDate[contains(text(),'2016')]/../enclosure/@url" |
wget -O - http://arkiv.radio24syv.dk/audiopodcast/channel/4466232 | \
xpath -e "//pubDate[contains(text(),'2019')]/../enclosure/@url" | \
parallel -u wget '{= s/ url="//; s/"//; =}'
@ -4124,7 +4139,8 @@ separator before each customer (column 2 = $F[1]). Here we first make
a 50 character random string, which we then use as the separator:
sep=`perl -e 'print map { ("a".."z","A".."Z")[rand(52)] } (1..50);'`
cat my.csv | perl -ape '$F[1] ne $l and print "'$sep'"; $l = $F[1]' |
cat my.csv | \
perl -ape '$F[1] ne $l and print "'$sep'"; $l = $F[1]' | \
parallel --recend $sep --rrs --pipe -N1 wc
If your program can process multiple customers replace B<-N1> with a
@ -4141,13 +4157,13 @@ process. That means that if you just spawn more GNU B<parallel>s then
each of them can run 250 jobs. This will spawn up to 2500 jobs:
cat myinput |\
parallel --pipe -N 50 --round-robin -j50 parallel -j50 your_prg
parallel --pipe -N 50 --roundrobin -j50 parallel -j50 your_prg
This will spawn up to 62500 jobs (use with caution - you need 64 GB
RAM to do this, and you may need to increase /proc/sys/kernel/pid_max):
cat myinput |\
parallel --pipe -N 250 --round-robin -j250 parallel -j250 your_prg
parallel --pipe -N 250 --roundrobin -j250 parallel -j250 your_prg
=head1 EXAMPLE: Working as mutex and counting semaphore

125
src/parallel_alternatives.pod
View file

@ -2050,7 +2050,8 @@ be done by the user.
Stdout (standard output) is prepended with o:. Stderr (standard error)
is sendt to stdout (standard output) and prepended with e:.
For short jobs with little output B<par> is 20% faster than GNU B<parallel>.
For short jobs with little output B<par> is 20% faster than GNU
B<parallel> and 60% slower than B<xargs>.
http://savannah.nongnu.org/projects/par (Last checked: 2019-02)
@ -2060,13 +2061,131 @@ http://savannah.nongnu.org/projects/par (Last checked: 2019-02)
B<fd> does not support composed commands, so commands must be wrapped
in B<sh -c>.
It buffers output in RAM.
It buffers output in RAM.
It only takes file names from the filesystem as input (similar to B<find>).
https://github.com/sharkdp/fd (Last checked: 2019-02)
=head2 DIFFERENCES BETWEEN lateral AND GNU Parallel
B<lateral> is very similar to B<sem>: It takes a single command and
runs it in the background. The design means that output from parallel
running jobs may mix. If it dies unexpectly it leaves a socket in
~/.lateral/socket.PID.
B<lateral> deals badly with too long command lines. This makes the
B<lateral> server crash:
lateral run echo `seq 100000| head -c 1000k`
Any options will be read by B<lateral> so this does not work
(B<lateral> interprets the B<-l>):
lateral run ls -l
Composed commands do not work:
lateral run pwd ';' ls
Functions do not work:
myfunc() { echo a; }
export -f myfunc
lateral run myfunc
Running B<emacs> in the terminal causes the parent shell to die:
echo '#!/bin/bash' > mycmd
echo emacs -nw >> mycmd
chmod +x mycmd
lateral start
lateral run ./mycmd
Here are the examples from https://github.com/akramer/lateral with the
corresponding GNU B<sem> and GNU B<parallel> commands:
1$ lateral start
1$ for i in $(cat /tmp/names); do
1$ lateral run -- some_command $i
1$ done
1$ lateral wait
1$
1$ for i in $(cat /tmp/names); do
1$ sem some_command $i
1$ done
1$ sem --wait
1$
1$ parallel some_command :::: /tmp/names
2$ lateral start
2$ for i in $(seq 1 100); do
2$ lateral run -- my_slow_command < workfile$i > /tmp/logfile$i
2$ done
2$ lateral wait
2$
2$ for i in $(seq 1 100); do
2$ sem my_slow_command < workfile$i > /tmp/logfile$i
2$ done
2$ sem --wait
2$
2$ parallel 'my_slow_command < workfile{} > /tmp/logfile{}' \
::: {1..100}
3$ lateral start -p 0 # yup, it will just queue tasks
3$ for i in $(seq 1 100); do
3$ lateral run -- command_still_outputs_but_wont_spam inputfile$i
3$ done
3$ lateral config -p 10; lateral wait # command output spam can commence
3$
3$ for i in $(seq 1 100); do
3$ echo "command inputfile$i" >> joblist
3$ done
3$ parallel -j 10 :::: joblist
3$
3$ echo 1 > /tmp/njobs
3$ parallel -j /tmp/njobs command inputfile{} \
::: {1..100} &
3$ echo 10 >/tmp/njobs
3$ wait
https://github.com/akramer/lateral (Last checked: 2019-03)
=head2 DIFFERENCES BETWEEN with-this AND GNU Parallel
The examples from https://github.com/amritb/with-this.git and the
corresponding GNU B<parallel> command:
with -v "$(cat myurls.txt)" "curl -L this"
parallel curl -L ::: myurls.txt
with -v "$(cat myregions.txt)" \
"aws --region=this ec2 describe-instance-status"
parallel aws --region={} ec2 describe-instance-status \
:::: myregions.txt
with -v "$(ls)" "kubectl --kubeconfig=this get pods"
ls | parallel kubectl --kubeconfig={} get pods
with -v "$(ls | grep config)" "kubectl --kubeconfig=this get pods"
ls | grep config | parallel kubectl --kubeconfig={} get pods
with -v "$(echo {1..10})" "echo 123"
parallel -N0 echo 123 ::: {1..10}
Stderr is merged with stdout. B<with-this> buffers in RAM. It uses 3x
the output size, so you cannot have output larger than 1/3rd the
amount of RAM. The input values cannot contain spaces. Composed
commands do not work.
B<with-this> gives some additional information, so the output has to
be cleaned before piping it to the next command.
https://github.com/amritb/with-this.git (Last checked: 2019-03)
=head2 Todo
Url for spread
@ -2079,8 +2198,6 @@ https://github.com/bahmutov/with-package
https://github.com/xuchenCN/go-pssh
https://github.com/amritb/with-this.git
https://github.com/flesler/parallel
https://github.com/Julian/Verge

180
src/parallel_cheat.fodt
View file

@ -1,24 +1,24 @@
<?xml version="1.0" encoding="UTF-8"?>
<office:document xmlns:office="urn:oasis:names:tc:opendocument:xmlns:office:1.0" xmlns:style="urn:oasis:names:tc:opendocument:xmlns:style:1.0" xmlns:text="urn:oasis:names:tc:opendocument:xmlns:text:1.0" xmlns:table="urn:oasis:names:tc:opendocument:xmlns:table:1.0" xmlns:draw="urn:oasis:names:tc:opendocument:xmlns:drawing:1.0" xmlns:fo="urn:oasis:names:tc:opendocument:xmlns:xsl-fo-compatible:1.0" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:meta="urn:oasis:names:tc:opendocument:xmlns:meta:1.0" xmlns:number="urn:oasis:names:tc:opendocument:xmlns:datastyle:1.0" xmlns:svg="urn:oasis:names:tc:opendocument:xmlns:svg-compatible:1.0" xmlns:chart="urn:oasis:names:tc:opendocument:xmlns:chart:1.0" xmlns:dr3d="urn:oasis:names:tc:opendocument:xmlns:dr3d:1.0" xmlns:math="http://www.w3.org/1998/Math/MathML" xmlns:form="urn:oasis:names:tc:opendocument:xmlns:form:1.0" xmlns:script="urn:oasis:names:tc:opendocument:xmlns:script:1.0" xmlns:config="urn:oasis:names:tc:opendocument:xmlns:config:1.0" xmlns:ooo="http://openoffice.org/2004/office" xmlns:ooow="http://openoffice.org/2004/writer" xmlns:oooc="http://openoffice.org/2004/calc" xmlns:dom="http://www.w3.org/2001/xml-events" xmlns:xforms="http://www.w3.org/2002/xforms" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:rpt="http://openoffice.org/2005/report" xmlns:of="urn:oasis:names:tc:opendocument:xmlns:of:1.2" xmlns:xhtml="http://www.w3.org/1999/xhtml" xmlns:grddl="http://www.w3.org/2003/g/data-view#" xmlns:officeooo="http://openoffice.org/2009/office" xmlns:tableooo="http://openoffice.org/2009/table" xmlns:drawooo="http://openoffice.org/2010/draw" xmlns:calcext="urn:org:documentfoundation:names:experimental:calc:xmlns:calcext:1.0" xmlns:loext="urn:org:documentfoundation:names:experimental:office:xmlns:loext:1.0" xmlns:field="urn:openoffice:names:experimental:ooo-ms-interop:xmlns:field:1.0" xmlns:formx="urn:openoffice:names:experimental:ooxml-odf-interop:xmlns:form:1.0" xmlns:css3t="http://www.w3.org/TR/css3-text/" office:version="1.2" office:mimetype="application/vnd.oasis.opendocument.text">
<office:meta><meta:creation-date>2019-02-27T22:13:36.781924718</meta:creation-date><dc:date>2019-03-04T21:52:31.122787645</dc:date><meta:editing-duration>P4DT21H57M39S</meta:editing-duration><meta:editing-cycles>25</meta:editing-cycles><meta:generator>LibreOffice/6.0.6.2$Linux_X86_64 LibreOffice_project/00m0$Build-2</meta:generator><meta:print-date>2019-03-04T02:37:46.666605685</meta:print-date><meta:document-statistic meta:table-count="9" meta:image-count="0" meta:object-count="0" meta:page-count="1" meta:paragraph-count="46" meta:word-count="304" meta:character-count="1719" meta:non-whitespace-character-count="1464"/></office:meta>
<office:meta><meta:creation-date>2019-02-27T22:13:36.781924718</meta:creation-date><dc:date>2019-03-12T01:40:23.357999303</dc:date><meta:editing-duration>P11DT5H49M15S</meta:editing-duration><meta:editing-cycles>40</meta:editing-cycles><meta:generator>LibreOffice/6.0.6.2$Linux_X86_64 LibreOffice_project/00m0$Build-2</meta:generator><meta:print-date>2019-03-08T23:06:56.779293422</meta:print-date><meta:document-statistic meta:table-count="9" meta:image-count="0" meta:object-count="0" meta:page-count="1" meta:paragraph-count="47" meta:word-count="337" meta:character-count="1950" meta:non-whitespace-character-count="1661"/></office:meta>
<office:settings>
<config:config-item-set config:name="ooo:view-settings">
<config:config-item config:name="ViewAreaTop" config:type="long">471</config:config-item>
<config:config-item config:name="ViewAreaTop" config:type="long">15875</config:config-item>
<config:config-item config:name="ViewAreaLeft" config:type="long">0</config:config-item>
<config:config-item config:name="ViewAreaWidth" config:type="long">22287</config:config-item>
<config:config-item config:name="ViewAreaHeight" config:type="long">10515</config:config-item>
<config:config-item config:name="ViewAreaHeight" config:type="long">11042</config:config-item>
<config:config-item config:name="ShowRedlineChanges" config:type="boolean">true</config:config-item>
<config:config-item config:name="InBrowseMode" config:type="boolean">false</config:config-item>
<config:config-item-map-indexed config:name="Views">
<config:config-item-map-entry>
<config:config-item config:name="ViewId" config:type="string">view2</config:config-item>
<config:config-item config:name="ViewLeft" config:type="long">8511</config:config-item>
<config:config-item config:name="ViewTop" config:type="long">2536</config:config-item>
<config:config-item config:name="ViewLeft" config:type="long">18724</config:config-item>
<config:config-item config:name="ViewTop" config:type="long">25474</config:config-item>
<config:config-item config:name="VisibleLeft" config:type="long">0</config:config-item>
<config:config-item config:name="VisibleTop" config:type="long">471</config:config-item>
<config:config-item config:name="VisibleTop" config:type="long">15875</config:config-item>
<config:config-item config:name="VisibleRight" config:type="long">22285</config:config-item>
<config:config-item config:name="VisibleBottom" config:type="long">10984</config:config-item>
<config:config-item config:name="VisibleBottom" config:type="long">26915</config:config-item>
<config:config-item config:name="ZoomType" config:type="short">0</config:config-item>
<config:config-item config:name="ViewLayoutColumns" config:type="short">1</config:config-item>
<config:config-item config:name="ViewLayoutBookMode" config:type="boolean">false</config:config-item>
@ -41,7 +41,7 @@
<config:config-item config:name="AlignTabStopPosition" config:type="boolean">true</config:config-item>
<config:config-item config:name="PrintLeftPages" config:type="boolean">true</config:config-item>
<config:config-item config:name="IgnoreFirstLineIndentInNumbering" config:type="boolean">false</config:config-item>
<config:config-item config:name="PrinterSetup" config:type="base64Binary">lwH+/1hlcm94X1BoYXNlcl82MTQwRE4AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQ1VQUzpYZXJveF9QaGFzZXJfNjE0MEROAAAAAAAAAAAWAAMAuAAAAAAAAAAEAAhSAAAEdAAASm9iRGF0YSAxCnByaW50ZXI9WGVyb3hfUGhhc2VyXzYxNDBETgpvcmllbnRhdGlvbj1Qb3J0cmFpdApjb3BpZXM9MQpjb2xsYXRlPWZhbHNlCm1hcmdpbmRhanVzdG1lbnQ9MCwwLDAsMApjb2xvcmRlcHRoPTI0CnBzbGV2ZWw9MApwZGZkZXZpY2U9MQpjb2xvcmRldmljZT0wClBQRENvbnRleERhdGEKUGFnZVNpemU6QTQAABIAQ09NUEFUX0RVUExFWF9NT0RFDwBEdXBsZXhNb2RlOjpPZmY=</config:config-item>
<config:config-item config:name="PrinterSetup" config:type="base64Binary">uAH+/1hlcm94X1BoYXNlcl82MTQwRE4AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQ1VQUzpYZXJveF9QaGFzZXJfNjE0MEROAAAAAAAAAAAWAAMA2QAAAAAAAAAEAAhSAAAEdAAASm9iRGF0YSAxCnByaW50ZXI9WGVyb3hfUGhhc2VyXzYxNDBETgpvcmllbnRhdGlvbj1Qb3J0cmFpdApjb3BpZXM9MQpjb2xsYXRlPWZhbHNlCm1hcmdpbmRhanVzdG1lbnQ9MCwwLDAsMApjb2xvcmRlcHRoPTI0CnBzbGV2ZWw9MApwZGZkZXZpY2U9MQpjb2xvcmRldmljZT0wClBQRENvbnRleERhdGEKRHVwbGV4Ok5vbmUAUGFnZVNpemU6QTQASW5wdXRTbG90OkF1dG9TZWxlY3QAABIAQ09NUEFUX0RVUExFWF9NT0RFDwBEdXBsZXhNb2RlOjpPZmY=</config:config-item>
<config:config-item config:name="CollapseEmptyCellPara" config:type="boolean">true</config:config-item>
<config:config-item config:name="RedlineProtectionKey" config:type="base64Binary"/>
<config:config-item config:name="UseOldPrinterMetrics" config:type="boolean">false</config:config-item>
@ -92,7 +92,7 @@
<config:config-item config:name="UnxForceZeroExtLeading" config:type="boolean">false</config:config-item>
<config:config-item config:name="TabAtLeftIndentForParagraphsInList" config:type="boolean">false</config:config-item>
<config:config-item config:name="PrintRightPages" config:type="boolean">true</config:config-item>
<config:config-item config:name="Rsid" config:type="int">1506730</config:config-item>
<config:config-item config:name="Rsid" config:type="int">1662577</config:config-item>
<config:config-item config:name="MathBaselineAlignment" config:type="boolean">true</config:config-item>
<config:config-item config:name="MsWordCompTrailingBlanks" config:type="boolean">false</config:config-item>
<config:config-item config:name="InvertBorderSpacing" config:type="boolean">false</config:config-item>
@ -366,10 +366,10 @@
</style:table-properties>
</style:style>
<style:style style:name="Table10.A" style:family="table-column">
<style:table-column-properties style:column-width="2.5729in" style:rel-column-width="25185*"/>
<style:table-column-properties style:column-width="2.3736in" style:rel-column-width="3418*"/>
</style:style>
<style:style style:name="Table10.B" style:family="table-column">
<style:table-column-properties style:column-width="4.1222in" style:rel-column-width="40350*"/>
<style:table-column-properties style:column-width="4.3215in" style:rel-column-width="6223*"/>
</style:style>
<style:style style:name="Table10.1" style:family="table-row">
<style:table-row-properties style:min-row-height="0.1778in" fo:background-color="#000000" fo:keep-together="auto">
@ -611,30 +611,41 @@
<style:style style:name="P16" style:family="paragraph" style:parent-style-name="Standard">
<style:text-properties officeooo:paragraph-rsid="0015544e"/>
</style:style>
<style:style style:name="P17" style:family="paragraph" style:parent-style-name="Table_20_Contents">
<style:text-properties officeooo:paragraph-rsid="0016fdaa"/>
<style:style style:name="P17" style:family="paragraph" style:parent-style-name="Standard">
<style:paragraph-properties fo:text-align="end" style:justify-single-word="false"/>
<style:text-properties fo:font-size="6pt" officeooo:rsid="00180262" officeooo:paragraph-rsid="00180262" style:font-size-asian="6pt" style:font-size-complex="6pt"/>
</style:style>
<style:style style:name="P18" style:family="paragraph" style:parent-style-name="Table_20_Contents">
<style:text-properties officeooo:paragraph-rsid="0016fdaa"/>
</style:style>
<style:style style:name="P19" style:family="paragraph" style:parent-style-name="Table_20_Contents">
<style:paragraph-properties fo:text-align="start" style:justify-single-word="false"/>
<style:text-properties fo:color="#000000" style:text-outline="false" style:text-line-through-style="none" style:text-line-through-type="none" style:font-name="Thorndale" fo:font-size="12pt" fo:font-style="normal" fo:text-shadow="none" style:text-underline-style="none" fo:font-weight="bold" officeooo:rsid="0010bcb2" officeooo:paragraph-rsid="001483d1" style:font-size-asian="12pt" style:font-style-asian="normal" style:font-weight-asian="bold" style:font-size-complex="12pt" style:font-style-complex="normal" style:font-weight-complex="bold" style:text-overline-style="none" style:text-overline-color="font-color"/>
</style:style>
<style:style style:name="P19" style:family="paragraph" style:parent-style-name="Table_20_Contents">
<style:style style:name="P20" style:family="paragraph" style:parent-style-name="Table_20_Contents">
<style:text-properties fo:font-weight="bold" officeooo:rsid="0016fdaa" officeooo:paragraph-rsid="0016fdaa" style:font-weight-asian="bold" style:font-weight-complex="bold"/>
</style:style>
<style:style style:name="P20" style:family="paragraph" style:parent-style-name="Table_20_Contents">
<style:style style:name="P21" style:family="paragraph" style:parent-style-name="Table_20_Contents">
<style:text-properties fo:font-weight="bold" officeooo:paragraph-rsid="0016fdaa" style:font-weight-asian="bold" style:font-weight-complex="bold"/>
</style:style>
<style:style style:name="P21" style:family="paragraph" style:parent-style-name="Table_20_Contents">
<style:style style:name="P22" style:family="paragraph" style:parent-style-name="Table_20_Contents">
<style:text-properties officeooo:rsid="0016fdaa" officeooo:paragraph-rsid="0016fdaa"/>
</style:style>
<style:style style:name="P22" style:family="paragraph" style:parent-style-name="Table_20_Heading">
<style:style style:name="P23" style:family="paragraph" style:parent-style-name="Table_20_Contents">
<style:text-properties officeooo:paragraph-rsid="00180262"/>
</style:style>
<style:style style:name="P24" style:family="paragraph" style:parent-style-name="Table_20_Heading">
<style:paragraph-properties fo:text-align="start" style:justify-single-word="false"/>
<style:text-properties style:use-window-font-color="true" style:text-outline="false" style:text-line-through-style="none" style:text-line-through-type="none" style:font-name="FreeSans" fo:font-size="12pt" fo:font-style="normal" fo:text-shadow="none" style:text-underline-style="none" fo:font-weight="600" officeooo:rsid="001483d1" officeooo:paragraph-rsid="001483d1" style:font-size-asian="12pt" style:font-style-asian="normal" style:font-weight-asian="600" style:font-size-complex="12pt" style:font-style-complex="normal" style:font-weight-complex="600" style:text-overline-style="none" style:text-overline-color="font-color"/>
</style:style>
<style:style style:name="P23" style:family="paragraph" style:parent-style-name="Table_20_Heading">
<style:style style:name="P25" style:family="paragraph" style:parent-style-name="Table_20_Heading">
<style:paragraph-properties fo:text-align="start" style:justify-single-word="false"/>
<style:text-properties style:use-window-font-color="true" style:text-outline="false" style:text-line-through-style="none" style:text-line-through-type="none" style:font-name="FreeSans" fo:font-size="12pt" fo:font-style="normal" fo:text-shadow="none" style:text-underline-style="none" fo:font-weight="600" officeooo:rsid="0015544e" officeooo:paragraph-rsid="0015544e" style:font-size-asian="12pt" style:font-style-asian="normal" style:font-weight-asian="600" style:font-size-complex="12pt" style:font-style-complex="normal" style:font-weight-complex="600" style:text-overline-style="none" style:text-overline-color="font-color"/>
</style:style>
<style:style style:name="P26" style:family="paragraph" style:parent-style-name="Table_20_Heading">
<style:paragraph-properties fo:text-align="start" style:justify-single-word="false"/>
<style:text-properties officeooo:paragraph-rsid="001483d1"/>
</style:style>
<style:style style:name="T1" style:family="text">
<style:text-properties fo:font-weight="bold" style:font-weight-asian="bold" style:font-weight-complex="bold"/>
</style:style>
@ -642,56 +653,83 @@
<style:text-properties fo:font-weight="bold" officeooo:rsid="0015544e" style:font-weight-asian="bold" style:font-weight-complex="bold"/>
</style:style>
<style:style style:name="T3" style:family="text">
<style:text-properties officeooo:rsid="0012b9fe"/>
<style:text-properties fo:font-weight="bold" officeooo:rsid="00180262" style:font-weight-asian="bold" style:font-weight-complex="bold"/>
</style:style>
<style:style style:name="T4" style:family="text">
<style:text-properties fo:font-weight="normal" style:font-weight-asian="normal" style:font-weight-complex="normal"/>
<style:text-properties officeooo:rsid="0012b9fe"/>
</style:style>
<style:style style:name="T5" style:family="text">
<style:text-properties fo:font-weight="normal" officeooo:rsid="001483d1" style:font-weight-asian="normal" style:font-weight-complex="normal"/>
<style:text-properties fo:font-weight="normal" style:font-weight-asian="normal" style:font-weight-complex="normal"/>
</style:style>
<style:style style:name="T6" style:family="text">
<style:text-properties fo:font-weight="normal" officeooo:rsid="0015544e" style:font-weight-asian="normal" style:font-weight-complex="normal"/>
<style:text-properties fo:font-weight="normal" officeooo:rsid="001483d1" style:font-weight-asian="normal" style:font-weight-complex="normal"/>
</style:style>
<style:style style:name="T7" style:family="text">
<style:text-properties officeooo:rsid="00137669"/>
<style:text-properties fo:font-weight="normal" officeooo:rsid="0015544e" style:font-weight-asian="normal" style:font-weight-complex="normal"/>
</style:style>
<style:style style:name="T8" style:family="text">
<style:text-properties officeooo:rsid="001483d1"/>
<style:text-properties fo:font-weight="normal" officeooo:rsid="0016fdaa" style:font-weight-asian="normal" style:font-weight-complex="normal"/>
</style:style>
<style:style style:name="T9" style:family="text">
<style:text-properties style:font-name="DejaVu Sans" officeooo:rsid="000ccaed"/>
<style:text-properties officeooo:rsid="00137669"/>
</style:style>
<style:style style:name="T10" style:family="text">
<style:text-properties fo:color="#000000" style:text-outline="false" style:text-line-through-style="none" style:text-line-through-type="none" fo:font-style="normal" fo:text-shadow="none" style:text-underline-style="none" fo:font-weight="normal" officeooo:rsid="0010bcb2" style:font-style-asian="normal" style:font-weight-asian="normal" style:font-style-complex="normal" style:font-weight-complex="normal" style:text-overline-style="none" style:text-overline-color="font-color"/>
<style:text-properties officeooo:rsid="001483d1"/>
</style:style>
<style:style style:name="T11" style:family="text">
<style:text-properties fo:color="#000000" style:text-outline="false" style:text-line-through-style="none" style:text-line-through-type="none" fo:font-style="normal" fo:text-shadow="none" style:text-underline-style="none" fo:font-weight="normal" officeooo:rsid="0015544e" style:font-style-asian="normal" style:font-weight-asian="normal" style:font-style-complex="normal" style:font-weight-complex="normal" style:text-overline-style="none" style:text-overline-color="font-color"/>
<style:text-properties style:font-name="DejaVu Sans" officeooo:rsid="000ccaed"/>
</style:style>
<style:style style:name="T12" style:family="text">
<style:text-properties fo:color="#000000" style:text-outline="false" style:text-line-through-style="none" style:text-line-through-type="none" fo:font-style="normal" fo:text-shadow="none" style:text-underline-style="none" fo:font-weight="normal" officeooo:rsid="001483d1" style:font-style-asian="normal" style:font-weight-asian="normal" style:font-style-complex="normal" style:font-weight-complex="normal" style:text-overline-style="none" style:text-overline-color="font-color"/>
<style:text-properties fo:color="#000000" style:text-outline="false" style:text-line-through-style="none" style:text-line-through-type="none" fo:font-style="normal" fo:text-shadow="none" style:text-underline-style="none" fo:font-weight="normal" officeooo:rsid="0010bcb2" style:font-style-asian="normal" style:font-weight-asian="normal" style:font-style-complex="normal" style:font-weight-complex="normal" style:text-overline-style="none" style:text-overline-color="font-color"/>
</style:style>
<style:style style:name="T13" style:family="text">
<style:text-properties officeooo:rsid="0015544e"/>
<style:text-properties fo:color="#000000" style:text-outline="false" style:text-line-through-style="none" style:text-line-through-type="none" fo:font-style="normal" fo:text-shadow="none" style:text-underline-style="none" fo:font-weight="normal" officeooo:rsid="0015544e" style:font-style-asian="normal" style:font-weight-asian="normal" style:font-style-complex="normal" style:font-weight-complex="normal" style:text-overline-style="none" style:text-overline-color="font-color"/>
</style:style>
<style:style style:name="T14" style:family="text">
<style:text-properties style:use-window-font-color="true" style:text-outline="false" style:text-line-through-style="none" style:text-line-through-type="none" style:font-name="FreeSans" fo:font-size="12pt" fo:font-style="normal" fo:text-shadow="none" style:text-underline-style="none" fo:font-weight="600" officeooo:rsid="001483d1" style:font-size-asian="12pt" style:font-style-asian="normal" style:font-weight-asian="600" style:font-size-complex="12pt" style:font-style-complex="normal" style:font-weight-complex="600" style:text-overline-style="none" style:text-overline-color="font-color"/>
<style:text-properties fo:color="#000000" style:text-outline="false" style:text-line-through-style="none" style:text-line-through-type="none" fo:font-style="normal" fo:text-shadow="none" style:text-underline-style="none" fo:font-weight="normal" officeooo:rsid="001483d1" style:font-style-asian="normal" style:font-weight-asian="normal" style:font-style-complex="normal" style:font-weight-complex="normal" style:text-overline-style="none" style:text-overline-color="font-color"/>
</style:style>
<style:style style:name="T15" style:family="text">
<style:text-properties style:font-name="Bitstream Vera Serif1" fo:font-weight="normal" officeooo:rsid="0015544e" style:font-weight-asian="normal" style:font-weight-complex="normal"/>
<style:text-properties officeooo:rsid="0015544e"/>
</style:style>
<style:style style:name="T16" style:family="text">
<style:text-properties style:font-name="Bitstream Vera Serif1" fo:font-weight="bold" officeooo:rsid="0015544e" style:font-weight-asian="bold" style:font-weight-complex="bold"/>
<style:text-properties style:use-window-font-color="true" style:text-outline="false" style:text-line-through-style="none" style:text-line-through-type="none" style:font-name="FreeSans" fo:font-size="12pt" fo:font-style="normal" fo:text-shadow="none" style:text-underline-style="none" fo:font-weight="600" style:font-size-asian="12pt" style:font-style-asian="normal" style:font-weight-asian="600" style:font-size-complex="12pt" style:font-style-complex="normal" style:font-weight-complex="600" style:text-overline-style="none" style:text-overline-color="font-color"/>
</style:style>
<style:style style:name="T17" style:family="text">
<style:text-properties officeooo:rsid="0016fdaa"/>
<style:text-properties style:use-window-font-color="true" style:text-outline="false" style:text-line-through-style="none" style:text-line-through-type="none" style:font-name="FreeSans" fo:font-size="12pt" fo:font-style="normal" fo:text-shadow="none" style:text-underline-style="none" fo:font-weight="600" officeooo:rsid="001483d1" style:font-size-asian="12pt" style:font-style-asian="normal" style:font-weight-asian="600" style:font-size-complex="12pt" style:font-style-complex="normal" style:font-weight-complex="600" style:text-overline-style="none" style:text-overline-color="font-color"/>
</style:style>
<style:style style:name="T18" style:family="text">
<style:text-properties fo:font-style="italic" style:font-style-asian="italic" style:font-style-complex="italic"/>
<style:text-properties style:use-window-font-color="true" style:text-outline="false" style:text-line-through-style="none" style:text-line-through-type="none" style:font-name="FreeSans" fo:font-size="12pt" fo:font-style="normal" fo:text-shadow="none" style:text-underline-style="none" fo:font-weight="600" officeooo:rsid="00195e71" style:font-size-asian="12pt" style:font-style-asian="normal" style:font-weight-asian="600" style:font-size-complex="12pt" style:font-style-complex="normal" style:font-weight-complex="600" style:text-overline-style="none" style:text-overline-color="font-color"/>
</style:style>
<style:style style:name="T19" style:family="text">
<style:text-properties style:font-name="Bitstream Vera Serif1" fo:font-weight="normal" officeooo:rsid="0015544e" style:font-weight-asian="normal" style:font-weight-complex="normal"/>
</style:style>
<style:style style:name="T20" style:family="text">
<style:text-properties style:font-name="Bitstream Vera Serif1" fo:font-weight="bold" officeooo:rsid="0015544e" style:font-weight-asian="bold" style:font-weight-complex="bold"/>
</style:style>
<style:style style:name="T21" style:family="text">
<style:text-properties officeooo:rsid="0016fdaa"/>
</style:style>
<style:style style:name="T22" style:family="text">
<style:text-properties fo:font-style="italic" style:font-style-asian="italic" style:font-style-complex="italic"/>
</style:style>
<style:style style:name="T23" style:family="text">
<style:text-properties fo:font-style="italic" fo:font-weight="normal" officeooo:rsid="0015544e" style:font-style-asian="italic" style:font-weight-asian="normal" style:font-style-complex="italic" style:font-weight-complex="normal"/>
</style:style>
<style:style style:name="T24" style:family="text">
<style:text-properties officeooo:rsid="00180262"/>
</style:style>
<style:style style:name="T25" style:family="text">
<style:text-properties fo:font-style="normal" style:font-style-asian="normal" style:font-style-complex="normal"/>
</style:style>
<style:style style:name="T26" style:family="text">
<style:text-properties fo:font-style="normal" fo:font-weight="bold" style:font-style-asian="normal" style:font-weight-asian="bold" style:font-style-complex="normal" style:font-weight-complex="bold"/>
</style:style>
<style:style style:name="T27" style:family="text">
<style:text-properties fo:font-style="normal" fo:font-weight="bold" officeooo:rsid="00180262" style:font-style-asian="normal" style:font-weight-asian="bold" style:font-style-complex="normal" style:font-weight-complex="bold"/>
</style:style>
<style:style style:name="T28" style:family="text">
<style:text-properties officeooo:rsid="00195e71"/>
</style:style>
<style:page-layout style:name="pm1">
<style:page-layout-properties fo:page-width="8.2701in" fo:page-height="11.6902in" style:num-format="1" style:print-orientation="portrait" fo:margin-top="0.7874in" fo:margin-bottom="0.7874in" fo:margin-left="0.7874in" fo:margin-right="0.7874in" style:writing-mode="lr-tb" style:layout-grid-color="#c0c0c0" style:layout-grid-lines="20" style:layout-grid-base-height="0.278in" style:layout-grid-ruby-height="0.139in" style:layout-grid-mode="none" style:layout-grid-ruby-below="false" style:layout-grid-print="false" style:layout-grid-display="false" style:footnote-max-height="0in">
<style:columns fo:column-count="1" fo:column-gap="0in"/>
@ -720,12 +758,12 @@
<table:table-column table:style-name="Table7.A"/>
<table:table-row table:style-name="Table7.1">
<table:table-cell table:style-name="Table7.A1" office:value-type="string">
<text:p text:style-name="P15">GNU Para<text:span text:style-name="T18">ll</text:span>el Cheat Sheet</text:p>
<text:p text:style-name="P15">GNU Para<text:span text:style-name="T22">ll</text:span>el Cheat Sheet</text:p>
</table:table-cell>
</table:table-row>
<table:table-row table:style-name="Table7.2">
<table:table-cell table:style-name="Table7.A2" office:value-type="float" office:value="0">
<text:p text:style-name="P6">GNU Parallel is a replacement for <text:span text:style-name="Source_20_Text"><text:span text:style-name="T18">xargs</text:span></text:span>, and <text:span text:style-name="Source_20_Text"><text:span text:style-name="T18">for</text:span></text:span> loops. It can also split a file or a stream into blocks and pass those to commands <text:span text:style-name="T13">running in parallel.</text:span></text:p>
<text:p text:style-name="P6">GNU Parallel is a replacement for <text:span text:style-name="Source_20_Text"><text:span text:style-name="T22">xargs</text:span></text:span> and <text:span text:style-name="Source_20_Text"><text:span text:style-name="T22">for</text:span></text:span> loops. It can also split a file or a stream into blocks and pass those to commands <text:span text:style-name="T15">running in parallel.</text:span></text:p>
</table:table-cell>
</table:table-row>
</table:table>
@ -734,22 +772,22 @@
<table:table-column table:style-name="Table8.A"/>
<table:table-row table:style-name="Table8.1">
<table:table-cell table:style-name="Table8.A1" office:value-type="string">
<text:p text:style-name="P13">Example<text:span text:style-name="T8">s</text:span></text:p>
<text:p text:style-name="P13">Example<text:span text:style-name="T10">s</text:span></text:p>
</table:table-cell>
</table:table-row>
<table:table-row table:style-name="Table8.2">
<table:table-cell table:style-name="Table8.A2" office:value-type="float" office:value="0">
<text:p text:style-name="P12"><text:span text:style-name="T1">Compress all *.html files in parallel – 2 per CPU thread</text:span><text:line-break/><text:span text:style-name="Source_20_Text"><text:span text:style-name="T10">parallel --</text:span></text:span><text:span text:style-name="Source_20_Text"><text:span text:style-name="T11">jobs 200% </text:span></text:span><text:span text:style-name="Source_20_Text"><text:span text:style-name="T10">gzip ::: *.html</text:span></text:span></text:p>
<text:p text:style-name="P12"><text:span text:style-name="T1">Compress all *.html files in parallel – 2 </text:span><text:span text:style-name="T3">jobs </text:span><text:span text:style-name="T1">per CPU thread </text:span><text:span text:style-name="T3">in parallel</text:span><text:line-break/><text:span text:style-name="Source_20_Text"><text:span text:style-name="T12">parallel --</text:span></text:span><text:span text:style-name="Source_20_Text"><text:span text:style-name="T13">jobs 200% </text:span></text:span><text:span text:style-name="Source_20_Text"><text:span text:style-name="T12">gzip ::: *.html</text:span></text:span></text:p>
</table:table-cell>
</table:table-row>
<table:table-row table:style-name="Table8.3">
<table:table-cell table:style-name="Table8.A2" office:value-type="float" office:value="0">
<text:p text:style-name="P12"><text:span text:style-name="T1">Convert all *.wav to *.mp3 using </text:span><text:span text:style-name="Source_20_Text"><text:span text:style-name="T18">lame</text:span></text:span><text:line-break/><text:span text:style-name="Source_20_Text"><text:span text:style-name="T10">parallel lame {} -o {.}.mp3 ::: *.wav</text:span></text:span></text:p>
<text:p text:style-name="P23"><text:span text:style-name="T1">Convert all *.wav to *.mp3 using </text:span><text:span text:style-name="Source_20_Text"><text:span text:style-name="T22">lame</text:span></text:span><text:span text:style-name="T1"> – </text:span><text:span text:style-name="T3">1</text:span><text:span text:style-name="T1"> job per CPU thread in parallel </text:span><text:span text:style-name="T3">(default)</text:span><text:line-break/><text:span text:style-name="Source_20_Text"><text:span text:style-name="T12">parallel lame {} -o {.}.mp3 ::: *.wav</text:span></text:span></text:p>
</table:table-cell>
</table:table-row>
<table:table-row table:style-name="Table8.4">
<table:table-cell table:style-name="Table8.A2" office:value-type="float" office:value="0">
<text:p text:style-name="P12"><text:span text:style-name="T1">Chop bigfile into 1MB blocks and grep for the string foobar</text:span><text:line-break/><text:span text:style-name="Source_20_Text"><text:span text:style-name="T12">cat bigfile | </text:span></text:span><text:span text:style-name="Source_20_Text"><text:span text:style-name="T10">parallel --pipe grep foobar</text:span></text:span></text:p>
<text:p text:style-name="P12"><text:span text:style-name="T1">Chop bigfile into 1MB blocks and grep for the string foobar</text:span><text:line-break/><text:span text:style-name="Source_20_Text"><text:span text:style-name="T14">cat bigfile | </text:span></text:span><text:span text:style-name="Source_20_Text"><text:span text:style-name="T12">parallel --pipe grep foobar</text:span></text:span></text:p>
</table:table-cell>
</table:table-row>
</table:table>
@ -763,32 +801,32 @@
</table:table-row>
<table:table-row table:style-name="Table9.2">
<table:table-cell table:style-name="Table9.A2" office:value-type="string">
<text:p text:style-name="P11"><text:span text:style-name="Source_20_Text"><text:span text:style-name="T4">parallel echo ::: cmd line input source</text:span></text:span></text:p>
<text:p text:style-name="P11"><text:span text:style-name="Source_20_Text"><text:span text:style-name="T5">parallel echo ::: cmd line input source</text:span></text:span></text:p>
</table:table-cell>
</table:table-row>
<table:table-row table:style-name="Table9.3">
<table:table-cell table:style-name="Table9.A2" office:value-type="string">
<text:p text:style-name="P11"><text:span text:style-name="Source_20_Text"><text:span text:style-name="T4">cat input_from_stdin | parallel echo</text:span></text:span></text:p>
<text:p text:style-name="P11"><text:span text:style-name="Source_20_Text"><text:span text:style-name="T5">cat input_from_stdin | parallel echo</text:span></text:span></text:p>
</table:table-cell>
</table:table-row>
<table:table-row table:style-name="Table9.4">
<table:table-cell table:style-name="Table9.A2" office:value-type="string">
<text:p text:style-name="P11"><text:span text:style-name="Source_20_Text"><text:span text:style-name="T4">parallel echo ::: multiple input ::: source values</text:span></text:span></text:p>
<text:p text:style-name="P11"><text:span text:style-name="Source_20_Text"><text:span text:style-name="T5">parallel echo ::: multiple input source</text:span></text:span><text:span text:style-name="Source_20_Text"><text:span text:style-name="T8">s ::: with </text:span></text:span><text:span text:style-name="Source_20_Text"><text:span text:style-name="T5">values</text:span></text:span></text:p>
</table:table-cell>
</table:table-row>
<table:table-row table:style-name="Table9.5">
<table:table-cell table:style-name="Table9.A2" office:value-type="string">
<text:p text:style-name="P11"><text:span text:style-name="Source_20_Text"><text:span text:style-name="T4">parallel -a input_from_file echo</text:span></text:span></text:p>
<text:p text:style-name="P11"><text:span text:style-name="Source_20_Text"><text:span text:style-name="T5">parallel -a input_from_file echo</text:span></text:span></text:p>
</table:table-cell>
</table:table-row>
<table:table-row table:style-name="Table9.6">
<table:table-cell table:style-name="Table9.A2" office:value-type="string">
<text:p text:style-name="P11"><text:span text:style-name="Source_20_Text"><text:span text:style-name="T4">parallel echo :::: input_from_file</text:span></text:span></text:p>
<text:p text:style-name="P11"><text:span text:style-name="Source_20_Text"><text:span text:style-name="T5">parallel echo :::: input_from_file</text:span></text:span></text:p>
</table:table-cell>
</table:table-row>
<table:table-row table:style-name="Table9.7">
<table:table-cell table:style-name="Table9.A2" office:value-type="string">
<text:p text:style-name="P10"><text:span text:style-name="Source_20_Text"><text:span text:style-name="T4">parallel echo :::: input_from_file ::: and command line</text:span></text:span></text:p>
<text:p text:style-name="P10"><text:span text:style-name="Source_20_Text"><text:span text:style-name="T5">parallel echo :::: input_from_file ::: and command line</text:span></text:span></text:p>
</table:table-cell>
</table:table-row>
</table:table>
@ -801,7 +839,7 @@
<text:p text:style-name="P13">Replacement string</text:p>
</table:table-cell>
<table:table-cell table:style-name="Table10.B1" office:value-type="string">
<text:p text:style-name="P13">Value</text:p>
<text:p text:style-name="P26"><text:span text:style-name="T16">Value </text:span><text:span text:style-name="T18">if input is mydir/mysubdir/myfile.myext</text:span></text:p>
</table:table-cell>
</table:table-row>
<table:table-row table:style-name="Table10.2">
@ -822,10 +860,10 @@
</table:table-row>
<table:table-row table:style-name="Table10.4">
<table:table-cell table:style-name="Table10.A2" office:value-type="string">
<text:p text:style-name="P17"><text:span text:style-name="Source_20_Text">{/}, {//}, </text:span><text:span text:style-name="Source_20_Text">{/.}</text:span></text:p>
<text:p text:style-name="P18"><text:span text:style-name="Source_20_Text">{/}, {//}, </text:span><text:span text:style-name="Source_20_Text">{/.}</text:span></text:p>
</table:table-cell>
<table:table-cell table:style-name="Table10.B2" office:value-type="string">
<text:p text:style-name="P17"><text:span text:style-name="Source_20_Text">myfile.myext, mydir/mysubdir, </text:span><text:span text:style-name="Source_20_Text">myfile</text:span></text:p>
<text:p text:style-name="P18"><text:span text:style-name="Source_20_Text">myfile.myext, mydir/mysubdir, </text:span><text:span text:style-name="Source_20_Text">myfile</text:span></text:p>
</table:table-cell>
</table:table-row>
<table:table-row table:style-name="Table10.5">
@ -833,7 +871,7 @@
<text:p text:style-name="P11"><text:span text:style-name="Source_20_Text">{#}</text:span></text:p>
</table:table-cell>
<table:table-cell table:style-name="Table10.B2" office:value-type="string">
<text:p text:style-name="P8"><text:span text:style-name="T3">T</text:span>he sequence number of the job</text:p>
<text:p text:style-name="P8"><text:span text:style-name="T4">T</text:span>he sequence number of the job</text:p>
</table:table-cell>
</table:table-row>
<table:table-row table:style-name="Table10.6">
@ -841,7 +879,7 @@
<text:p text:style-name="P11"><text:span text:style-name="Source_20_Text">{%}</text:span></text:p>
</table:table-cell>
<table:table-cell table:style-name="Table10.B2" office:value-type="string">
<text:p text:style-name="P8"><text:span text:style-name="T3">T</text:span>he job slot number</text:p>
<text:p text:style-name="P8"><text:span text:style-name="T4">T</text:span>he job slot number</text:p>
</table:table-cell>
</table:table-row>
<table:table-row table:style-name="Table10.7">
@ -849,20 +887,20 @@
<text:p text:style-name="P5"><text:span text:style-name="Source_20_Text">{2}</text:span></text:p>
</table:table-cell>
<table:table-cell table:style-name="Table10.B2" office:value-type="string">
<text:p text:style-name="P7"><text:span text:style-name="T17">Value from the s</text:span>econd input source</text:p>
<text:p text:style-name="P7"><text:span text:style-name="T21">Value from the s</text:span>econd input source</text:p>
</table:table-cell>
</table:table-row>
<table:table-row table:style-name="Table10.8">
<table:table-cell table:style-name="Table10.A2" office:value-type="string">
<text:p text:style-name="P5"><text:span text:style-name="Source_20_Text">{2.} </text:span><text:span text:style-name="Source_20_Text"><text:span text:style-name="T17">{2/} {2//} {2/.}</text:span></text:span></text:p>
<text:p text:style-name="P5"><text:span text:style-name="Source_20_Text">{2.} </text:span><text:span text:style-name="Source_20_Text"><text:span text:style-name="T21">{2/} {2//} {2/.}</text:span></text:span></text:p>
</table:table-cell>
<table:table-cell table:style-name="Table10.B2" office:value-type="string">
<text:p text:style-name="P7">Comb<text:span text:style-name="T17">ination </text:span>of {2} and {.} <text:span text:style-name="T17">{/} {//} {/.}</text:span></text:p>
<text:p text:style-name="P7">Comb<text:span text:style-name="T21">ination </text:span>of {2} and {.} <text:span text:style-name="T21">{/} {//} {/.}</text:span></text:p>
</table:table-cell>
</table:table-row>
<table:table-row table:style-name="Table10.9">
<table:table-cell table:style-name="Table10.A2" office:value-type="string">
<text:p text:style-name="P9"><text:span text:style-name="Source_20_Text">{= perl </text:span><text:span text:style-name="Source_20_Text"><text:span text:style-name="T13">expression</text:span></text:span><text:span text:style-name="Source_20_Text"> =}</text:span></text:p>
<text:p text:style-name="P9"><text:span text:style-name="Source_20_Text">{= perl </text:span><text:span text:style-name="Source_20_Text"><text:span text:style-name="T15">expression</text:span></text:span><text:span text:style-name="Source_20_Text"> =}</text:span></text:p>
</table:table-cell>
<table:table-cell table:style-name="Table10.B2" office:value-type="string">
<text:p text:style-name="P7">Change $_ with perl expression</text:p>
@ -874,12 +912,12 @@
<table:table-column table:style-name="Table11.A"/>
<table:table-row table:style-name="Table11.1">
<table:table-cell table:style-name="Table11.A1" office:value-type="string">
<text:p text:style-name="P13">Control the output – <text:span text:style-name="T8">keep the order the same as the input</text:span></text:p>
<text:p text:style-name="P13">Control the output – <text:span text:style-name="T10">keep the same order as the input, prepend with input value</text:span></text:p>
</table:table-cell>
</table:table-row>
<table:table-row table:style-name="Table11.2">
<table:table-cell table:style-name="Table11.A2" office:value-type="string">
<text:p text:style-name="P17"><text:span text:style-name="Source_20_Text"><text:span text:style-name="T8">parallel --keep-order &quot;sleep {}; echo {}&quot; ::: 5 4 3 2 1</text:span></text:span></text:p>
<text:p text:style-name="P18"><text:span text:style-name="Source_20_Text"><text:span text:style-name="T10">parallel --keep-order --tag &quot;sleep {}; echo {}&quot; ::: 5 4 3 2 1</text:span></text:span></text:p>
</table:table-cell>
</table:table-row>
</table:table>
@ -893,14 +931,14 @@
</table:table-row>
<table:table-row table:style-name="Table12.3">
<table:table-cell table:style-name="Table12.A2" office:value-type="string">
<text:p text:style-name="P19">Run 2 jobs in parallel</text:p>
<text:p text:style-name="P17"><text:span text:style-name="Source_20_Text"><text:span text:style-name="T8">parallel --jobs 2 &quot;sleep {}; echo {}&quot; ::: 5 4 3 2 1</text:span></text:span></text:p>
<text:p text:style-name="P20">Run 2 jobs in parallel – <text:span text:style-name="T24">command is a composed command</text:span></text:p>
<text:p text:style-name="P18"><text:span text:style-name="Source_20_Text"><text:span text:style-name="T10">parallel --jobs 2 &quot;sleep {}; echo {}&quot; ::: 5 4 3 2 1</text:span></text:span></text:p>
</table:table-cell>
</table:table-row>
<table:table-row table:style-name="Table12.3">
<table:table-cell table:style-name="Table12.A2" office:value-type="string">
<text:p text:style-name="P19">See what will be run</text:p>
<text:p text:style-name="P21"><text:span text:style-name="Source_20_Text">parallel --dryrun echo ::: Red Green Blue ::: S M L</text:span></text:p>
<text:p text:style-name="P20">See what will be run</text:p>
<text:p text:style-name="P22"><text:span text:style-name="Source_20_Text">parallel --dryrun echo </text:span><text:span text:style-name="Source_20_Text"><text:span text:style-name="T28">{2} {1} </text:span></text:span><text:span text:style-name="Source_20_Text">::: </text:span><text:span text:style-name="Source_20_Text"><text:span text:style-name="T28">bird flower fish ::: </text:span></text:span><text:span text:style-name="Source_20_Text">Red Green Blue</text:span></text:p>
</table:table-cell>
</table:table-row>
</table:table>
@ -909,12 +947,12 @@
<table:table-column table:style-name="Table13.A"/>
<table:table-row table:style-name="Table13.1">
<table:table-cell table:style-name="Table13.A1" office:value-type="string">
<text:p text:style-name="P22">Remote execution</text:p>
<text:p text:style-name="P24">Remote execution</text:p>
</table:table-cell>
</table:table-row>
<table:table-row table:style-name="Table13.2">
<table:table-cell table:style-name="Table13.A2" office:value-type="string">
<text:p text:style-name="P17"><text:span text:style-name="Source_20_Text"><text:span text:style-name="T8">parallel -S server1 -S server2 &quot;hostname; echo {}&quot; ::: foo bar </text:span></text:span></text:p>
<text:p text:style-name="P18"><text:span text:style-name="Source_20_Text"><text:span text:style-name="T10">parallel -S server1 -S server2 &quot;hostname; echo {}&quot; ::: foo bar </text:span></text:span></text:p>
</table:table-cell>
</table:table-row>
</table:table>
@ -923,17 +961,17 @@
<table:table-column table:style-name="Table1.A"/>
<table:table-row table:style-name="Table1.1">
<table:table-cell table:style-name="Table1.A1" office:value-type="string">
<text:p text:style-name="P22">Pipe mode</text:p>
<text:p text:style-name="P24">Pipe mode</text:p>
</table:table-cell>
</table:table-row>
<table:table-row table:style-name="Table1.2">
<table:table-cell table:style-name="Table1.A2" office:value-type="string">
<text:p text:style-name="P11"><text:span text:style-name="Source_20_Text"><text:span text:style-name="T8">cat bigfile | parallel --pipe wc -l</text:span></text:span></text:p>
<text:p text:style-name="P11"><text:span text:style-name="Source_20_Text"><text:span text:style-name="T10">cat bigfile | parallel --pipe wc -l</text:span></text:span></text:p>
</table:table-cell>
</table:table-row>
<table:table-row table:style-name="Table1.3">
<table:table-cell table:style-name="Table1.A2" office:value-type="string">
<text:p text:style-name="P4"><text:span text:style-name="Source_20_Text"><text:span text:style-name="T16">Chop bigfile into one block per CPU thread and </text:span></text:span><text:span text:style-name="Source_20_Text"><text:span text:style-name="T19">grep</text:span></text:span><text:span text:style-name="Source_20_Text"><text:span text:style-name="T16"> for foobar</text:span></text:span><text:span text:style-name="Source_20_Text"><text:span text:style-name="T4"><text:line-break/>parallel -a bigfile --pipepart --block -1 grep foobar</text:span></text:span></text:p>
<text:p text:style-name="P4"><text:span text:style-name="Source_20_Text"><text:span text:style-name="T20">Chop bigfile into one block per CPU thread and </text:span></text:span><text:span text:style-name="Source_20_Text"><text:span text:style-name="T23">grep</text:span></text:span><text:span text:style-name="Source_20_Text"><text:span text:style-name="T20"> for foobar</text:span></text:span><text:span text:style-name="Source_20_Text"><text:span text:style-name="T5"><text:line-break/>parallel -a bigfile --pipepart --block -1 grep foobar</text:span></text:span></text:p>
</table:table-cell>
</table:table-row>
</table:table>
@ -942,23 +980,21 @@
<table:table-column table:style-name="Table2.A"/>
<table:table-row table:style-name="Table2.1">
<table:table-cell table:style-name="Table2.A1" office:value-type="string">
<text:p text:style-name="P23">Read more – <text:span text:style-name="T17">Your command line will love you for it</text:span></text:p>
<text:p text:style-name="P25">Read more – <text:span text:style-name="T21">Your command line will love you for it</text:span></text:p>
</table:table-cell>
</table:table-row>
<table:table-row table:style-name="Table2.3">
<table:table-cell table:style-name="Table2.A2" office:value-type="string">
<text:p text:style-name="P20"><text:span text:style-name="Source_20_Text"><text:span text:style-name="T17">parallel --help; <text:s/></text:span></text:span><text:span text:style-name="Source_20_Text">man parallel; <text:s/></text:span><text:span text:style-name="Source_20_Text"><text:span text:style-name="T17">man parallel_tutorial</text:span></text:span></text:p>
<text:p text:style-name="P21"><text:span text:style-name="Source_20_Text"><text:span text:style-name="T21">parallel --help; <text:s/></text:span></text:span><text:span text:style-name="Source_20_Text">man parallel; <text:s/></text:span><text:span text:style-name="Source_20_Text"><text:span text:style-name="T21">man parallel_tutorial; <text:s/>www.pi.dk/1</text:span></text:span></text:p>
</table:table-cell>
</table:table-row>
<table:table-row table:style-name="Table2.3">
<table:table-cell table:style-name="Table2.A2" office:value-type="string">
<text:p text:style-name="P6"><text:span text:style-name="T13">GNU Parallel 2018 </text:span>https://doi.org/10.5281/zenodo.1146014</text:p>
<text:p text:style-name="P6"><text:span text:style-name="T15">GNU Parallel 2018 </text:span>https://doi.org/10.5281/zenodo.1146014</text:p>
</table:table-cell>
</table:table-row>
</table:table>
<text:p text:style-name="P1"/>
<text:p text:style-name="P1"/>
<text:p text:style-name="P1"/>
<text:p text:style-name="P17">(CC-By-SA) 2019-03-<text:span text:style-name="T28">11</text:span> Ole Tange</text:p>
<text:p text:style-name="P1"/>
</office:text>
</office:body>

8
src/parallel_tutorial.pod
View file

@ -2395,13 +2395,13 @@ Output (the order may be different):
GNU B<parallel> treats each line as a record. If the order of records
is unimportant (e.g. you need all lines processed, but you do not care
which is processed first), then you can use B<--round-robin>. Without
B<--round-robin> GNU B<parallel> will start a command per block; with
B<--round-robin> only the requested number of jobs will be started
which is processed first), then you can use B<--roundrobin>. Without
B<--roundrobin> GNU B<parallel> will start a command per block; with
B<--roundrobin> only the requested number of jobs will be started
(B<--jobs>). The records will then be distributed between the running
jobs:
cat num1000000 | parallel --pipe -j4 --round-robin wc
cat num1000000 | parallel --pipe -j4 --roundrobin wc
Output will be similar to:

2
src/parcat.pod
View file

@ -44,7 +44,7 @@ the files.
=head2 Simple line buffered output
GNU Parallel saves output to tempfiles. If the amount of data is
GNU B<parallel> saves output to tempfiles. If the amount of data is
bigger than the free disk space, then you can use this technique to do
line buffering without saving to disk:

5
testsuite/tests-to-run/parallel-local-0.3s.sh
View file

@ -941,6 +941,11 @@ par_shard() {
perl -pe 's/(.*\d{5,}){3}/OK/'
}
par_fish() {
echo '### https://github.com/fish-shell/fish-shell/issues/5582'
echo OK | stdout fish -c 'parallel --pipe cat'
}
export -f $(compgen -A function | grep par_)
compgen -A function | grep par_ | LC_ALL=C sort |
parallel --timeout 30 -j6 --tag -k --joblog +/tmp/jl-`basename $0` '{} 2>&1'

13
testsuite/tests-to-run/parallel-local-1s.sh
View file

@ -2232,6 +2232,19 @@ CPU revision : 4
:::+ "2-8-8-8" "1-4-8-4" "1-2-4-2" "1-2-2-2" "2-24-48-24" "1-2-2-2" "1-8-8-8"
}
par_null_resume() {
echo '### --null --resume --jl'
log=/tmp/null-resume-$$.log
true > "$log"
printf "%s\n" a b c | parallel --resume -k --jl $log echo
printf "%s\n" a b c | parallel --resume -k --jl $log echo
true > "$log"
printf "%s\0" A B C | parallel --null --resume -k --jl $log echo
printf "%s\0" A B C | parallel --null --resume -k --jl $log echo
rm "$log"
}
export -f $(compgen -A function | grep par_)
compgen -A function | grep par_ | LC_ALL=C sort |
parallel -j6 --tag -k --joblog +/tmp/jl-`basename $0` '{} 2>&1'

6
testsuite/tests-to-run/parallel-local-30s.sh
View file

@ -4,11 +4,11 @@
# Each should be taking 30-100s and be possible to run in parallel
# I.e.: No race conditions, no logins
par_sigterm() {
echo '### Test SIGTERM'
par_sighup() {
echo '### Test SIGHUP'
parallel -k -j5 sleep 15';' echo ::: {1..99} >/tmp/parallel$$ 2>&1 &
A=$!
sleep 29; kill -TERM $A
sleep 29; kill -HUP $A
wait
LC_ALL=C sort /tmp/parallel$$
rm /tmp/parallel$$

17
testsuite/tests-to-run/parallel-local-3s.sh
View file

@ -46,8 +46,8 @@ par_slow_args_generation() {
seq 1 3 | parallel -j1 "sleep 2; echo {}" | parallel -kj2 echo
}
par_kill_term_twice() {
echo '### Are children killed if GNU Parallel receives TERM twice? There should be no sleep at the end'
par_kill_term() {
echo '### Are children killed if GNU Parallel receives TERM? There should be no sleep at the end'
parallel -q bash -c 'sleep 120 & pid=$!; wait $pid' ::: 1 &
T=$!
@ -56,8 +56,17 @@ par_kill_term_twice() {
kill -TERM $T
sleep 1
pstree $$
kill -TERM $T
sleep 1
}
par_kill_hup() {
echo '### Are children killed if GNU Parallel receives HUP? There should be no sleep at the end'
parallel -j 2 -q bash -c 'sleep {} & pid=$!; wait $pid' ::: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 &
T=$!
sleep 2
pstree $$
kill -HUP $T
sleep 2
pstree $$
}

2
testsuite/wanted-results/parallel-local-0.3s
View file

@ -412,6 +412,8 @@ par_file_ending_in_newline ### Hans found a bug giving unitialized variable
par_file_ending_in_newline gzip /tmp/parallel_f1
par_file_ending_in_newline gzip '/tmp/parallel_f2
par_file_ending_in_newline '
par_fish ### https://github.com/fish-shell/fish-shell/issues/5582
par_fish OK
par_halt_on_error_division_by_zero ### --halt-on-error soon,fail=100% with no input should not give division by zero
par_halt_on_error_division_by_zero 0
par_halt_one_job # Halt soon if there is a single job

4
testsuite/wanted-results/parallel-local-100s
View file

@ -409,7 +409,7 @@ par_test_build_and_install rm /tmp/parallel-install/bin/sem || true
par_test_build_and_install ln -s parallel /tmp/parallel-install/bin/sem
par_test_build_and_install make[0]: Leaving directory '/tmp/parallel-00000000/src'
par_test_build_and_install /bin/mkdir -p '/tmp/parallel-install/share/doc/parallel'
par_test_build_and_install /usr/bin/install -c -m 644 parallel.html env_parallel.html sem.html sql.html niceload.html parallel_tutorial.html parallel_book.html parallel_design.html parallel_alternatives.html parcat.html parset.html parallel.texi env_parallel.texi sem.texi sql.texi niceload.texi parallel_tutorial.texi parallel_book.texi parallel_design.texi parallel_alternatives.texi parcat.texi parset.texi parallel.pdf env_parallel.pdf sem.pdf sql.pdf niceload.pdf parallel_tutorial.pdf parallel_book.pdf parallel_design.pdf parallel_alternatives.pdf parcat.pdf parset.pdf '/tmp/parallel-install/share/doc/parallel'
par_test_build_and_install /usr/bin/install -c -m 644 parallel.html env_parallel.html sem.html sql.html niceload.html parallel_tutorial.html parallel_book.html parallel_design.html parallel_alternatives.html parcat.html parset.html parallel.texi env_parallel.texi sem.texi sql.texi niceload.texi parallel_tutorial.texi parallel_book.texi parallel_design.texi parallel_alternatives.texi parcat.texi parset.texi parallel.pdf env_parallel.pdf sem.pdf sql.pdf niceload.pdf parallel_tutorial.pdf parallel_book.pdf parallel_design.pdf parallel_alternatives.pdf parcat.pdf parset.pdf parallel_cheat.pdf '/tmp/parallel-install/share/doc/parallel'
par_test_build_and_install /bin/mkdir -p '/tmp/parallel-install/share/man/man1'
par_test_build_and_install /usr/bin/install -c -m 644 parallel.1 env_parallel.1 sem.1 sql.1 niceload.1 parcat.1 parset.1 '/tmp/parallel-install/share/man/man1'
par_test_build_and_install /bin/mkdir -p '/tmp/parallel-install/share/man/man7'
@ -823,7 +823,7 @@ par_test_build_and_install || echo "Warning: pod2pdf not found. Using old parset
par_test_build_and_install /bin/bash: pod2pdf: command not found
par_test_build_and_install Warning: pod2pdf not found. Using old parset.pdf
par_test_build_and_install /bin/mkdir -p '/tmp/parallel-install/share/doc/parallel'
par_test_build_and_install /usr/bin/install -c -m 644 parallel.html env_parallel.html sem.html sql.html niceload.html parallel_tutorial.html parallel_book.html parallel_design.html parallel_alternatives.html parcat.html parset.html parallel.texi env_parallel.texi sem.texi sql.texi niceload.texi parallel_tutorial.texi parallel_book.texi parallel_design.texi parallel_alternatives.texi parcat.texi parset.texi parallel.pdf env_parallel.pdf sem.pdf sql.pdf niceload.pdf parallel_tutorial.pdf parallel_book.pdf parallel_design.pdf parallel_alternatives.pdf parcat.pdf parset.pdf '/tmp/parallel-install/share/doc/parallel'
par_test_build_and_install /usr/bin/install -c -m 644 parallel.html env_parallel.html sem.html sql.html niceload.html parallel_tutorial.html parallel_book.html parallel_design.html parallel_alternatives.html parcat.html parset.html parallel.texi env_parallel.texi sem.texi sql.texi niceload.texi parallel_tutorial.texi parallel_book.texi parallel_design.texi parallel_alternatives.texi parcat.texi parset.texi parallel.pdf env_parallel.pdf sem.pdf sql.pdf niceload.pdf parallel_tutorial.pdf parallel_book.pdf parallel_design.pdf parallel_alternatives.pdf parcat.pdf parset.pdf parallel_cheat.pdf '/tmp/parallel-install/share/doc/parallel'
par_test_build_and_install pod2man --release='00000000' --center='parallel' \
par_test_build_and_install --section=1 ./parallel.pod > ./parallel.1n \
par_test_build_and_install && mv ./parallel.1n ./parallel.1 \

4
testsuite/wanted-results/parallel-local-10s
View file

@ -27,8 +27,8 @@ par_interactive sleep 0.1; echo opt-p 2 ?...n
par_interactive sleep 0.1; echo opt-p 3 ?...y
par_interactive spawn /tmp/parallel-script-for-expect
par_k ### Test -k
par_k parallel: Warning: Only enough file handles to run 9 jobs in parallel.
par_k parallel: Warning: Running 'parallel -j0 -N 9 --pipe parallel -j0' or
par_k parallel: Warning: Only enough file handles to run 8 jobs in parallel.
par_k parallel: Warning: Running 'parallel -j0 -N 8 --pipe parallel -j0' or
par_k parallel: Warning: raising 'ulimit -n' or 'nofile' in /etc/security/limits.conf
par_k parallel: Warning: or /proc/sys/fs/file-max may help.
par_k begin

7
testsuite/wanted-results/parallel-local-1s
View file

@ -172,6 +172,13 @@ par_incomplete_linebuffer 7
par_incomplete_linebuffer 8
par_incomplete_linebuffer 9
par_incomplete_linebuffer 10
par_null_resume ### --null --resume --jl
par_null_resume a
par_null_resume b
par_null_resume c
par_null_resume A
par_null_resume B
par_null_resume C
par_parset ### test parset
par_parset Put output into $myarray
par_parset 10

2
testsuite/wanted-results/parallel-local12
View file

@ -60,7 +60,9 @@ This helps funding further development; AND IT WON'T COST YOU A CENT.
If you pay 10000 EUR you should feel free to use GNU Parallel without citing.
More about funding GNU Parallel and the citation notice:
https://lists.gnu.org/archive/html/parallel/2013-11/msg00006.html
https://www.gnu.org/software/parallel/parallel_design.html#Citation-notice
https://git.savannah.gnu.org/cgit/parallel.git/tree/doc/citation-notice-faq.txt
If you send a copy of your published article to tange@gnu.org, it will be
mentioned in the release notes of next version of GNU Parallel.