tange/parallel
1
0
Fork 0
mirror of https://git.savannah.gnu.org/git/parallel.git synced 2024-11-24 23:17:55 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Added env_parallel.csh env_parallel.pdksh.

Browse source 
env_parallel.fish now supports arrays.
Documentation updates.
This commit is contained in:
Ole Tange 2016-03-26 10:35:13 +01:00
parent 1cad1cf939
commit e663fe2636
22 changed files with 680 additions and 324 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
README
View file

@ -75,7 +75,7 @@ system is old or Microsoft Windows):
After this you should be able to do:
parallel -j0 ping -nc 3 ::: foss.org.my gnu.org freenetproject.org
parallel -j0 ping -nc 3 ::: qubes-os.org gnu.org freenetproject.org
This will send 3 ping packets to 3 different hosts in parallel and print
the output when they complete.

6
doc/release_new_version
View file

@ -248,6 +248,12 @@ for Big Data Applications https://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumb
* <<Citation needed: Introspecting for RSA Key Material to Assist Intrusion Detection http://ieeexplore.ieee.org/xpl/login.jsp?tp=&arnumber=7331177&url=http%3A%2F%2Fieeexplore.ieee.org%2Fxpls%2Fabs_all.jsp%3Farnumber%3D7331177>>
* GNU Parallel was cited in: Tools and techniques for computational reproducibility http://biorxiv.org/content/biorxiv/early/2016/03/17/022707.full.pdf
* qbatch uses GNU Parallel: https://pypi.python.org/pypi/qbatch/1.0rc2
* FaceCrop uses GNU Parallel: https://github.com/EderSantana/FaceCrop
* Bug fixes and man page updates.
GNU Parallel - For people who live life in the parallel lane.

19
src/Makefile.am
View file

@ -1,15 +1,20 @@
bin_SCRIPTS = parallel sql niceload \
env_parallel env_parallel.bash env_parallel.zsh env_parallel.fish env_parallel.ksh
env_parallel env_parallel.bash env_parallel.zsh env_parallel.fish \
env_parallel.ksh env_parallel.pdksh env_parallel.csh
install-exec-hook:
rm $(DESTDIR)$(bindir)/sem || true
$(LN_S) parallel $(DESTDIR)$(bindir)/sem
if DOCUMENTATION
man_MANS = parallel.1 env_parallel.1 sem.1 sql.1 niceload.1 parallel_tutorial.7 parallel_design.7
doc_DATA = parallel.html env_parallel.html sem.html sql.html niceload.html parallel_tutorial.html parallel_design.html \
parallel.texi env_parallel.texi sem.texi sql.texi niceload.texi parallel_tutorial.texi parallel_design.texi \
parallel.pdf env_parallel.pdf sem.pdf sql.pdf niceload.pdf parallel_tutorial.pdf parallel_design.pdf
man_MANS = parallel.1 env_parallel.1 sem.1 sql.1 niceload.1 \
parallel_tutorial.7 parallel_design.7
doc_DATA = parallel.html env_parallel.html sem.html sql.html niceload.html \
parallel_tutorial.html parallel_design.html \
parallel.texi env_parallel.texi sem.texi sql.texi niceload.texi \
parallel_tutorial.texi parallel_design.texi \
parallel.pdf env_parallel.pdf sem.pdf sql.pdf niceload.pdf \
parallel_tutorial.pdf parallel_design.pdf
endif
# Build documentation file if the tool to build exists.
@ -174,5 +179,7 @@ DISTCLEANFILES = parallel.1 env_parallel.1 sem.1 sql.1 niceload.1 \
EXTRA_DIST = parallel sem sql niceload env_parallel \
env_parallel.bash env_parallel.zsh env_parallel.fish env_parallel.ksh \
sem.pod parallel.pod env_parallel.pod niceload.pod parallel_tutorial.pod parallel_design.pod \
env_parallel.pdksh env_parallel.csh \
sem.pod parallel.pod env_parallel.pod niceload.pod parallel_tutorial.pod \
parallel_design.pod \
$(DISTCLEANFILES)

20
src/Makefile.in
View file

@ -218,12 +218,18 @@ top_build_prefix = @top_build_prefix@
top_builddir = @top_builddir@
top_srcdir = @top_srcdir@
bin_SCRIPTS = parallel sql niceload \
env_parallel env_parallel.bash env_parallel.zsh env_parallel.fish env_parallel.ksh
env_parallel env_parallel.bash env_parallel.zsh env_parallel.fish \
env_parallel.ksh env_parallel.pdksh env_parallel.csh
@DOCUMENTATION_TRUE@man_MANS = parallel.1 env_parallel.1 sem.1 sql.1 niceload.1 parallel_tutorial.7 parallel_design.7
@DOCUMENTATION_TRUE@doc_DATA = parallel.html env_parallel.html sem.html sql.html niceload.html parallel_tutorial.html parallel_design.html \
@DOCUMENTATION_TRUE@ parallel.texi env_parallel.texi sem.texi sql.texi niceload.texi parallel_tutorial.texi parallel_design.texi \
@DOCUMENTATION_TRUE@ parallel.pdf env_parallel.pdf sem.pdf sql.pdf niceload.pdf parallel_tutorial.pdf parallel_design.pdf
@DOCUMENTATION_TRUE@man_MANS = parallel.1 env_parallel.1 sem.1 sql.1 niceload.1 \
@DOCUMENTATION_TRUE@ parallel_tutorial.7 parallel_design.7
@DOCUMENTATION_TRUE@doc_DATA = parallel.html env_parallel.html sem.html sql.html niceload.html \
@DOCUMENTATION_TRUE@ parallel_tutorial.html parallel_design.html \
@DOCUMENTATION_TRUE@ parallel.texi env_parallel.texi sem.texi sql.texi niceload.texi \
@DOCUMENTATION_TRUE@ parallel_tutorial.texi parallel_design.texi \
@DOCUMENTATION_TRUE@ parallel.pdf env_parallel.pdf sem.pdf sql.pdf niceload.pdf \
@DOCUMENTATION_TRUE@ parallel_tutorial.pdf parallel_design.pdf
DISTCLEANFILES = parallel.1 env_parallel.1 sem.1 sql.1 niceload.1 \
parallel_tutorial.7 parallel_design.7 \
@ -236,7 +242,9 @@ DISTCLEANFILES = parallel.1 env_parallel.1 sem.1 sql.1 niceload.1 \
EXTRA_DIST = parallel sem sql niceload env_parallel \
env_parallel.bash env_parallel.zsh env_parallel.fish env_parallel.ksh \
sem.pod parallel.pod env_parallel.pod niceload.pod parallel_tutorial.pod parallel_design.pod \
env_parallel.pdksh env_parallel.csh \
sem.pod parallel.pod env_parallel.pod niceload.pod parallel_tutorial.pod \
parallel_design.pod \
$(DISTCLEANFILES)
all: all-am

8
src/env_parallel
View file

@ -27,14 +27,14 @@ bash: Put this in $HOME/.bashrc: . `which env_parallel.bash`
Supports: aliases, functions, variables, arrays
zsh: Put this in $HOME/.zshrc: . `which env_parallel.zsh`
E.g. by doing: echo '. `which env_parallel.zsh`' >> $HOME/.zshrc
E.g. by doing: echo '. `which env_parallel.zsh`' >> $HOME/.zshenv
Supports: functions, variables, arrays
fish: Put this in $HOME/.config/fish/config.fish:
source (which env_parallel.fish)
E.g. by doing:
echo 'source (which env_parallel.fish)' >> $HOME/.config/fish/config.fish
Supports: aliases, functions, variables
Supports: aliases, functions, variables, arrays
ksh: Put this in $HOME/.kshrc: source `which env_parallel.ksh`
E.g. by doing: echo 'source `which env_parallel.ksh`' >> $HOME/.kshrc
@ -44,6 +44,10 @@ pdksh: Put this in $HOME/.profile: source `which env_parallel.pdksh`
E.g. by doing: echo 'source `which env_parallel.pdksh`' >> $HOME/.profile
Supports: aliases, functions, variables, arrays
csh: Put this in $HOME/.cshrc: source `which env_parallel.csh`
E.g. by doing: echo 'source `which env_parallel.csh`' >> $HOME/.cshrc
Supports: aliases
For details: see man env_parallel
_EOS

4
src/env_parallel.bash
View file

@ -1,3 +1,5 @@
#!/bin/bash
# This file must be sourced in bash:
#
# source `which env_parallel.bash`
@ -5,7 +7,7 @@
# after which 'env_parallel' works
#
#
# Copyright (C) 2007,2008,2009,2010,2011,2012,2013,2014,2015,2016
# Copyright (C) 2016
# Ole Tange and Free Software Foundation, Inc.
#
# This program is free software; you can redistribute it and/or modify

68
src/env_parallel.fish
View file

@ -1,3 +1,5 @@
#!/usr/bin/fish
# This file must be sourced in fish:
#
# source (which env_parallel.fish)
@ -5,7 +7,7 @@
# after which 'env_parallel' works
#
#
# Copyright (C) 2007,2008,2009,2010,2011,2012,2013,2014,2015,2016
# Copyright (C) 2016
# Ole Tange and Free Software Foundation, Inc.
#
# This program is free software; you can redistribute it and/or modify
@ -23,8 +25,70 @@
# or write to the Free Software Foundation, Inc., 51 Franklin St,
# Fifth Floor, Boston, MA 02110-1301 USA
# If you are a fisherman feel free to improve the code
#
# The code needs to deal with variables like:
# set funky (perl -e 'print pack "c*", 2..254')
#
# Problem:
# Tell the difference between:
# set tmp "a' 'b' 'c"
# set tmparr1 "a' 'b" 'c'
# set tmparr2 'a' "b' 'c"
# The output from `set` is exactly the same.
# Solution:
# for-loop for each variable. Each value is separated with a
# separator.
function env_parallel
setenv PARALLEL_ENV (begin; functions -n | perl -pe 's/,/\n/g' | while read d; functions $d; end; perl -e 'print map { "$_///$ENV{$_}\n"} grep !/^(PWD|SHLVL|PATH)$/, keys %ENV'| sh -c 'parallel --shellquote' | perl -pe 's:^([^/]+)///:setenv $1 :'; end |perl -pe 's/\001/\\cb/g;s/\n/\001/')
setenv PARALLEL_ENV (
begin;
# Export function definitions
functions -n | perl -pe 's/,/\n/g' | while read d; functions $d; end;
# Convert scalar vars to fish \XX quoting
eval (set -L | perl -ne 'chomp;
($name,$val)=split(/ /,$_,2);
$name=~/^(COLUMNS|FISH_VERSION|LINES|PWD|SHLVL|_|history|status|version)$/ and next;
if($val=~/^'"'"'/) { next; }
print "set $name \"\$$name\";\n";
')
# Generate commands to set scalar variables
begin;
for v in (set -n);
# Separate variables with the string: \000
eval "for i in \$$v;
echo -n $v \$i;
perl -e print\\\"\\\\0\\\";
end;"
end;
# A final line to flush the last variable in Perl
perl -e print\"\\0\";
end | perl -0 -ne '
# Remove separator string
chop;
($name,$val)=split(/ /,$_,2);
# Ignore read-only vars
$name=~/^(COLUMNS|FISH_VERSION|LINES|PWD|SHLVL|_|history|status|version)$/ and next;
# Quote $val
$val=~s/[\002-\011\013-\032\\\#\?\`\(\)\{\}\[\]\^\*\<\=\>\~\|\; \"\!\$\&\202-\377]/\\\$&/go;
# Quote single quote
$val=~s/'"'"'/\\\$&/go;
# Quote newline as '\n'
$val =~ s/[\n]/\\\n/go;
if($name ne $last and $last) {
# The $name is different, so this is a new variable.
# Print the last one.
# Separate list elements by 2 spaces
$"=" ";
print "set $last @qval;\n";
@qval=();
}
push @qval,$val;
$last=$name;
';
end |perl -pe 's/\001/\\cb/g and print STDERR "env_parallel: Warning: ASCII value 1 in variables is not supported\n";
s/\n/\001/'
)
parallel $argv;
set -e PARALLEL_ENV
end

4
src/env_parallel.ksh
View file

@ -1,3 +1,5 @@
#!/usr/bin/ksh
# This file must be sourced in ksh:
#
# source `which env_parallel.ksh`
@ -5,7 +7,7 @@
# after which 'env_parallel' works
#
#
# Copyright (C) 2007,2008,2009,2010,2011,2012,2013,2014,2015,2016
# Copyright (C) 2016
# Ole Tange and Free Software Foundation, Inc.
#
# This program is free software; you can redistribute it and/or modify

4
src/env_parallel.pdksh
View file

@ -1,3 +1,5 @@
#!/bin/pdksh
# This file must be sourced in ksh:
#
# source `which env_parallel.ksh`
@ -5,7 +7,7 @@
# after which 'env_parallel' works
#
#
# Copyright (C) 2007,2008,2009,2010,2011,2012,2013,2014,2015,2016
# Copyright (C) 2016
# Ole Tange and Free Software Foundation, Inc.
#
# This program is free software; you can redistribute it and/or modify

120
src/env_parallel.pod
View file

@ -20,6 +20,11 @@ environment to GNU Parallel.
If the shell function is not loaded, a dummy script will be run
instead that explains how to install the function.
B<env_parallel> only works if the size of the current environment is
smaller than half of the max size and smaller than 25% of the max if
running remotely. E.g. The max size of Bash's environment is 256 KB,
so B<env_parallel> will fail if B<set | wc -c> is bigger than 128 KB.
=head1 OPTIONS
@ -30,63 +35,102 @@ Same as GNU B<parallel>.
=head2 Bash
Installation
Put this in $HOME/.bashrc:
. `which env_parallel.bash`
E.g. by doing:
echo '. `which env_parallel.bash`' >> $HOME/.bashrc
=over 8
=item aliases
alias myecho=echo
env_parallel myecho ::: test
env_parallel -S server myecho ::: test
=item functions
myfunc() { echo $*; }
env_parallel myfunc ::: test
env_parallel -S server myfunc ::: test
=item variables
myvar=test
env_parallel echo '$myvar' ::: test
env_parallel -S server echo '$myvar' ::: test
=item arrays
myarray=(foo bar baz)
env_parallel echo '${myarray[{}]}' ::: 0 1 2
env_parallel -S server echo '${myarray[{}]}' ::: 0 1 2
=back
=head2 Zsh
Installation
Put this in $HOME/.zshrc:
. `which env_parallel.zsh`
E.g. by doing:
echo '. `which env_parallel.zsh`' >> $HOME/.zshenv
=over 8
=item aliases
Not supported
Not supported - Zsh does not support aliases defined in the same
parsing as they are used.
=item functions
myfunc() { echo $*; }
env_parallel myfunc ::: test
env_parallel -S server myfunc ::: test
=item variables
myvar=test
env_parallel echo '$myvar' ::: test
env_parallel -S server echo '$myvar' ::: test
=item arrays
myarray=(foo bar baz)
env_parallel echo '${myarray[{}]}' ::: 1 2 3
env_parallel -S server echo '${myarray[{}]}' ::: 1 2 3
=back
=head2 fish
Installation
Put this in $HOME/.config/fish/config.fish:
source (which env_parallel.fish)
E.g. by doing:
echo 'source (which env_parallel.fish)' >> $HOME/.config/fish/config.fish
=over 8
=item aliases
alias myecho=echo
env_parallel myecho ::: test
env_parallel -S server myecho ::: test
=item functions
@ -94,67 +138,133 @@ Not supported
echo $argv
end
env_parallel myfunc ::: test
env_parallel -S server myfunc ::: test
=item variables
set myvar test
env_parallel echo '$myvar' ::: test
env_parallel -S server echo '$myvar' ::: test
=item arrays
Not supported
set myarray foo bar baz
env_parallel echo '$myarray[{}]' ::: 1 2 3
env_parallel -S server echo '$myarray[{}]' ::: 1 2 3
=back
=head2 ksh
Installation
Put this in $HOME/.kshrc:
source `which env_parallel.ksh`
E.g. by doing:
echo 'source `which env_parallel.ksh`' >> $HOME/.kshrc
=over 8
=item aliases
alias myecho=echo
env_parallel myecho ::: test
env_parallel -S server myecho ::: test
=item functions
myfunc() { echo $*; }
env_parallel myfunc ::: test
env_parallel -S server myfunc ::: test
=item variables
myvar=test
env_parallel echo '$myvar' ::: test
env_parallel -S server echo '$myvar' ::: test
=item arrays
myarray=(foo bar baz)
env_parallel echo '${myarray[{}]}' ::: 0 1 2
env_parallel -S server echo '${myarray[{}]}' ::: 0 1 2
=back
=head2 pdksh
Installation
Put this in $HOME/.profile:
source `which env_parallel.pdksh`
E.g. by doing:
echo 'source `which env_parallel.pdksh`' >> $HOME/.profile
=over 8
=item aliases
alias myecho=echo
env_parallel myecho ::: test
env_parallel -S server myecho ::: test
=item functions
myfunc() { echo $*; }
env_parallel myfunc ::: test
env_parallel -S server myfunc ::: test
=item variables
myvar=test
env_parallel echo '$myvar' ::: test
env_parallel -S server echo '$myvar' ::: test
=item arrays
myarray=(foo bar baz)
env_parallel echo '${myarray[{}]}' ::: 0 1 2
env_parallel -S server echo '${myarray[{}]}' ::: 0 1 2
=back
=head2 csh
Installation
Put this in $HOME/.cshrc:
source `which env_parallel.csh`
E.g. by doing:
echo 'source `which env_parallel.csh`' >> $HOME/.cshrc
=over 8
=item aliases
alias myecho echo
env_parallel myecho ::: test
env_parallel -S server myecho ::: test
=item functions
Not supported by B<csh>.
=item variables
Not supported
=item arrays
Not supported
=back
@ -187,9 +297,9 @@ the manual of B<xargs> from GNU findutils 4.4.2.
=head1 LICENSE
Copyright (C) 2007,2008,2009,2010,2011,2012,2013,2014,2015 Free
Software Foundation, Inc.
v
Copyright (C) 2016
Ole Tange and Free Software Foundation, Inc.
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 3 of the License, or

6
src/env_parallel.zsh
View file

@ -1,3 +1,5 @@
#!/usr/bin/zsh
# This file must be sourced in zsh:
#
# source =env_parallel.zsh
@ -5,7 +7,7 @@
# after which 'env_parallel' works
#
#
# Copyright (C) 2007,2008,2009,2010,2011,2012,2013,2014,2015,2016
# Copyright (C) 2016
# Ole Tange and Free Software Foundation, Inc.
#
# This program is free software; you can redistribute it and/or modify
@ -24,7 +26,7 @@
# Fifth Floor, Boston, MA 02110-1301 USA
env_parallel() {
export PARALLEL_ENV="$(alias | perl -pe 's/^/alias /';typeset -p |
export PARALLEL_ENV="$(alias | perl -pe 's/^/alias /'; typeset -p |
grep -aFvf <(typeset -pr) |
egrep -iav 'ZSH_EVAL_CONTEXT|LINENO=| _=|aliases|^typeset [a-z_]+$'|
egrep -av '^(typeset -A (commands|functions|options)|typeset IFS=|..$)|cyan';

3
src/parallel
View file

@ -7491,7 +7491,6 @@ sub start {
$ENV{'PARALLEL_TMP'} = ::tmpname("par");
::debug("run", $Global::total_running, " processes . Starting (",
$job->seq(), "): $command\n");
if($opt::pipe) {
my ($stdin_fh) = ::gensym();
$pid = open3_setpgrp($stdin_fh,$stdout_fh,$stderr_fh,$command);
@ -9166,6 +9165,8 @@ sub get {
my $ret = $self->{'arg_sub_queue'}->get();
if($ret and
grep { index($_->orig(),"\0") > 0 } map { @$_ } $ret) {
# Allow for \0 in position 0 because GNU Parallel uses "\0"
# to mean no-string
::warning("a NUL character occurred in the input.",
"It cannot be passed through in the argument list.",
"Did you mean to use the --null option?");

50
src/parallel.pod
View file

@ -11,8 +11,8 @@ parallel - build and execute shell command lines from standard input in parallel
B<parallel> [options] [I<command> [arguments]] < list_of_arguments
B<parallel> [options] [I<command> [arguments]] ( B<:::> arguments |
B<::::> argfile(s) ) ...
B<parallel> [options] [I<command> [arguments]] ( B<:::> arguments | B<:::+> arguments |
B<::::> argfile(s) | B<::::+> argfile(s) ) ...
B<parallel> --semaphore [options] I<command>
@ -326,6 +326,18 @@ B<:::> and B<::::> can be mixed. So these are equivalent:
::: 1 2 3
=item B<:::+> I<arguments> (alpha testing)
Like B<:::> but linked like B<--xapply> to the previous input source.
Contrary to B<--xapply> values do not wrap: The shortest input source
determines the length.
Example:
parallel echo ::: a b c :::+ 1 2 3 ::: X Y :::+ 11 22
=item B<::::> I<argfiles>
Another way to write B<-a> I<argfile1> B<-a> I<argfile2> ...
@ -335,6 +347,14 @@ B<:::> and B<::::> can be mixed.
See B<-a>, B<:::> and B<--xapply>.
=item B<::::+> I<argfiles> (alpha testing)
Like B<::::+> but linked like B<--xapply> to the previous input source.
Contrary to B<--xapply> values do not wrap: The shortest input source
determines the length.
=item B<--null>
=item B<-0>
@ -1665,7 +1685,7 @@ Like this:
#!/usr/bin/parallel --shebang -r traceroute
foss.org.my
qubes-os.org
debian.org
freenetproject.org
@ -1675,7 +1695,7 @@ On FreeBSD B<env> is needed:
#!/usr/bin/env -S parallel --shebang -r traceroute
foss.org.my
qubes-os.org
debian.org
freenetproject.org
@ -2770,9 +2790,9 @@ printed as soon as possible you can use B<-u>.
Compare the output of:
parallel traceroute ::: foss.org.my debian.org freenetproject.org
parallel --line-buffer traceroute ::: foss.org.my debian.org freenetproject.org
parallel -u traceroute ::: foss.org.my debian.org freenetproject.org
parallel traceroute ::: qubes-os.org debian.org freenetproject.org
parallel --line-buffer traceroute ::: qubes-os.org debian.org freenetproject.org
parallel -u traceroute ::: qubes-os.org debian.org freenetproject.org
=head1 EXAMPLE: Tag output lines
@ -2781,12 +2801,12 @@ GNU B<parallel> groups the output lines, but it can be hard to see
where the different jobs begin. B<--tag> prepends the argument to make
that more visible:
parallel --tag traceroute ::: foss.org.my debian.org freenetproject.org
parallel --tag traceroute ::: qubes-os.org debian.org freenetproject.org
B<--tag> works with B<--line-buffer> but not with B<-u>:
parallel --tag --line-buffer traceroute \
::: foss.org.my debian.org freenetproject.org
::: qubes-os.org debian.org freenetproject.org
Check the uptime of the servers in I<~/.parallel/sshloginfile>:
@ -2810,17 +2830,17 @@ If you remove B<-k> some of the lines may come out in the wrong order.
Another example is B<traceroute>:
parallel traceroute ::: foss.org.my debian.org freenetproject.org
parallel traceroute ::: qubes-os.org debian.org freenetproject.org
will give traceroute of foss.org.my, debian.org and
will give traceroute of qubes-os.org, debian.org and
freenetproject.org, but it will be sorted according to which job
completed first.
To keep the order the same as input run:
parallel -k traceroute ::: foss.org.my debian.org freenetproject.org
parallel -k traceroute ::: qubes-os.org debian.org freenetproject.org
This will make sure the traceroute to foss.org.my will be printed
This will make sure the traceroute to qubes-os.org will be printed
first.
A bit more complex example is downloading a huge file in chunks in
@ -4730,7 +4750,7 @@ Copyright (C) 2007-10-18 Ole Tange, http://ole.tange.dk
Copyright (C) 2008,2009,2010 Ole Tange, http://ole.tange.dk
Copyright (C) 2010,2011,2012,2013,2014,2015 Ole Tange,
Copyright (C) 2010,2011,2012,2013,2014,2015,2016 Ole Tange,
http://ole.tange.dk and Free Software Foundation, Inc.
Parts of the manual concerning B<xargs> compatibility is inspired by
@ -4739,7 +4759,7 @@ the manual of B<xargs> from GNU findutils 4.4.2.
=head1 LICENSE
Copyright (C) 2007,2008,2009,2010,2011,2012,2013,2014,2015 Free
Copyright (C) 2007,2008,2009,2010,2011,2012,2013,2014,2015,2016 Free
Software Foundation, Inc.
This program is free software; you can redistribute it and/or modify

68
src/parallel_design.pod
View file

@ -6,8 +6,8 @@
This document describes design decisions made in the development of
GNU B<parallel> and the reasoning behind them. It will give an
overview of why some of the code looks like it does, and help new
maintainers understand the code better.
overview of why some of the code looks the way it does, and will help
new maintainers understand the code better.
=head2 One file program
@ -30,10 +30,10 @@ is CentOS 3.9 and Perl 5.8.0.
=head2 Exponentially back off
GNU B<parallel> busy waits. This is because the reason why a job is
not started may be due to load average, and thus it will not make
sense to wait for a job to finish. Instead the load average must be
checked again. Load average is not the only reason: B<--timeout> has a
similar problem.
not started may be due to load average (when using B<--load>), and
thus it will not make sense to wait for a job to finish. Instead the
load average must be checked again. Load average is not the only
reason: B<--timeout> has a similar problem.
To not burn up too much CPU GNU B<parallel> sleeps exponentially
longer and longer if nothing happens, maxing out at 1 second.
@ -169,6 +169,20 @@ to complete; remove {} and exit with the exit code from I<input>.
It is made this way to be compatible with B<*csh>.
=item --pipepart
< file perl -e 'while(@ARGV) {
sysseek(STDIN,shift,0) || die;
$left = shift;
while($read = sysread(STDIN,$buf, ($left > 131072 ? 131072 : $left))){
$left -= $read;
syswrite(STDOUT,$buf);
}
}' startposition length
This will read I<length> bytes from I<file> starting at
I<startposition> and send it to STDOUT.
=item --sshlogin I<sln>
ssh I<sln> I<shell quoted input>
@ -446,14 +460,13 @@ The wrapper looks like this:
Transferring of variables and functions given by B<--env> is done by
running a Perl script remotely that calls the actual command. The Perl
script sets B<$ENV{>I<variable>B<}> to the correct value before exec'ing the a
shell that runs the function definition followed by the actual
command.
script sets B<$ENV{>I<variable>B<}> to the correct value before
exec'ing the a shell that runs the function definition followed by the
actual command.
B<env_parallel> (mentioned in the man page) copies the full current
environment into the environment variable B<PARALLEL_ENV>. This
variable is picked up by GNU B<parallel> and used to create the Perl
script mentioned above.
The function B<env_parallel> copies the full current environment into
the environment variable B<PARALLEL_ENV>. This variable is picked up
by GNU B<parallel> and used to create the Perl script mentioned above.
=head2 Base64 encoded bzip2
@ -747,7 +760,8 @@ What can be done about it?
=back
Unfortunately it is not always possible to predict the root cause of the error.
Unfortunately it is not always possible to predict the root cause of
the error.
=head2 Computation of load
@ -766,8 +780,8 @@ B<ps> and put the result in a file, which is then used next time.
GNU B<parallel> kills jobs. It can be due to B<--memfree>, B<--halt>,
or when GNU B<parallel> meets a condition from which it cannot
recover. Every job is started as its own process group. This way any
(grand)*children will get killed, too. The process group is killed with
the specification mentioned in B<--termseq>.
(grand)*children will get killed, too. The process group is killed
with the specification mentioned in B<--termseq>.
=head2 SQL interface
@ -789,6 +803,10 @@ respectively.
The Signal column has been renamed to _Signal due to Signal being a
reserved word in MySQL.
=head2 Logo
The logo is inspired by the Cafe Wall illusion. The font is DejaVu
Sans.
=head1 Ideas for new design
@ -816,16 +834,13 @@ Will that require 2x record size memory?
Will that require 2x block size memory?
=head2 Logo
The logo is inpired by the Cafe Wall illusion. The font is DejaVu Sans.
=head1 Historical decisions
=head2 --tollef
You can read about the history of GNU B<parallel> on https://www.gnu.org/software/parallel/history.html
You can read about the history of GNU B<parallel> on
https://www.gnu.org/software/parallel/history.html
B<--tollef> was included to make GNU B<parallel> switch compatible
with the parallel from moreutils (which is made by Tollef Fog
@ -833,11 +848,12 @@ Heen). This was done so that users of that parallel easily could port
their use to GNU B<parallel>: Simply set B<PARALLEL="--tollef"> and
that would be it.
But several distributions chose to make B<--tollef> global (by putting it
into /etc/parallel/config), and that caused much confusion when people
tried out the examples from GNU B<parallel>'s man page and these did
not work. The users became frustrated because the distribution did
not make it clear to them that it has made B<--tollef> global.
But several distributions chose to make B<--tollef> global (by putting
it into /etc/parallel/config) without making the users aware of this,
and that caused much confusion when people tried out the examples from
GNU B<parallel>'s man page and these did not work. The users became
frustrated because the distribution did not make it clear to them that
it has made B<--tollef> global.
So to lessen the frustration and the resulting support, B<--tollef>
was obsoleted 20130222 and removed one year later.

242
src/parallel_tutorial.html
View file

@ -1746,7 +1746,25 @@
<h2 id="Transferring-environment-variables-and-functions">Transferring environment variables and functions</h2>
<p>Using <b>--env</b> GNU <b>parallel</b> can transfer an environment variable to the remote system.</p>
<p><b>env_parallel</b> is a shell function that transfers all aliases, functions, variables, and arrays. You active it by running:</p>
<pre><code> source `which env_parallel.bash`</code></pre>
<p>Replace <b>bash</b> with the shell you use.</p>
<p>Now you can use <b>env_parallel</b> instead of <b>parallel</b> and still have your environment:</p>
<pre><code> alias myecho=echo
myvar=&quot;Joe&#39;s var is&quot;
env_parallel -S $SERVER1 &#39;myecho $myvar&#39; ::: green</code></pre>
<p>Output:</p>
<pre><code> Joe&#39;s var is green</code></pre>
<p>The disadvantage is that if your environment is huge <b>env_parallel</b> will fail.</p>
<p>When <b>env_parallel</b> fails, you can still use <b>--env</b> to tell GNU <b>parallel</b> to transfer an environment variable to the remote system.</p>
<pre><code> MYVAR=&#39;foo bar&#39;
export MYVAR
@ -1797,26 +1815,19 @@
<h2 id="Showing-what-is-actually-run">Showing what is actually run</h2>
<p><b>--verbose</b> will show the command that would be run on the local machine. When a job is run on a remote machine, this is wrapped with <b>ssh</b> and possibly transferring files and environment variables, setting the workdir, and setting <b>--nice</b> value. <b>-vv</b> shows all of this.</p>
<p><b>--verbose</b> will show the command that would be run on the local machine.</p>
<pre><code> parallel -vv -S $SERVER1 echo ::: bar</code></pre>
<p>When using <b>--cat</b>, <b>--pipepart</b>, or when a job is run on a remote machine, the command is wrapped with helper scripts. <b>-vv</b> shows all of this.</p>
<pre><code> parallel -vv --pipepart wc :::: num30000</code></pre>
<p>Output:</p>
<pre><code> ssh lo -- exec perl -e \&#39;&#39;@GNU_Parallel=(&quot;use&quot;,&quot;IPC::Open3;&quot;,&quot;use&quot;,&quot;MIME::Base64&quot;);
eval&quot;@GNU_Parallel&quot;;my$eval=decode_base64(join&quot;&quot;,@ARGV);eval$eval;&#39;\&#39;
JEVOVnsiUEFSQUxMRUxfUElEIn09IjI3MzQiOyRFTlZ7IlBBUkFMTEVMX1NFUSJ9PSIx
IjskYmFzaGZ1bmMgPSAiIjtAQVJHVj0iZWNobyBiYXIiOyRzaGVsbD0iJEVOVntTSEVM
TH0iOyR0bXBkaXI9Ii90bXAiOyRuaWNlPTA7ZG97JEVOVntQQVJBTExFTF9UTVB9PSR0
bXBkaXIuIi9wYXIiLmpvaW4iIixtYXB7KDAuLjksImEiLi4ieiIsIkEiLi4iWiIpW3Jh
bmQoNjIpXX0oMS4uNSk7fXdoaWxlKC1lJEVOVntQQVJBTExFTF9UTVB9KTskU0lHe0NI
TER9PXN1YnskZG9uZT0xO307JHBpZD1mb3JrO3VubGVzcygkcGlkKXtzZXRwZ3JwO2V2
YWx7c2V0cHJpb3JpdHkoMCwwLCRuaWNlKX07ZXhlYyRzaGVsbCwiLWMiLCgkYmFzaGZ1
bmMuIkBBUkdWIik7ZGllImV4ZWM6JCFcbiI7fWRveyRzPSRzPDE/MC4wMDErJHMqMS4w
MzokcztzZWxlY3QodW5kZWYsdW5kZWYsdW5kZWYsJHMpO311bnRpbCgkZG9uZXx8Z2V0
cHBpZD09MSk7a2lsbChTSUdIVVAsLSR7cGlkfSl1bmxlc3MkZG9uZTt3YWl0O2V4aXQo
JD8mMTI3PzEyOCsoJD8mMTI3KToxKyQ/Pj44KQ==;
bar</code></pre>
<pre><code> &lt;num30000 perl -e &#39;while(@ARGV) { sysseek(STDIN,shift,0) || die;
$left = shift; while($read = sysread(STDIN,$buf, ($left &gt; 131072
? 131072 : $left))){ $left -= $read; syswrite(STDOUT,$buf); } }&#39;
0 0 0 168894 | (wc)
30000 30000 168894</code></pre>
<p>When the command gets more complex, the output is so hard to read, that it is only useful for debugging:</p>
@ -1828,44 +1839,40 @@
<p>Output will be similar to:</p>
<pre><code> ( ssh lo -- mkdir -p ./.parallel/tmp/hk-3492-1;rsync --protocol 30
-rlDzR -essh ./abc-file lo:./.parallel/tmp/hk-3492-1 );ssh lo --
exec perl -e \&#39;&#39;@GNU_Parallel=(&quot;use&quot;,&quot;IPC::Open3;&quot;,&quot;use&quot;,&quot;MIME::Base64&quot;);
<pre><code> ( ssh server -- mkdir -p ./.parallel/tmp/aspire-1928520-1;rsync
--protocol 30 -rlDzR -essh ./abc-file
server:./.parallel/tmp/aspire-1928520-1 );ssh server -- exec perl -e
\&#39;&#39;@GNU_Parallel=(&quot;use&quot;,&quot;IPC::Open3;&quot;,&quot;use&quot;,&quot;MIME::Base64&quot;);
eval&quot;@GNU_Parallel&quot;;my$eval=decode_base64(join&quot;&quot;,@ARGV);eval$eval;&#39;\&#39;
c3lzdGVtKCJta2RpciIsIi1wIiwiLS0iLCIucGFyYWxsZWwvdG1wL2hrLTM0OTItMSIp
OyBjaGRpciAiLnBhcmFsbGVsL3RtcC9oay0zNDkyLTEiIHx8cHJpbnQoU1RERVJSICJw
YXJhbGxlbDogQ2Fubm90IGNoZGlyIHRvIC5wYXJhbGxlbC90bXAvaGstMzQ5Mi0xXG4i
KSAmJiBleGl0IDI1NTskRU5WeyJHUEdfQUdFTlRfSU5GTyJ9PSIvdG1wL2dwZy10WjVI
U0QvUy5ncGctYWdlbnQ6MjM5NzoxIjskRU5WeyJQQVJBTExFTF9TRVEifT0iMSI7JEVO
VnsiU1FMSVRFVEJMIn09InNxbGl0ZTM6Ly8vJTJGdG1wJTJGcGFyYWxsZWwuZGIyL3Bh
cnNxbDIiOyRFTlZ7IlBBUkFMTEVMX1BJRCJ9PSIzNDkyIjskRU5WeyJTUUxJVEUifT0i
c3FsaXRlMzovLy8lMkZ0bXAlMkZwYXJhbGxlbC5kYjIiOyRFTlZ7IlBBUkFMTEVMX1BJ
RCJ9PSIzNDkyIjskRU5WeyJQQVJBTExFTF9TRVEifT0iMSI7QGJhc2hfZnVuY3Rpb25z
PXF3KG15X2Z1bmMzKTsgaWYoJEVOVnsiU0hFTEwifT1+L2NzaC8pIHsgcHJpbnQgU1RE
RVJSICJDU0gvVENTSCBETyBOT1QgU1VQUE9SVCBuZXdsaW5lcyBJTiBWQVJJQUJMRVMv
RlVOQ1RJT05TLiBVbnNldCBAYmFzaF9mdW5jdGlvbnNcbiI7IGV4ZWMgImZhbHNlIjsg
fSAKJGJhc2hmdW5jID0gIm15X2Z1bmMzKCkgeyAgZWNobyBpbiBteV9mdW5jIFwkMSA+
IFwkMS5vdXQKfTtleHBvcnQgLWYgbXlfZnVuYzMgPi9kZXYvbnVsbDsiO0BBUkdWPSJt
eV9mdW5jMyBhYmMtZmlsZSI7JHNoZWxsPSIkRU5We1NIRUxM
fSI7JHRtcGRpcj0iL3RtcCI7JG5pY2U9MTc7ZG97JEVOVntQQVJBTExFTF9UTVB9PSR0
bXBkaXIuIi9wYXIiLmpvaW4iIixtYXB7KDAuLjksImEiLi4ieiIsIkEiLi4iWiIpW3Jh
bmQoNjIpXX0oMS4uNSk7fXdoaWxlKC1lJEVOVntQQVJBTExFTF9UTVB9KTskU0lHe0NI
TER9PXN1YnskZG9uZT0xO307JHBpZD1mb3JrO3VubGVzcygkcGlkKXtzZXRwZ3JwO2V2
YWx7c2V0cHJpb3JpdHkoMCwwLCRuaWNlKX07ZXhlYyRzaGVsbCwiLWMiLCgkYmFzaGZ1
bmMuIkBBUkdWIik7ZGllImV4ZWM6JCFcbiI7fWRveyRzPSRzPDE/MC4wMDErJHMqMS4w
MzokcztzZWxlY3QodW5kZWYsdW5kZWYsdW5kZWYsJHMpO311bnRpbCgkZG9uZXx8Z2V0
cHBpZD09MSk7a2lsbChTSUdIVVAsLSR7cGlkfSl1bmxlc3MkZG9uZTt3YWl0O2V4aXQo
JD8mMTI3PzEyOCsoJD8mMTI3KToxKyQ/Pj44KQ==;_EXIT_status=$?;
mkdir -p ./.; rsync --protocol 30 --rsync-path=cd\
./.parallel/tmp/hk-3492-1/./.\;\ rsync -rlDzR -essh
lo:./abc-file.out ./.;ssh lo -- \(rm\ -f\
./.parallel/tmp/hk-3492-1/abc-file\;\ sh\ -c\ \&#39;rmdir\
./.parallel/tmp/hk-3492-1/\ ./.parallel/tmp/\ ./.parallel/\
2\&gt;/dev/null\&#39;\;rm\ -rf\ ./.parallel/tmp/hk-3492-1\;\);ssh lo --
\(rm\ -f\ ./.parallel/tmp/hk-3492-1/abc-file.out\;\ sh\ -c\ \&#39;rmdir\
./.parallel/tmp/hk-3492-1/\ ./.parallel/tmp/\ ./.parallel/\
2\&gt;/dev/null\&#39;\;rm\ -rf\ ./.parallel/tmp/hk-3492-1\;\);ssh lo -- rm
-rf .parallel/tmp/hk-3492-1; exit $_EXIT_status;</code></pre>
c3lzdGVtKCJta2RpciIsIi1wIiwiLS0iLCIucGFyYWxsZWwvdG1wL2FzcGlyZS0xOTI4N
TsgY2hkaXIgIi5wYXJhbGxlbC90bXAvYXNwaXJlLTE5Mjg1MjAtMSIgfHxwcmludChTVE
BhcmFsbGVsOiBDYW5ub3QgY2hkaXIgdG8gLnBhcmFsbGVsL3RtcC9hc3BpcmUtMTkyODU
iKSAmJiBleGl0IDI1NTskRU5WeyJPTERQV0QifT0iL2hvbWUvdGFuZ2UvcHJpdmF0L3Bh
IjskRU5WeyJQQVJBTExFTF9QSUQifT0iMTkyODUyMCI7JEVOVnsiUEFSQUxMRUxfU0VRI
0BiYXNoX2Z1bmN0aW9ucz1xdyhteV9mdW5jMyk7IGlmKCRFTlZ7IlNIRUxMIn09fi9jc2
ByaW50IFNUREVSUiAiQ1NIL1RDU0ggRE8gTk9UIFNVUFBPUlQgbmV3bGluZXMgSU4gVkF
TL0ZVTkNUSU9OUy4gVW5zZXQgQGJhc2hfZnVuY3Rpb25zXG4iOyBleGVjICJmYWxzZSI7
YXNoZnVuYyA9ICJteV9mdW5jMygpIHsgIGVjaG8gaW4gbXlfZnVuYyBcJDEgPiBcJDEub
Xhwb3J0IC1mIG15X2Z1bmMzID4vZGV2L251bGw7IjtAQVJHVj0ibXlfZnVuYzMgYWJjLW
RzaGVsbD0iJEVOVntTSEVMTH0iOyR0bXBkaXI9Ii90bXAiOyRuaWNlPTE3O2RveyRFTlZ
MRUxfVE1QfT0kdG1wZGlyLiIvcGFyIi5qb2luIiIsbWFweygwLi45LCJhIi4uInoiLCJB
KVtyYW5kKDYyKV19KDEuLjUpO313aGlsZSgtZSRFTlZ7UEFSQUxMRUxfVE1QfSk7JFNJ
fT1zdWJ7JGRvbmU9MTt9OyRwaWQ9Zm9yazt1bmxlc3MoJHBpZCl7c2V0cGdycDtldmFse
W9yaXR5KDAsMCwkbmljZSl9O2V4ZWMkc2hlbGwsIi1jIiwoJGJhc2hmdW5jLiJAQVJHVi
JleGVjOiQhXG4iO31kb3skcz0kczwxPzAuMDAxKyRzKjEuMDM6JHM7c2VsZWN0KHVuZGV
mLHVuZGVmLCRzKTt9dW50aWwoJGRvbmV8fGdldHBwaWQ9PTEpO2tpbGwoU0lHSFVQLC0k
dW5sZXNzJGRvbmU7d2FpdDtleGl0KCQ/JjEyNz8xMjgrKCQ/JjEyNyk6MSskPz4+OCk=;
_EXIT_status=$?; mkdir -p ./.; rsync --protocol 30 --rsync-path=cd\
./.parallel/tmp/aspire-1928520-1/./.\;\ rsync -rlDzR -essh
server:./abc-file.out ./.;ssh server -- \(rm\ -f\
./.parallel/tmp/aspire-1928520-1/abc-file\;\ sh\ -c\ \&#39;rmdir\
./.parallel/tmp/aspire-1928520-1/\ ./.parallel/tmp/\ ./.parallel/\
2\&gt;/dev/null\&#39;\;rm\ -rf\ ./.parallel/tmp/aspire-1928520-1\;\);ssh
server -- \(rm\ -f\ ./.parallel/tmp/aspire-1928520-1/abc-file.out\;\
sh\ -c\ \&#39;rmdir\ ./.parallel/tmp/aspire-1928520-1/\ ./.parallel/tmp/\
./.parallel/\ 2\&gt;/dev/null\&#39;\;rm\ -rf\
./.parallel/tmp/aspire-1928520-1\;\);ssh server -- rm -rf
.parallel/tmp/aspire-1928520-1; exit $_EXIT_status;</code></pre>
<h1 id="Saving-to-an-SQL-base-advanced">Saving to an SQL base (advanced)</h1>
@ -1901,6 +1908,18 @@
<p>A proper CSV reader (like LibreOffice or R&#39;s read.csv) will read this format correctly - even with fields containing newlines as above.</p>
<p>If the output is big you may want to put it into files using <b>--results</b>:</p>
<pre><code> parallel --results outdir --sqlandworker csv:////%2Ftmp%2Flog2.csv seq ::: 10 ::: 12 13 14
cat /tmp/log2.csv</code></pre>
<p>Output will be similar to:</p>
<pre><code> Seq,Host,Starttime,JobRuntime,Send,Receive,Exitval,_Signal,Command,V1,V2,Stdout,Stderr
1,:,1458824738.287,0.029,0,9,0,0,&quot;seq 10 12&quot;,10,12,outdir/1/10/2/12/stdout,outdir/1/10/2/12/stderr
2,:,1458824738.298,0.025,0,12,0,0,&quot;seq 10 13&quot;,10,13,outdir/1/10/2/13/stdout,outdir/1/10/2/13/stderr
3,:,1458824738.309,0.026,0,15,0,0,&quot;seq 10 14&quot;,10,14,outdir/1/10/2/14/stdout,outdir/1/10/2/14/stderr</code></pre>
<h2 id="DBURL-as-table">DBURL as table</h2>
<p>The CSV file is an example of a DBURL.</p>
@ -1984,7 +2003,7 @@
149796 149796 1048572
85349 85349 597444</code></pre>
<p>The size of the chunk is not exactly 1 MB because GNU <b>parallel</b> only passes full lines - never half a line, thus the blocksize is only average 1 MB. You can change the block size to 2 MB with <b>--block</b>:</p>
<p>The size of the chunk is not exactly 1 MB because GNU <b>parallel</b> only passes full lines - never half a line, thus the blocksize is only 1 MB on average. You can change the block size to 2 MB with <b>--block</b>:</p>
<pre><code> cat num1000000 | parallel --pipe --block 2M wc</code></pre>
@ -1995,7 +2014,7 @@
299593 299593 2097151
85349 85349 597444</code></pre>
<p>GNU <b>parallel</b> treats each line as a record. If the order of record is unimportant (e.g. you need all lines processed, but you do not care which is processed first), then you can use <b>--round-robin</b>. Without <b>--round-robin</b> GNU <b>parallel</b> will start a command per block; with <b>--round-robin</b> only the requested number of jobs will be started (<b>--jobs</b>). The records will then be distributed between the running jobs:</p>
<p>GNU <b>parallel</b> treats each line as a record. If the order of records is unimportant (e.g. you need all lines processed, but you do not care which is processed first), then you can use <b>--round-robin</b>. Without <b>--round-robin</b> GNU <b>parallel</b> will start a command per block; with <b>--round-robin</b> only the requested number of jobs will be started (<b>--jobs</b>). The records will then be distributed between the running jobs:</p>
<pre><code> cat num1000000 | parallel --pipe -j4 --round-robin wc</code></pre>
@ -2027,7 +2046,7 @@
140000 140000 980000
20000 20000 140001</code></pre>
<p>Notice that the last job could not get the full 140000 lines, but only 20000 lines.</p>
<p>Note how that the last job could not get the full 140000 lines, but only 20000 lines.</p>
<p>If a record is 75 lines <b>-L</b> can be used:</p>
@ -2044,7 +2063,7 @@
85350 85350 597450
25 25 176</code></pre>
<p>Notice GNU <b>parallel</b> still reads a block of around 1 MB; but instead of passing full lines to <b>wc</b> it passes full 75 lines at a time. This of course does not hold for the last job (which in this case got 25 lines).</p>
<p>Note how GNU <b>parallel</b> still reads a block of around 1 MB; but instead of passing full lines to <b>wc</b> it passes full 75 lines at a time. This of course does not hold for the last job (which in this case got 25 lines).</p>
<h2 id="Record-separators">Record separators</h2>
@ -2386,7 +2405,7 @@
<p>GNU <b>parallel</b> can work as a counting semaphore. This is slower and less efficient than its normal mode.</p>
<p>A counting semaphore is like a row of toilets. People needing a toilet can use any toilet, but if there are more people than toilets, they will have to wait for one of the toilets to be available.</p>
<p>A counting semaphore is like a row of toilets. People needing a toilet can use any toilet, but if there are more people than toilets, they will have to wait for one of the toilets to become available.</p>
<p>An alias for <b>parallel --semaphore</b> is <b>sem</b>.</p>
@ -2427,7 +2446,7 @@
<p>The difference between this and just running the command, is that a mutex is set, so if other <b>sem</b>s were running in the background only one would run at a time.</p>
<p>To tell the difference between which semaphore is used, use <b>--semaphorename</b>/<b>--id</b>. Run this in one terminal:</p>
<p>To control which semaphore is used, use <b>--semaphorename</b>/<b>--id</b>. Run this in one terminal:</p>
<pre><code> sem --id my_id -u &#39;echo First started; sleep 10; echo The first finished&#39;</code></pre>
@ -2490,52 +2509,59 @@
<p>Output:</p>
<pre><code> Usage:
parallel [options] [command [arguments]] &lt; list_of_arguments
parallel [options] [command [arguments]] (::: arguments|:::: argfile(s))...
cat ... | parallel --pipe [options] [command [arguments]]
-j n Run n jobs in parallel
-k Keep same order
-X Multiple arguments with context replace
--colsep regexp Split input on regexp for positional replacements
{} {.} {/} {/.} {#} Replacement strings
{3} {3.} {3/} {3/.} Positional replacement strings
-j n Run n jobs in parallel
-k Keep same order
-X Multiple arguments with context replace
--colsep regexp Split input on regexp for positional replacements
{} {.} {/} {/.} {#} {%} {= perl code =} Replacement strings
{3} {3.} {3/} {3/.} {=3 perl code =} Positional replacement strings
With --plus: {} = {+/}/{/} = {.}.{+.} = {+/}/{/.}.{+.} = {..}.{+..} =
{+/}/{/..}.{+..} = {...}.{+...} = {+/}/{/...}.{+...}
-S sshlogin Example: foo@server.example.com
--slf .. Use ~/.parallel/sshloginfile as the list of sshlogins
--trc {}.bar Shorthand for --transfer --return {}.bar --cleanup
--onall Run the given command with argument on all sshlogins
--nonall Run the given command with no arguments on all sshlogins
-S sshlogin Example: foo@server.example.com
--slf .. Use ~/.parallel/sshloginfile as the list of sshlogins
--trc {}.bar Shorthand for --transfer --return {}.bar --cleanup
--onall Run the given command with argument on all sshlogins
--nonall Run the given command with no arguments on all sshlogins
--pipe Split stdin (standard input) to multiple jobs.
--recend str Record end separator for --pipe.
--recstart str Record start separator for --pipe.
--pipe Split stdin (standard input) to multiple jobs.
--recend str Record end separator for --pipe.
--recstart str Record start separator for --pipe.
See &#39;man parallel&#39; for details
When using GNU Parallel for a publication please cite:
Academic tradition requires you to cite works you base your article on.
When using programs that use GNU Parallel to process data for publication
please cite:
O. Tange (2011): GNU Parallel - The Command-Line Power Tool,
;login: The USENIX Magazine, February 2011:42-47.
O. Tange (2011): GNU Parallel - The Command-Line Power Tool,
;login: The USENIX Magazine, February 2011:42-47.
This helps funding further development; AND IT WON&#39;T COST YOU A CENT.
If you pay 10000 EUR you should feel free to use GNU Parallel without citing.</code></pre>
When asking for help, always report the full output of this:
<p>When asking for help, always report the full output of this:</p>
parallel --version</code></pre>
<pre><code> parallel --version</code></pre>
<p>Output:</p>
<pre><code> GNU parallel 20130822
Copyright (C) 2007,2008,2009,2010,2011,2012,2013 Ole Tange and Free Software Foundation, Inc.
<pre><code> GNU parallel 20160323
Copyright (C) 2007,2008,2009,2010,2011,2012,2013,2014,2015,2016
Ole Tange and Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later &lt;http://gnu.org/licenses/gpl.html&gt;
This is free software: you are free to change and redistribute it.
GNU parallel comes with no warranty.
Web site: http://www.gnu.org/software/parallel
When using GNU Parallel for a publication please cite:
O. Tange (2011): GNU Parallel - The Command-Line Power Tool,
;login: The USENIX Magazine, February 2011:42-47.</code></pre>
When using programs that use GNU Parallel to process data for publication
please cite as described in &#39;parallel --bibtex&#39;.</code></pre>
<p>In scripts <b>--minversion</b> can be used to ensure the user has at least this version:</p>
@ -2543,7 +2569,7 @@
<p>Output:</p>
<pre><code> 20130722
<pre><code> 20160322
Your version is at least 20130722.</code></pre>
<p>If using GNU <b>parallel</b> for research the BibTeX citation can be generated using <b>--bibtex</b>:</p>
@ -2552,18 +2578,32 @@
<p>Output:</p>
<pre><code> @article{Tange2011a,
title = {GNU Parallel - The Command-Line Power Tool},
author = {O. Tange},
address = {Frederiksberg, Denmark},
journal = {;login: The USENIX Magazine},
month = {Feb},
number = {1},
volume = {36},
url = {http://www.gnu.org/s/parallel},
year = {2011},
pages = {42-47}
}</code></pre>
<pre><code> Academic tradition requires you to cite works you base your article on.
When using programs that use GNU Parallel to process data for publication
please cite:
@article{Tange2011a,
title = {GNU Parallel - The Command-Line Power Tool},
author = {O. Tange},
address = {Frederiksberg, Denmark},
journal = {;login: The USENIX Magazine},
month = {Feb},
number = {1},
volume = {36},
url = {http://www.gnu.org/s/parallel},
year = {2011},
pages = {42-47},
doi = {10.5281/zenodo.16303}
}
(Feel free to use \nocite{Tange2011a})
This helps funding further development; AND IT WON&#39;T COST YOU A CENT.
If you pay 10000 EUR you should feel free to use GNU Parallel without citing.
If you send a copy of your published article to tange@gnu.org, it will be
mentioned in the release notes of next version of GNU Parallel.
</code></pre>
<p>With <b>--max-line-length-allowed</b> GNU <b>parallel</b> will report the maximal size of the command line:</p>
@ -2606,9 +2646,9 @@
<p>Output:</p>
<pre><code> \nice -n17 /bin/bash -c echo\ A
\nice -n17 /bin/bash -c echo\ B
\nice -n17 /bin/bash -c echo\ C</code></pre>
<pre><code> echo A
echo B
echo C</code></pre>
<h1 id="Spread-the-word">Spread the word</h1>

260
src/parallel_tutorial.pod
View file

@ -1731,8 +1731,29 @@ B<--jobs> sets the number of servers to log in to in parallel.
=head2 Transferring environment variables and functions
Using B<--env> GNU B<parallel> can transfer an environment variable to the
remote system.
B<env_parallel> is a shell function that transfers all aliases,
functions, variables, and arrays. You active it by running:
source `which env_parallel.bash`
Replace B<bash> with the shell you use.
Now you can use B<env_parallel> instead of B<parallel> and still have
your environment:
alias myecho=echo
myvar="Joe's var is"
env_parallel -S $SERVER1 'myecho $myvar' ::: green
Output:
Joe's var is green
The disadvantage is that if your environment is huge B<env_parallel>
will fail.
When B<env_parallel> fails, you can still use B<--env> to tell GNU
B<parallel> to transfer an environment variable to the remote system.
MYVAR='foo bar'
export MYVAR
@ -1784,31 +1805,26 @@ Output:
foo
in my_func2 foo bar
=head2 Showing what is actually run
B<--verbose> will show the command that would be run on the local
machine. When a job is run on a remote machine, this is wrapped with
B<ssh> and possibly transferring files and environment variables, setting
the workdir, and setting B<--nice> value. B<-vv> shows all of this.
machine.
parallel -vv -S $SERVER1 echo ::: bar
When using B<--cat>, B<--pipepart>, or when a job is run on a remote
machine, the command is wrapped with helper scripts. B<-vv> shows all
of this.
parallel -vv --pipepart wc :::: num30000
Output:
ssh lo -- exec perl -e \''@GNU_Parallel=("use","IPC::Open3;","use","MIME::Base64");
eval"@GNU_Parallel";my$eval=decode_base64(join"",@ARGV);eval$eval;'\'
JEVOVnsiUEFSQUxMRUxfUElEIn09IjI3MzQiOyRFTlZ7IlBBUkFMTEVMX1NFUSJ9PSIx
IjskYmFzaGZ1bmMgPSAiIjtAQVJHVj0iZWNobyBiYXIiOyRzaGVsbD0iJEVOVntTSEVM
TH0iOyR0bXBkaXI9Ii90bXAiOyRuaWNlPTA7ZG97JEVOVntQQVJBTExFTF9UTVB9PSR0
bXBkaXIuIi9wYXIiLmpvaW4iIixtYXB7KDAuLjksImEiLi4ieiIsIkEiLi4iWiIpW3Jh
bmQoNjIpXX0oMS4uNSk7fXdoaWxlKC1lJEVOVntQQVJBTExFTF9UTVB9KTskU0lHe0NI
TER9PXN1YnskZG9uZT0xO307JHBpZD1mb3JrO3VubGVzcygkcGlkKXtzZXRwZ3JwO2V2
YWx7c2V0cHJpb3JpdHkoMCwwLCRuaWNlKX07ZXhlYyRzaGVsbCwiLWMiLCgkYmFzaGZ1
bmMuIkBBUkdWIik7ZGllImV4ZWM6JCFcbiI7fWRveyRzPSRzPDE/MC4wMDErJHMqMS4w
MzokcztzZWxlY3QodW5kZWYsdW5kZWYsdW5kZWYsJHMpO311bnRpbCgkZG9uZXx8Z2V0
cHBpZD09MSk7a2lsbChTSUdIVVAsLSR7cGlkfSl1bmxlc3MkZG9uZTt3YWl0O2V4aXQo
JD8mMTI3PzEyOCsoJD8mMTI3KToxKyQ/Pj44KQ==;
bar
<num30000 perl -e 'while(@ARGV) { sysseek(STDIN,shift,0) || die;
$left = shift; while($read = sysread(STDIN,$buf, ($left > 131072
? 131072 : $left))){ $left -= $read; syswrite(STDOUT,$buf); } }'
0 0 0 168894 | (wc)
30000 30000 168894
When the command gets more complex, the output is so hard to read,
that it is only useful for debugging:
@ -1821,44 +1837,41 @@ that it is only useful for debugging:
Output will be similar to:
( ssh lo -- mkdir -p ./.parallel/tmp/hk-3492-1;rsync --protocol 30
-rlDzR -essh ./abc-file lo:./.parallel/tmp/hk-3492-1 );ssh lo --
exec perl -e \''@GNU_Parallel=("use","IPC::Open3;","use","MIME::Base64");
( ssh server -- mkdir -p ./.parallel/tmp/aspire-1928520-1;rsync
--protocol 30 -rlDzR -essh ./abc-file
server:./.parallel/tmp/aspire-1928520-1 );ssh server -- exec perl -e
\''@GNU_Parallel=("use","IPC::Open3;","use","MIME::Base64");
eval"@GNU_Parallel";my$eval=decode_base64(join"",@ARGV);eval$eval;'\'
c3lzdGVtKCJta2RpciIsIi1wIiwiLS0iLCIucGFyYWxsZWwvdG1wL2hrLTM0OTItMSIp
OyBjaGRpciAiLnBhcmFsbGVsL3RtcC9oay0zNDkyLTEiIHx8cHJpbnQoU1RERVJSICJw
YXJhbGxlbDogQ2Fubm90IGNoZGlyIHRvIC5wYXJhbGxlbC90bXAvaGstMzQ5Mi0xXG4i
KSAmJiBleGl0IDI1NTskRU5WeyJHUEdfQUdFTlRfSU5GTyJ9PSIvdG1wL2dwZy10WjVI
U0QvUy5ncGctYWdlbnQ6MjM5NzoxIjskRU5WeyJQQVJBTExFTF9TRVEifT0iMSI7JEVO
VnsiU1FMSVRFVEJMIn09InNxbGl0ZTM6Ly8vJTJGdG1wJTJGcGFyYWxsZWwuZGIyL3Bh
cnNxbDIiOyRFTlZ7IlBBUkFMTEVMX1BJRCJ9PSIzNDkyIjskRU5WeyJTUUxJVEUifT0i
c3FsaXRlMzovLy8lMkZ0bXAlMkZwYXJhbGxlbC5kYjIiOyRFTlZ7IlBBUkFMTEVMX1BJ
RCJ9PSIzNDkyIjskRU5WeyJQQVJBTExFTF9TRVEifT0iMSI7QGJhc2hfZnVuY3Rpb25z
PXF3KG15X2Z1bmMzKTsgaWYoJEVOVnsiU0hFTEwifT1+L2NzaC8pIHsgcHJpbnQgU1RE
RVJSICJDU0gvVENTSCBETyBOT1QgU1VQUE9SVCBuZXdsaW5lcyBJTiBWQVJJQUJMRVMv
RlVOQ1RJT05TLiBVbnNldCBAYmFzaF9mdW5jdGlvbnNcbiI7IGV4ZWMgImZhbHNlIjsg
fSAKJGJhc2hmdW5jID0gIm15X2Z1bmMzKCkgeyAgZWNobyBpbiBteV9mdW5jIFwkMSA+
IFwkMS5vdXQKfTtleHBvcnQgLWYgbXlfZnVuYzMgPi9kZXYvbnVsbDsiO0BBUkdWPSJt
eV9mdW5jMyBhYmMtZmlsZSI7JHNoZWxsPSIkRU5We1NIRUxM
fSI7JHRtcGRpcj0iL3RtcCI7JG5pY2U9MTc7ZG97JEVOVntQQVJBTExFTF9UTVB9PSR0
bXBkaXIuIi9wYXIiLmpvaW4iIixtYXB7KDAuLjksImEiLi4ieiIsIkEiLi4iWiIpW3Jh
bmQoNjIpXX0oMS4uNSk7fXdoaWxlKC1lJEVOVntQQVJBTExFTF9UTVB9KTskU0lHe0NI
TER9PXN1YnskZG9uZT0xO307JHBpZD1mb3JrO3VubGVzcygkcGlkKXtzZXRwZ3JwO2V2
YWx7c2V0cHJpb3JpdHkoMCwwLCRuaWNlKX07ZXhlYyRzaGVsbCwiLWMiLCgkYmFzaGZ1
bmMuIkBBUkdWIik7ZGllImV4ZWM6JCFcbiI7fWRveyRzPSRzPDE/MC4wMDErJHMqMS4w
MzokcztzZWxlY3QodW5kZWYsdW5kZWYsdW5kZWYsJHMpO311bnRpbCgkZG9uZXx8Z2V0
cHBpZD09MSk7a2lsbChTSUdIVVAsLSR7cGlkfSl1bmxlc3MkZG9uZTt3YWl0O2V4aXQo
JD8mMTI3PzEyOCsoJD8mMTI3KToxKyQ/Pj44KQ==;_EXIT_status=$?;
mkdir -p ./.; rsync --protocol 30 --rsync-path=cd\
./.parallel/tmp/hk-3492-1/./.\;\ rsync -rlDzR -essh
lo:./abc-file.out ./.;ssh lo -- \(rm\ -f\
./.parallel/tmp/hk-3492-1/abc-file\;\ sh\ -c\ \'rmdir\
./.parallel/tmp/hk-3492-1/\ ./.parallel/tmp/\ ./.parallel/\
2\>/dev/null\'\;rm\ -rf\ ./.parallel/tmp/hk-3492-1\;\);ssh lo --
\(rm\ -f\ ./.parallel/tmp/hk-3492-1/abc-file.out\;\ sh\ -c\ \'rmdir\
./.parallel/tmp/hk-3492-1/\ ./.parallel/tmp/\ ./.parallel/\
2\>/dev/null\'\;rm\ -rf\ ./.parallel/tmp/hk-3492-1\;\);ssh lo -- rm
-rf .parallel/tmp/hk-3492-1; exit $_EXIT_status;
c3lzdGVtKCJta2RpciIsIi1wIiwiLS0iLCIucGFyYWxsZWwvdG1wL2FzcGlyZS0xOTI4N
TsgY2hkaXIgIi5wYXJhbGxlbC90bXAvYXNwaXJlLTE5Mjg1MjAtMSIgfHxwcmludChTVE
BhcmFsbGVsOiBDYW5ub3QgY2hkaXIgdG8gLnBhcmFsbGVsL3RtcC9hc3BpcmUtMTkyODU
iKSAmJiBleGl0IDI1NTskRU5WeyJPTERQV0QifT0iL2hvbWUvdGFuZ2UvcHJpdmF0L3Bh
IjskRU5WeyJQQVJBTExFTF9QSUQifT0iMTkyODUyMCI7JEVOVnsiUEFSQUxMRUxfU0VRI
0BiYXNoX2Z1bmN0aW9ucz1xdyhteV9mdW5jMyk7IGlmKCRFTlZ7IlNIRUxMIn09fi9jc2
ByaW50IFNUREVSUiAiQ1NIL1RDU0ggRE8gTk9UIFNVUFBPUlQgbmV3bGluZXMgSU4gVkF
TL0ZVTkNUSU9OUy4gVW5zZXQgQGJhc2hfZnVuY3Rpb25zXG4iOyBleGVjICJmYWxzZSI7
YXNoZnVuYyA9ICJteV9mdW5jMygpIHsgIGVjaG8gaW4gbXlfZnVuYyBcJDEgPiBcJDEub
Xhwb3J0IC1mIG15X2Z1bmMzID4vZGV2L251bGw7IjtAQVJHVj0ibXlfZnVuYzMgYWJjLW
RzaGVsbD0iJEVOVntTSEVMTH0iOyR0bXBkaXI9Ii90bXAiOyRuaWNlPTE3O2RveyRFTlZ
MRUxfVE1QfT0kdG1wZGlyLiIvcGFyIi5qb2luIiIsbWFweygwLi45LCJhIi4uInoiLCJB
KVtyYW5kKDYyKV19KDEuLjUpO313aGlsZSgtZSRFTlZ7UEFSQUxMRUxfVE1QfSk7JFNJ
fT1zdWJ7JGRvbmU9MTt9OyRwaWQ9Zm9yazt1bmxlc3MoJHBpZCl7c2V0cGdycDtldmFse
W9yaXR5KDAsMCwkbmljZSl9O2V4ZWMkc2hlbGwsIi1jIiwoJGJhc2hmdW5jLiJAQVJHVi
JleGVjOiQhXG4iO31kb3skcz0kczwxPzAuMDAxKyRzKjEuMDM6JHM7c2VsZWN0KHVuZGV
mLHVuZGVmLCRzKTt9dW50aWwoJGRvbmV8fGdldHBwaWQ9PTEpO2tpbGwoU0lHSFVQLC0k
dW5sZXNzJGRvbmU7d2FpdDtleGl0KCQ/JjEyNz8xMjgrKCQ/JjEyNyk6MSskPz4+OCk=;
_EXIT_status=$?; mkdir -p ./.; rsync --protocol 30 --rsync-path=cd\
./.parallel/tmp/aspire-1928520-1/./.\;\ rsync -rlDzR -essh
server:./abc-file.out ./.;ssh server -- \(rm\ -f\
./.parallel/tmp/aspire-1928520-1/abc-file\;\ sh\ -c\ \'rmdir\
./.parallel/tmp/aspire-1928520-1/\ ./.parallel/tmp/\ ./.parallel/\
2\>/dev/null\'\;rm\ -rf\ ./.parallel/tmp/aspire-1928520-1\;\);ssh
server -- \(rm\ -f\ ./.parallel/tmp/aspire-1928520-1/abc-file.out\;\
sh\ -c\ \'rmdir\ ./.parallel/tmp/aspire-1928520-1/\ ./.parallel/tmp/\
./.parallel/\ 2\>/dev/null\'\;rm\ -rf\
./.parallel/tmp/aspire-1928520-1\;\);ssh server -- rm -rf
.parallel/tmp/aspire-1928520-1; exit $_EXIT_status;
=head1 Saving to an SQL base (advanced)
@ -1897,6 +1910,19 @@ Output will be similar to:
A proper CSV reader (like LibreOffice or R's read.csv) will read this
format correctly - even with fields containing newlines as above.
If the output is big you may want to put it into files using B<--results>:
parallel --results outdir --sqlandworker csv:////%2Ftmp%2Flog2.csv seq ::: 10 ::: 12 13 14
cat /tmp/log2.csv
Output will be similar to:
Seq,Host,Starttime,JobRuntime,Send,Receive,Exitval,_Signal,Command,V1,V2,Stdout,Stderr
1,:,1458824738.287,0.029,0,9,0,0,"seq 10 12",10,12,outdir/1/10/2/12/stdout,outdir/1/10/2/12/stderr
2,:,1458824738.298,0.025,0,12,0,0,"seq 10 13",10,13,outdir/1/10/2/13/stdout,outdir/1/10/2/13/stderr
3,:,1458824738.309,0.026,0,15,0,0,"seq 10 14",10,14,outdir/1/10/2/14/stdout,outdir/1/10/2/14/stderr
=head2 DBURL as table
The CSV file is an example of a DBURL.
@ -1996,7 +2022,7 @@ Output (the order may be different):
The size of the chunk is not exactly 1 MB because GNU B<parallel> only
passes full lines - never half a line, thus the blocksize is only
average 1 MB. You can change the block size to 2 MB with B<--block>:
1 MB on average. You can change the block size to 2 MB with B<--block>:
cat num1000000 | parallel --pipe --block 2M wc
@ -2007,8 +2033,8 @@ Output (the order may be different):
299593 299593 2097151
85349 85349 597444
GNU B<parallel> treats each line as a record. If the order of record is
unimportant (e.g. you need all lines processed, but you do not care
GNU B<parallel> treats each line as a record. If the order of records
is unimportant (e.g. you need all lines processed, but you do not care
which is processed first), then you can use B<--round-robin>. Without
B<--round-robin> GNU B<parallel> will start a command per block; with
B<--round-robin> only the requested number of jobs will be started
@ -2047,8 +2073,8 @@ Output (the order may be different):
140000 140000 980000
20000 20000 140001
Notice that the last job could not get the full 140000 lines, but only
20000 lines.
Note how that the last job could not get the full 140000 lines, but
only 20000 lines.
If a record is 75 lines B<-L> can be used:
@ -2065,10 +2091,10 @@ Output (the order may be different):
85350 85350 597450
25 25 176
Notice GNU B<parallel> still reads a block of around 1 MB; but instead of
passing full lines to B<wc> it passes full 75 lines at a time. This
of course does not hold for the last job (which in this case got 25
lines).
Note how GNU B<parallel> still reads a block of around 1 MB; but
instead of passing full lines to B<wc> it passes full 75 lines at a
time. This of course does not hold for the last job (which in this
case got 25 lines).
=head2 Record separators
@ -2405,7 +2431,7 @@ efficient than its normal mode.
A counting semaphore is like a row of toilets. People needing a toilet
can use any toilet, but if there are more people than toilets, they
will have to wait for one of the toilets to be available.
will have to wait for one of the toilets to become available.
An alias for B<parallel --semaphore> is B<sem>.
@ -2455,7 +2481,7 @@ The difference between this and just running the command, is that a
mutex is set, so if other B<sem>s were running in the background only one
would run at a time.
To tell the difference between which semaphore is used, use
To control which semaphore is used, use
B<--semaphorename>/B<--id>. Run this in one terminal:
sem --id my_id -u 'echo First started; sleep 10; echo The first finished'
@ -2525,52 +2551,59 @@ B<--help> will print a summary of the most important options:
Output:
Usage:
parallel [options] [command [arguments]] < list_of_arguments
parallel [options] [command [arguments]] (::: arguments|:::: argfile(s))...
cat ... | parallel --pipe [options] [command [arguments]]
-j n Run n jobs in parallel
-k Keep same order
-X Multiple arguments with context replace
--colsep regexp Split input on regexp for positional replacements
{} {.} {/} {/.} {#} Replacement strings
{3} {3.} {3/} {3/.} Positional replacement strings
-j n Run n jobs in parallel
-k Keep same order
-X Multiple arguments with context replace
--colsep regexp Split input on regexp for positional replacements
{} {.} {/} {/.} {#} {%} {= perl code =} Replacement strings
{3} {3.} {3/} {3/.} {=3 perl code =} Positional replacement strings
With --plus: {} = {+/}/{/} = {.}.{+.} = {+/}/{/.}.{+.} = {..}.{+..} =
{+/}/{/..}.{+..} = {...}.{+...} = {+/}/{/...}.{+...}
-S sshlogin Example: foo@server.example.com
--slf .. Use ~/.parallel/sshloginfile as the list of sshlogins
--trc {}.bar Shorthand for --transfer --return {}.bar --cleanup
--onall Run the given command with argument on all sshlogins
--nonall Run the given command with no arguments on all sshlogins
-S sshlogin Example: foo@server.example.com
--slf .. Use ~/.parallel/sshloginfile as the list of sshlogins
--trc {}.bar Shorthand for --transfer --return {}.bar --cleanup
--onall Run the given command with argument on all sshlogins
--nonall Run the given command with no arguments on all sshlogins
--pipe Split stdin (standard input) to multiple jobs.
--recend str Record end separator for --pipe.
--recstart str Record start separator for --pipe.
--pipe Split stdin (standard input) to multiple jobs.
--recend str Record end separator for --pipe.
--recstart str Record start separator for --pipe.
See 'man parallel' for details
When using GNU Parallel for a publication please cite:
Academic tradition requires you to cite works you base your article on.
When using programs that use GNU Parallel to process data for publication
please cite:
O. Tange (2011): GNU Parallel - The Command-Line Power Tool,
;login: The USENIX Magazine, February 2011:42-47.
O. Tange (2011): GNU Parallel - The Command-Line Power Tool,
;login: The USENIX Magazine, February 2011:42-47.
This helps funding further development; AND IT WON'T COST YOU A CENT.
If you pay 10000 EUR you should feel free to use GNU Parallel without citing.
When asking for help, always report the full output of this:
When asking for help, always report the full output of this:
parallel --version
Output:
GNU parallel 20130822
Copyright (C) 2007,2008,2009,2010,2011,2012,2013 Ole Tange and Free Software Foundation, Inc.
GNU parallel 20160323
Copyright (C) 2007,2008,2009,2010,2011,2012,2013,2014,2015,2016
Ole Tange and Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
GNU parallel comes with no warranty.
Web site: http://www.gnu.org/software/parallel
When using GNU Parallel for a publication please cite:
O. Tange (2011): GNU Parallel - The Command-Line Power Tool,
;login: The USENIX Magazine, February 2011:42-47.
When using programs that use GNU Parallel to process data for publication
please cite as described in 'parallel --bibtex'.
In scripts B<--minversion> can be used to ensure the user has at least
this version:
@ -2579,7 +2612,7 @@ this version:
Output:
20130722
20160322
Your version is at least 20130722.
If using GNU B<parallel> for research the BibTeX citation can be
@ -2589,19 +2622,32 @@ generated using B<--bibtex>:
Output:
Academic tradition requires you to cite works you base your article on.
When using programs that use GNU Parallel to process data for publication
please cite:
@article{Tange2011a,
title = {GNU Parallel - The Command-Line Power Tool},
author = {O. Tange},
address = {Frederiksberg, Denmark},
journal = {;login: The USENIX Magazine},
month = {Feb},
number = {1},
volume = {36},
url = {http://www.gnu.org/s/parallel},
year = {2011},
pages = {42-47}
title = {GNU Parallel - The Command-Line Power Tool},
author = {O. Tange},
address = {Frederiksberg, Denmark},
journal = {;login: The USENIX Magazine},
month = {Feb},
number = {1},
volume = {36},
url = {http://www.gnu.org/s/parallel},
year = {2011},
pages = {42-47},
doi = {10.5281/zenodo.16303}
}
(Feel free to use \nocite{Tange2011a})
This helps funding further development; AND IT WON'T COST YOU A CENT.
If you pay 10000 EUR you should feel free to use GNU Parallel without citing.
If you send a copy of your published article to tange@gnu.org, it will be
mentioned in the release notes of next version of GNU Parallel.
With B<--max-line-length-allowed> GNU B<parallel> will report the maximal
size of the command line:
@ -2648,9 +2694,9 @@ Profiles can be combined:
Output:
\nice -n17 /bin/bash -c echo\ A
\nice -n17 /bin/bash -c echo\ B
\nice -n17 /bin/bash -c echo\ C
echo A
echo B
echo C
=head1 Spread the word

5
testsuite/tests-to-run/parallel-local-0.3s.sh
View file

@ -271,6 +271,11 @@ echo 'bug #47086: [PATCH] Initialize total_completed from joblog'
echo '**'
echo 'bug #47290: xargs: Warning: a NUL character occurred in the input'
perl -e 'print "foo\0not printed"' | parallel echo
echo '**'
EOF
echo '### 1 .par file from --files expected'

42
testsuite/tests-to-run/parallel-local-ssh4.sh
View file

@ -128,16 +128,7 @@ func_echo() {
echo ${assocarr[a]}
echo Funky-"$funky"-funky
}
env_parallel() {
export PARALLEL_ENV="$(echo "shopt -s expand_aliases 2>/dev/null"; alias;typeset -p |
grep -vFf <(readonly) |
grep -v 'declare .. (GROUPS|FUNCNAME|DIRSTACK|_|PIPESTATUS|USERNAME|BASH_[A-Z_]+) ';
typeset -f)";
`which parallel` "$@";
unset PARALLEL_ENV;
}
. `which env_parallel.bash`
env_parallel alias_echo ::: alias_works
env_parallel func_echo ::: function_works
env_parallel -S lo alias_echo ::: alias_works_over_ssh
@ -165,16 +156,8 @@ func_echo() {
echo Funky-"$funky"-funky
}
env_parallel() {
export PARALLEL_ENV="$(alias | perl -pe 's/^/alias /';typeset -p |
grep -aFvf <(typeset -pr)|egrep -iav 'ZSH_EVAL_CONTEXT|LINENO=| _=|aliases|^typeset [a-z_]+$'|
egrep -av '^(typeset IFS=|..$)|cyan';
typeset -f)";
parallel "$@";
unset PARALLEL_ENV;
}
# alias does not work: http://unix.stackexchange.com/questions/223534/defining-an-alias-and-immediately-use-it
# alias does not work:
# http://unix.stackexchange.com/questions/223534/defining-an-alias-and-immediately-use-it
env_parallel alias_echo ::: alias_does_not_work
env_parallel func_echo ::: function_works
env_parallel -S zsh@lo alias_echo ::: alias_does_not_work_over_ssh
@ -202,11 +185,6 @@ func_echo() {
echo Funky-"$funky"-funky
}
env_parallel() {
export PARALLEL_ENV="$(alias | perl -pe 's/^/alias /';typeset -p|egrep -v 'typeset( -i)? -r|PIPESTATUS';typeset -f)";
`which parallel` "$@";
unset PARALLEL_ENV;
}
env_parallel alias_echo ::: alias_works
env_parallel func_echo ::: function_works
env_parallel -S ksh@lo alias_echo ::: alias_works_over_ssh
@ -218,9 +196,6 @@ EOS
echo
echo "### Fish environment"
stdout ssh -q fish@lo <<'EOS' | egrep -v 'Welcome to |packages can be updated|security updates'
# All variables cannot reliably be exported
# perl -e 'print map { "setenv///$_///$ENV{$_}\n"} grep !/^(PWD|SHLVL|PATH)$/, keys %ENV'| sh -c 'parallel --shellquote' | perl -pe 's:///: :g' |fish
set myvar "myvar works"
setenv myenvvar "myenvvar works"
@ -238,8 +213,7 @@ function func_echo
echo $argv;
echo "$myvar"
echo "$myenvvar"
# Arrays are not exported
# echo $myarray[2]
echo $myarray[2]
# Assoc arrays do not exist in fish
# echo ${assocarr[a]}
echo
@ -252,12 +226,6 @@ function func_echo
echo
end
function env_parallel
setenv PARALLEL_ENV (begin; functions -n | perl -pe 's/,/\n/g' | while read d; functions $d; end; perl -e 'print map { "$_///$ENV{$_}\n"} grep !/^(PWD|SHLVL|PATH)$/, keys %ENV'| sh -c 'parallel --shellquote' | perl -pe 's:^([^/]+)///:setenv $1 :'; end |perl -pe 's/\001/\\cb/g;s/\n/\001/')
parallel $argv;
set -e PARALLEL_ENV
end
env_parallel alias_echo ::: alias_works
env_parallel func_echo ::: function_works
env_parallel -S fish@lo alias_echo ::: alias_works_over_ssh
@ -322,7 +290,7 @@ alias alias_echo_var 'echo $argv; echo $myvar; echo ${myarray[2]}; echo Funky-"$
# s/^/\001alias /;
# Quoted: s/\^/\\001alias\ /\;
alias env_parallel 'setenv PARALLEL_ENV "`alias | perl -pe s/\\047/\\047\\042\\047\\042\\047/g\;s/\^\(\\S+\)\(\\s+\)\\\(\(.\*\)\\\)/\\1\\2\\3/\;s/\^\(\\S+\)\(\\s+\)\(.\*\)/\\1\\2\\047\\3\\047/\;s/\^/\\001alias\ /\;s/\\\!/\\\\\\\!/g;`";parallel \!*; setenv PARALLEL_ENV'
#!# alias env_parallel 'setenv PARALLEL_ENV "`alias | perl -pe s/\\047/\\047\\042\\047\\042\\047/g\;s/\^\(\\S+\)\(\\s+\)\\\(\(.\*\)\\\)/\\1\\2\\3/\;s/\^\(\\S+\)\(\\s+\)\(.\*\)/\\1\\2\\047\\3\\047/\;s/\^/\\001alias\ /\;s/\\\!/\\\\\\\!/g;`";parallel \!*; setenv PARALLEL_ENV'
## set tmpfile=`tempfile`

2
testsuite/wanted-results/parallel-install
View file

@ -175,7 +175,7 @@ pod2pdf --output-file ./parallel_design.pdf ./parallel_design.pod --title "GNU P
Warning: pod2pdf not found. Using old parallel_design.pdf
make[0]: Entering directory `/tmp/parallel-00000000/src'
/bin/mkdir -p '/usr/local/bin'
/usr/bin/install -c parallel sql niceload env_parallel env_parallel.bash env_parallel.zsh env_parallel.fish env_parallel.ksh '/usr/local/bin'
/usr/bin/install -c parallel sql niceload env_parallel env_parallel.bash env_parallel.zsh env_parallel.fish env_parallel.ksh env_parallel.pdksh env_parallel.csh '/usr/local/bin'
make install-exec-hook
make[0]: Entering directory `/tmp/parallel-00000000/src'
rm /usr/local/bin/sem || true

9
testsuite/wanted-results/parallel-local-0.3s
View file

@ -588,5 +588,14 @@ parallel: This job failed:
echo 3;exit 1
echo '**'
**
echo 'bug #47290: xargs: Warning: a NUL character occurred in the input'
bug #47290: xargs: Warning: a NUL character occurred in the input
perl -e 'print "foo\0not printed"' | parallel echo
foo
parallel: Warning: a NUL character occurred in the input.
parallel: Warning: It cannot be passed through in the argument list.
parallel: Warning: Did you mean to use the --null option?
echo '**'
**
### 1 .par file from --files expected
1

22
testsuite/wanted-results/parallel-local-ssh4
View file

@ -143,7 +143,7 @@ Funky-
* Documentation: http://www.linuxmint.com
zsh:129: command not found: alias_echo
zsh:130: command not found: alias_echo
function_works
myvar works
array_val2
@ -151,7 +151,7 @@ assoc_val_a
Funky-

 !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~€亗儎厗噲墛媽崕彁憭摂晼棙櫄洔潪煚、￥ウЖ┆<D096><E29486><EFBFBD>辈炒刀犯购患骄坷谅媚牌侨墒颂臀闲岩釉罩棕仝圮蒉哙徕沅彐玷殛腱眍镳耱篝貊鼬<E8B28A><E9BCAC><EFBFBD><EFBFBD>-funky
zsh:129: command not found: alias_echo
zsh:130: command not found: alias_echo
function_works_over_ssh
myvar works
array_val2
@ -191,26 +191,36 @@ Funky-
* Documentation: http://www.linuxmint.com
env_parallel: Warning: ASCII value 1 in variables is not supported
env_parallel: Warning: ASCII value 1 in variables is not supported
3 arg alias_works
env_parallel: Warning: ASCII value 1 in variables is not supported
env_parallel: Warning: ASCII value 1 in variables is not supported
function_works
myvar works
myenvvar works
3
Funky--funky
Funky-   !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~€亗儎厗噲墛媽崕彁憭摂晼棙櫄洔潪煚、￥ウЖ┆<D096><E29486><EFBFBD>辈炒刀犯购患骄坷谅媚牌侨墒颂臀闲岩釉罩棕仝圮蒉哙徕沅彐玷殛腱眍镳耱篝貊鼬<E8B28A><E9BCAC><EFBFBD><EFBFBD>-funky
Funky-   !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~€亗儎厗噲墛媽崕彁憭摂晼棙櫄洔潪煚、￥ウЖ┆<D096><E29486><EFBFBD>辈炒刀犯购患骄坷谅媚牌侨墒颂臀闲岩釉罩棕仝圮蒉哙徕沅彐玷殛腱眍镳耱篝貊鼬<E8B28A><E9BCAC><EFBFBD><EFBFBD>-funky
env_parallel: Warning: ASCII value 1 in variables is not supported
env_parallel: Warning: ASCII value 1 in variables is not supported
3 arg alias_works_over_ssh
env_parallel: Warning: ASCII value 1 in variables is not supported
env_parallel: Warning: ASCII value 1 in variables is not supported
function_works_over_ssh
myvar works
myenvvar works
3
Funky--funky
Funky-   !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~€亗儎厗噲墛媽崕彁憭摂晼棙櫄洔潪煚、￥ウЖ┆<D096><E29486><EFBFBD>辈炒刀犯购患骄坷谅媚牌侨墒颂臀闲岩釉罩棕仝圮蒉哙徕沅彐玷殛腱眍镳耱篝貊鼬<E8B28A><E9BCAC><EFBFBD><EFBFBD>-funky
Funky-   !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~€亗儎厗噲墛媽崕彁憭摂晼棙櫄洔潪煚、￥ウЖ┆<D096><E29486><EFBFBD>辈炒刀犯购患骄坷谅媚牌侨墒颂臀闲岩釉罩棕仝圮蒉哙徕沅彐玷殛腱眍镳耱篝貊鼬<E8B28A><E9BCAC><EFBFBD><EFBFBD>-funky

40
testsuite/wanted-results/parallel-tutorial
View file

@ -704,6 +704,10 @@ bar
parallel --nonall --tag -S $SERVER1,$SERVER2 echo foo bar
csh@lo foo bar
parallel@lo foo bar
source `which env_parallel.bash`
alias myecho=echo
myvar="Joe's var is"
env_parallel -S $SERVER1 'myecho $myvar' ::: green
MYVAR='foo bar'
export MYVAR
parallel --env MYVAR -S $SERVER1 echo '$MYVAR' ::: baz
@ -815,9 +819,9 @@ _
parallel --env _ -S $SERVER1 'echo $VAR; my_func2' ::: bar
/bin/bash: my_func2: command not found
parallel -vv -S $SERVER1 echo ::: bar
ssh -l parallel lo -- exec perl -e \''@GNU_Parallel=("use","IPC::Open3;","use","MIME::Base64");eval"@GNU_Parallel";my$eval=decode_base64(join"",@ARGV);eval$eval;'\' BASE64;
bar
parallel -vv --pipepart wc :::: num30000
<num30000 perl -e 'while(@ARGV) { sysseek(STDIN,shift,0) || die; $left = shift; while($read = sysread(STDIN,$buf, ($left > 131072 ? 131072 : $left))){ $left -= $read; syswrite(STDOUT,$buf); } }' 0 0 0 168894 | (wc)
30000 30000 168894
my_func3() {
echo in my_func $1 > $1.out
}
@ -854,6 +858,24 @@ Seq,Host,Starttime,JobRuntime,Send,Receive,Exitval,_Signal,Command,V1,V2,Stdout,
13
14
",
parallel --results outdir --sqlandworker csv:////%2Ftmp%2Flog2.csv seq ::: 10 ::: 12 13 14
cat /tmp/log2.csv
10
11
12
10
11
12
13
10
11
12
13
14
Seq,Host,Starttime,JobRuntime,Send,Receive,Exitval,_Signal,Command,V1,V2,Stdout,Stderr
1,:,000000000.000,0.000,0,9,0,0,"seq 10 12",10,12,outdir/1/10/2/12/stdout,outdir/1/10/2/12/stderr
2,:,000000000.000,0.000,0,12,0,0,"seq 10 13",10,13,outdir/1/10/2/13/stdout,outdir/1/10/2/13/stderr
3,:,000000000.000,0.000,0,15,0,0,"seq 10 14",10,14,outdir/1/10/2/14/stdout,outdir/1/10/2/14/stderr
vendor://[[user][:password]@][host][:port]/[database[/table]
/bin/bash: vendor://[[user][:password]@][host][:port]/[database[/table]: No such file or directory
sqlite3:///%2Ftmp%2Fmydatabase/mytable
@ -1205,6 +1227,18 @@ please cite:
This helps funding further development; AND IT WON'T COST YOU A CENT.
If you pay 10000 EUR you should feel free to use GNU Parallel without citing.
parallel --version
GNU parallel VERSION
Copyright (C) 2007,2008,2009,2010,2011,2012,2013,2014,2015,2016
Ole Tange and Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
GNU parallel comes with no warranty.
Web site: http://www.gnu.org/software/parallel
When using programs that use GNU Parallel to process data for publication
please cite as described in 'parallel --bibtex'.
parallel --minversion VERSION && echo Your version is at least VERSION.
VERSION
Your version is at least VERSION.