mirror of
https://git.savannah.gnu.org/git/parallel.git
synced 2024-11-21 21:47:54 +00:00
Added env_parallel.csh env_parallel.pdksh.
env_parallel.fish now supports arrays. Documentation updates.
This commit is contained in:
Ole Tange
parent
1cad1cf939
commit
e663fe2636
2
README
2
README
|
|
@ -75,7 +75,7 @@ system is old or Microsoft Windows):
|
|||
|
||||
After this you should be able to do:
|
||||
|
||||
parallel -j0 ping -nc 3 ::: foss.org.my gnu.org freenetproject.org
|
||||
parallel -j0 ping -nc 3 ::: qubes-os.org gnu.org freenetproject.org
|
||||
|
||||
This will send 3 ping packets to 3 different hosts in parallel and print
|
||||
the output when they complete.
|
||||
|
|
|
|||
|
|
@ -248,6 +248,12 @@ for Big Data Applications https://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumb
|
|||
|
||||
* <<Citation needed: Introspecting for RSA Key Material to Assist Intrusion Detection http://ieeexplore.ieee.org/xpl/login.jsp?tp=&arnumber=7331177&url=http%3A%2F%2Fieeexplore.ieee.org%2Fxpls%2Fabs_all.jsp%3Farnumber%3D7331177>>
|
||||
|
||||
* GNU Parallel was cited in: Tools and techniques for computational reproducibility http://biorxiv.org/content/biorxiv/early/2016/03/17/022707.full.pdf
|
||||
|
||||
* qbatch uses GNU Parallel: https://pypi.python.org/pypi/qbatch/1.0rc2
|
||||
|
||||
* FaceCrop uses GNU Parallel: https://github.com/EderSantana/FaceCrop
|
||||
|
||||
* Bug fixes and man page updates.
|
||||
|
||||
GNU Parallel - For people who live life in the parallel lane.
|
||||
|
|
|
|||
|
|
@ -1,15 +1,20 @@
|
|||
bin_SCRIPTS = parallel sql niceload \
|
||||
env_parallel env_parallel.bash env_parallel.zsh env_parallel.fish env_parallel.ksh
|
||||
env_parallel env_parallel.bash env_parallel.zsh env_parallel.fish \
|
||||
env_parallel.ksh env_parallel.pdksh env_parallel.csh
|
||||
|
||||
install-exec-hook:
|
||||
rm $(DESTDIR)$(bindir)/sem || true
|
||||
$(LN_S) parallel $(DESTDIR)$(bindir)/sem
|
||||
|
||||
if DOCUMENTATION
|
||||
man_MANS = parallel.1 env_parallel.1 sem.1 sql.1 niceload.1 parallel_tutorial.7 parallel_design.7
|
||||
doc_DATA = parallel.html env_parallel.html sem.html sql.html niceload.html parallel_tutorial.html parallel_design.html \
|
||||
parallel.texi env_parallel.texi sem.texi sql.texi niceload.texi parallel_tutorial.texi parallel_design.texi \
|
||||
parallel.pdf env_parallel.pdf sem.pdf sql.pdf niceload.pdf parallel_tutorial.pdf parallel_design.pdf
|
||||
man_MANS = parallel.1 env_parallel.1 sem.1 sql.1 niceload.1 \
|
||||
parallel_tutorial.7 parallel_design.7
|
||||
doc_DATA = parallel.html env_parallel.html sem.html sql.html niceload.html \
|
||||
parallel_tutorial.html parallel_design.html \
|
||||
parallel.texi env_parallel.texi sem.texi sql.texi niceload.texi \
|
||||
parallel_tutorial.texi parallel_design.texi \
|
||||
parallel.pdf env_parallel.pdf sem.pdf sql.pdf niceload.pdf \
|
||||
parallel_tutorial.pdf parallel_design.pdf
|
||||
endif
|
||||
|
||||
# Build documentation file if the tool to build exists.
|
||||
|
|
@ -174,5 +179,7 @@ DISTCLEANFILES = parallel.1 env_parallel.1 sem.1 sql.1 niceload.1 \
|
|||
|
||||
EXTRA_DIST = parallel sem sql niceload env_parallel \
|
||||
env_parallel.bash env_parallel.zsh env_parallel.fish env_parallel.ksh \
|
||||
sem.pod parallel.pod env_parallel.pod niceload.pod parallel_tutorial.pod parallel_design.pod \
|
||||
env_parallel.pdksh env_parallel.csh \
|
||||
sem.pod parallel.pod env_parallel.pod niceload.pod parallel_tutorial.pod \
|
||||
parallel_design.pod \
|
||||
$(DISTCLEANFILES)
|
||||
|
|
|
|||
|
|
@ -218,12 +218,18 @@ top_build_prefix = @top_build_prefix@
|
|||
top_builddir = @top_builddir@
|
||||
top_srcdir = @top_srcdir@
|
||||
bin_SCRIPTS = parallel sql niceload \
|
||||
env_parallel env_parallel.bash env_parallel.zsh env_parallel.fish env_parallel.ksh
|
||||
env_parallel env_parallel.bash env_parallel.zsh env_parallel.fish \
|
||||
env_parallel.ksh env_parallel.pdksh env_parallel.csh
|
||||
|
||||
@DOCUMENTATION_TRUE@man_MANS = parallel.1 env_parallel.1 sem.1 sql.1 niceload.1 parallel_tutorial.7 parallel_design.7
|
||||
@DOCUMENTATION_TRUE@doc_DATA = parallel.html env_parallel.html sem.html sql.html niceload.html parallel_tutorial.html parallel_design.html \
|
||||
@DOCUMENTATION_TRUE@ parallel.texi env_parallel.texi sem.texi sql.texi niceload.texi parallel_tutorial.texi parallel_design.texi \
|
||||
@DOCUMENTATION_TRUE@ parallel.pdf env_parallel.pdf sem.pdf sql.pdf niceload.pdf parallel_tutorial.pdf parallel_design.pdf
|
||||
@DOCUMENTATION_TRUE@man_MANS = parallel.1 env_parallel.1 sem.1 sql.1 niceload.1 \
|
||||
@DOCUMENTATION_TRUE@ parallel_tutorial.7 parallel_design.7
|
||||
|
||||
@DOCUMENTATION_TRUE@doc_DATA = parallel.html env_parallel.html sem.html sql.html niceload.html \
|
||||
@DOCUMENTATION_TRUE@ parallel_tutorial.html parallel_design.html \
|
||||
@DOCUMENTATION_TRUE@ parallel.texi env_parallel.texi sem.texi sql.texi niceload.texi \
|
||||
@DOCUMENTATION_TRUE@ parallel_tutorial.texi parallel_design.texi \
|
||||
@DOCUMENTATION_TRUE@ parallel.pdf env_parallel.pdf sem.pdf sql.pdf niceload.pdf \
|
||||
@DOCUMENTATION_TRUE@ parallel_tutorial.pdf parallel_design.pdf
|
||||
|
||||
DISTCLEANFILES = parallel.1 env_parallel.1 sem.1 sql.1 niceload.1 \
|
||||
parallel_tutorial.7 parallel_design.7 \
|
||||
|
|
@ -236,7 +242,9 @@ DISTCLEANFILES = parallel.1 env_parallel.1 sem.1 sql.1 niceload.1 \
|
|||
|
||||
EXTRA_DIST = parallel sem sql niceload env_parallel \
|
||||
env_parallel.bash env_parallel.zsh env_parallel.fish env_parallel.ksh \
|
||||
sem.pod parallel.pod env_parallel.pod niceload.pod parallel_tutorial.pod parallel_design.pod \
|
||||
env_parallel.pdksh env_parallel.csh \
|
||||
sem.pod parallel.pod env_parallel.pod niceload.pod parallel_tutorial.pod \
|
||||
parallel_design.pod \
|
||||
$(DISTCLEANFILES)
|
||||
|
||||
all: all-am
|
||||
|
|
|
|||
|
|
@ -27,14 +27,14 @@ bash: Put this in $HOME/.bashrc: . `which env_parallel.bash`
|
|||
Supports: aliases, functions, variables, arrays
|
||||
|
||||
zsh: Put this in $HOME/.zshrc: . `which env_parallel.zsh`
|
||||
E.g. by doing: echo '. `which env_parallel.zsh`' >> $HOME/.zshrc
|
||||
E.g. by doing: echo '. `which env_parallel.zsh`' >> $HOME/.zshenv
|
||||
Supports: functions, variables, arrays
|
||||
|
||||
fish: Put this in $HOME/.config/fish/config.fish:
|
||||
source (which env_parallel.fish)
|
||||
E.g. by doing:
|
||||
echo 'source (which env_parallel.fish)' >> $HOME/.config/fish/config.fish
|
||||
Supports: aliases, functions, variables
|
||||
Supports: aliases, functions, variables, arrays
|
||||
|
||||
ksh: Put this in $HOME/.kshrc: source `which env_parallel.ksh`
|
||||
E.g. by doing: echo 'source `which env_parallel.ksh`' >> $HOME/.kshrc
|
||||
|
|
@ -44,6 +44,10 @@ pdksh: Put this in $HOME/.profile: source `which env_parallel.pdksh`
|
|||
E.g. by doing: echo 'source `which env_parallel.pdksh`' >> $HOME/.profile
|
||||
Supports: aliases, functions, variables, arrays
|
||||
|
||||
csh: Put this in $HOME/.cshrc: source `which env_parallel.csh`
|
||||
E.g. by doing: echo 'source `which env_parallel.csh`' >> $HOME/.cshrc
|
||||
Supports: aliases
|
||||
|
||||
For details: see man env_parallel
|
||||
|
||||
_EOS
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
#!/bin/bash
|
||||
|
||||
# This file must be sourced in bash:
|
||||
#
|
||||
# source `which env_parallel.bash`
|
||||
|
|
@ -5,7 +7,7 @@
|
|||
# after which 'env_parallel' works
|
||||
#
|
||||
#
|
||||
# Copyright (C) 2007,2008,2009,2010,2011,2012,2013,2014,2015,2016
|
||||
# Copyright (C) 2016
|
||||
# Ole Tange and Free Software Foundation, Inc.
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or modify
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
#!/usr/bin/fish
|
||||
|
||||
# This file must be sourced in fish:
|
||||
#
|
||||
# source (which env_parallel.fish)
|
||||
|
|
@ -5,7 +7,7 @@
|
|||
# after which 'env_parallel' works
|
||||
#
|
||||
#
|
||||
# Copyright (C) 2007,2008,2009,2010,2011,2012,2013,2014,2015,2016
|
||||
# Copyright (C) 2016
|
||||
# Ole Tange and Free Software Foundation, Inc.
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or modify
|
||||
|
|
@ -23,8 +25,70 @@
|
|||
# or write to the Free Software Foundation, Inc., 51 Franklin St,
|
||||
# Fifth Floor, Boston, MA 02110-1301 USA
|
||||
|
||||
# If you are a fisherman feel free to improve the code
|
||||
#
|
||||
# The code needs to deal with variables like:
|
||||
# set funky (perl -e 'print pack "c*", 2..254')
|
||||
#
|
||||
# Problem:
|
||||
# Tell the difference between:
|
||||
# set tmp "a' 'b' 'c"
|
||||
# set tmparr1 "a' 'b" 'c'
|
||||
# set tmparr2 'a' "b' 'c"
|
||||
# The output from `set` is exactly the same.
|
||||
# Solution:
|
||||
# for-loop for each variable. Each value is separated with a
|
||||
# separator.
|
||||
|
||||
function env_parallel
|
||||
setenv PARALLEL_ENV (begin; functions -n | perl -pe 's/,/\n/g' | while read d; functions $d; end; perl -e 'print map { "$_///$ENV{$_}\n"} grep !/^(PWD|SHLVL|PATH)$/, keys %ENV'| sh -c 'parallel --shellquote' | perl -pe 's:^([^/]+)///:setenv $1 :'; end |perl -pe 's/\001/\\cb/g;s/\n/\001/')
|
||||
setenv PARALLEL_ENV (
|
||||
begin;
|
||||
# Export function definitions
|
||||
functions -n | perl -pe 's/,/\n/g' | while read d; functions $d; end;
|
||||
# Convert scalar vars to fish \XX quoting
|
||||
eval (set -L | perl -ne 'chomp;
|
||||
($name,$val)=split(/ /,$_,2);
|
||||
$name=~/^(COLUMNS|FISH_VERSION|LINES|PWD|SHLVL|_|history|status|version)$/ and next;
|
||||
if($val=~/^'"'"'/) { next; }
|
||||
print "set $name \"\$$name\";\n";
|
||||
')
|
||||
# Generate commands to set scalar variables
|
||||
begin;
|
||||
for v in (set -n);
|
||||
# Separate variables with the string: \000
|
||||
eval "for i in \$$v;
|
||||
echo -n $v \$i;
|
||||
perl -e print\\\"\\\\0\\\";
|
||||
end;"
|
||||
end;
|
||||
# A final line to flush the last variable in Perl
|
||||
perl -e print\"\\0\";
|
||||
end | perl -0 -ne '
|
||||
# Remove separator string
|
||||
chop;
|
||||
($name,$val)=split(/ /,$_,2);
|
||||
# Ignore read-only vars
|
||||
$name=~/^(COLUMNS|FISH_VERSION|LINES|PWD|SHLVL|_|history|status|version)$/ and next;
|
||||
# Quote $val
|
||||
$val=~s/[\002-\011\013-\032\\\#\?\`\(\)\{\}\[\]\^\*\<\=\>\~\|\; \"\!\$\&\202-\377]/\\\$&/go;
|
||||
# Quote single quote
|
||||
$val=~s/'"'"'/\\\$&/go;
|
||||
# Quote newline as '\n'
|
||||
$val =~ s/[\n]/\\\n/go;
|
||||
if($name ne $last and $last) {
|
||||
# The $name is different, so this is a new variable.
|
||||
# Print the last one.
|
||||
# Separate list elements by 2 spaces
|
||||
$"=" ";
|
||||
print "set $last @qval;\n";
|
||||
@qval=();
|
||||
}
|
||||
push @qval,$val;
|
||||
$last=$name;
|
||||
';
|
||||
end |perl -pe 's/\001/\\cb/g and print STDERR "env_parallel: Warning: ASCII value 1 in variables is not supported\n";
|
||||
s/\n/\001/'
|
||||
)
|
||||
parallel $argv;
|
||||
set -e PARALLEL_ENV
|
||||
end
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
#!/usr/bin/ksh
|
||||
|
||||
# This file must be sourced in ksh:
|
||||
#
|
||||
# source `which env_parallel.ksh`
|
||||
|
|
@ -5,7 +7,7 @@
|
|||
# after which 'env_parallel' works
|
||||
#
|
||||
#
|
||||
# Copyright (C) 2007,2008,2009,2010,2011,2012,2013,2014,2015,2016
|
||||
# Copyright (C) 2016
|
||||
# Ole Tange and Free Software Foundation, Inc.
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or modify
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
#!/bin/pdksh
|
||||
|
||||
# This file must be sourced in ksh:
|
||||
#
|
||||
# source `which env_parallel.ksh`
|
||||
|
|
@ -5,7 +7,7 @@
|
|||
# after which 'env_parallel' works
|
||||
#
|
||||
#
|
||||
# Copyright (C) 2007,2008,2009,2010,2011,2012,2013,2014,2015,2016
|
||||
# Copyright (C) 2016
|
||||
# Ole Tange and Free Software Foundation, Inc.
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or modify
|
||||
|
|
|
|||
|
|
@ -20,6 +20,11 @@ environment to GNU Parallel.
|
|||
If the shell function is not loaded, a dummy script will be run
|
||||
instead that explains how to install the function.
|
||||
|
||||
B<env_parallel> only works if the size of the current environment is
|
||||
smaller than half of the max size and smaller than 25% of the max if
|
||||
running remotely. E.g. The max size of Bash's environment is 256 KB,
|
||||
so B<env_parallel> will fail if B<set | wc -c> is bigger than 128 KB.
|
||||
|
||||
|
||||
=head1 OPTIONS
|
||||
|
||||
|
|
@ -30,63 +35,102 @@ Same as GNU B<parallel>.
|
|||
|
||||
=head2 Bash
|
||||
|
||||
Installation
|
||||
|
||||
Put this in $HOME/.bashrc:
|
||||
|
||||
. `which env_parallel.bash`
|
||||
|
||||
E.g. by doing:
|
||||
|
||||
echo '. `which env_parallel.bash`' >> $HOME/.bashrc
|
||||
|
||||
=over 8
|
||||
|
||||
=item aliases
|
||||
|
||||
alias myecho=echo
|
||||
env_parallel myecho ::: test
|
||||
env_parallel -S server myecho ::: test
|
||||
|
||||
=item functions
|
||||
|
||||
myfunc() { echo $*; }
|
||||
env_parallel myfunc ::: test
|
||||
env_parallel -S server myfunc ::: test
|
||||
|
||||
=item variables
|
||||
|
||||
myvar=test
|
||||
env_parallel echo '$myvar' ::: test
|
||||
env_parallel -S server echo '$myvar' ::: test
|
||||
|
||||
=item arrays
|
||||
|
||||
myarray=(foo bar baz)
|
||||
env_parallel echo '${myarray[{}]}' ::: 0 1 2
|
||||
env_parallel -S server echo '${myarray[{}]}' ::: 0 1 2
|
||||
|
||||
=back
|
||||
|
||||
=head2 Zsh
|
||||
|
||||
Installation
|
||||
|
||||
Put this in $HOME/.zshrc:
|
||||
|
||||
. `which env_parallel.zsh`
|
||||
|
||||
E.g. by doing:
|
||||
|
||||
echo '. `which env_parallel.zsh`' >> $HOME/.zshenv
|
||||
|
||||
=over 8
|
||||
|
||||
=item aliases
|
||||
|
||||
Not supported
|
||||
Not supported - Zsh does not support aliases defined in the same
|
||||
parsing as they are used.
|
||||
|
||||
=item functions
|
||||
|
||||
myfunc() { echo $*; }
|
||||
env_parallel myfunc ::: test
|
||||
env_parallel -S server myfunc ::: test
|
||||
|
||||
=item variables
|
||||
|
||||
myvar=test
|
||||
env_parallel echo '$myvar' ::: test
|
||||
env_parallel -S server echo '$myvar' ::: test
|
||||
|
||||
=item arrays
|
||||
|
||||
myarray=(foo bar baz)
|
||||
env_parallel echo '${myarray[{}]}' ::: 1 2 3
|
||||
env_parallel -S server echo '${myarray[{}]}' ::: 1 2 3
|
||||
|
||||
=back
|
||||
|
||||
=head2 fish
|
||||
|
||||
Installation
|
||||
|
||||
Put this in $HOME/.config/fish/config.fish:
|
||||
|
||||
source (which env_parallel.fish)
|
||||
|
||||
E.g. by doing:
|
||||
|
||||
echo 'source (which env_parallel.fish)' >> $HOME/.config/fish/config.fish
|
||||
|
||||
=over 8
|
||||
|
||||
=item aliases
|
||||
|
||||
alias myecho=echo
|
||||
env_parallel myecho ::: test
|
||||
env_parallel -S server myecho ::: test
|
||||
|
||||
=item functions
|
||||
|
||||
|
|
@ -94,67 +138,133 @@ Not supported
|
|||
echo $argv
|
||||
end
|
||||
env_parallel myfunc ::: test
|
||||
env_parallel -S server myfunc ::: test
|
||||
|
||||
=item variables
|
||||
|
||||
set myvar test
|
||||
env_parallel echo '$myvar' ::: test
|
||||
env_parallel -S server echo '$myvar' ::: test
|
||||
|
||||
=item arrays
|
||||
|
||||
Not supported
|
||||
set myarray foo bar baz
|
||||
env_parallel echo '$myarray[{}]' ::: 1 2 3
|
||||
env_parallel -S server echo '$myarray[{}]' ::: 1 2 3
|
||||
|
||||
=back
|
||||
|
||||
=head2 ksh
|
||||
|
||||
Installation
|
||||
|
||||
Put this in $HOME/.kshrc:
|
||||
|
||||
source `which env_parallel.ksh`
|
||||
|
||||
E.g. by doing:
|
||||
|
||||
echo 'source `which env_parallel.ksh`' >> $HOME/.kshrc
|
||||
|
||||
=over 8
|
||||
|
||||
=item aliases
|
||||
|
||||
alias myecho=echo
|
||||
env_parallel myecho ::: test
|
||||
env_parallel -S server myecho ::: test
|
||||
|
||||
=item functions
|
||||
|
||||
myfunc() { echo $*; }
|
||||
env_parallel myfunc ::: test
|
||||
env_parallel -S server myfunc ::: test
|
||||
|
||||
=item variables
|
||||
|
||||
myvar=test
|
||||
env_parallel echo '$myvar' ::: test
|
||||
env_parallel -S server echo '$myvar' ::: test
|
||||
|
||||
=item arrays
|
||||
|
||||
myarray=(foo bar baz)
|
||||
env_parallel echo '${myarray[{}]}' ::: 0 1 2
|
||||
env_parallel -S server echo '${myarray[{}]}' ::: 0 1 2
|
||||
|
||||
=back
|
||||
|
||||
=head2 pdksh
|
||||
|
||||
Installation
|
||||
|
||||
Put this in $HOME/.profile:
|
||||
|
||||
source `which env_parallel.pdksh`
|
||||
|
||||
E.g. by doing:
|
||||
|
||||
echo 'source `which env_parallel.pdksh`' >> $HOME/.profile
|
||||
|
||||
=over 8
|
||||
|
||||
=item aliases
|
||||
|
||||
alias myecho=echo
|
||||
env_parallel myecho ::: test
|
||||
env_parallel -S server myecho ::: test
|
||||
|
||||
=item functions
|
||||
|
||||
myfunc() { echo $*; }
|
||||
env_parallel myfunc ::: test
|
||||
env_parallel -S server myfunc ::: test
|
||||
|
||||
=item variables
|
||||
|
||||
myvar=test
|
||||
env_parallel echo '$myvar' ::: test
|
||||
env_parallel -S server echo '$myvar' ::: test
|
||||
|
||||
=item arrays
|
||||
|
||||
myarray=(foo bar baz)
|
||||
env_parallel echo '${myarray[{}]}' ::: 0 1 2
|
||||
env_parallel -S server echo '${myarray[{}]}' ::: 0 1 2
|
||||
|
||||
=back
|
||||
|
||||
=head2 csh
|
||||
|
||||
Installation
|
||||
|
||||
Put this in $HOME/.cshrc:
|
||||
|
||||
source `which env_parallel.csh`
|
||||
|
||||
E.g. by doing:
|
||||
|
||||
echo 'source `which env_parallel.csh`' >> $HOME/.cshrc
|
||||
|
||||
=over 8
|
||||
|
||||
=item aliases
|
||||
|
||||
alias myecho echo
|
||||
env_parallel myecho ::: test
|
||||
env_parallel -S server myecho ::: test
|
||||
|
||||
=item functions
|
||||
|
||||
Not supported by B<csh>.
|
||||
|
||||
=item variables
|
||||
|
||||
Not supported
|
||||
|
||||
=item arrays
|
||||
|
||||
Not supported
|
||||
|
||||
=back
|
||||
|
||||
|
|
@ -187,9 +297,9 @@ the manual of B<xargs> from GNU findutils 4.4.2.
|
|||
|
||||
=head1 LICENSE
|
||||
|
||||
Copyright (C) 2007,2008,2009,2010,2011,2012,2013,2014,2015 Free
|
||||
Software Foundation, Inc.
|
||||
v
|
||||
Copyright (C) 2016
|
||||
Ole Tange and Free Software Foundation, Inc.
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 3 of the License, or
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
#!/usr/bin/zsh
|
||||
|
||||
# This file must be sourced in zsh:
|
||||
#
|
||||
# source =env_parallel.zsh
|
||||
|
|
@ -5,7 +7,7 @@
|
|||
# after which 'env_parallel' works
|
||||
#
|
||||
#
|
||||
# Copyright (C) 2007,2008,2009,2010,2011,2012,2013,2014,2015,2016
|
||||
# Copyright (C) 2016
|
||||
# Ole Tange and Free Software Foundation, Inc.
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or modify
|
||||
|
|
@ -24,7 +26,7 @@
|
|||
# Fifth Floor, Boston, MA 02110-1301 USA
|
||||
|
||||
env_parallel() {
|
||||
export PARALLEL_ENV="$(alias | perl -pe 's/^/alias /';typeset -p |
|
||||
export PARALLEL_ENV="$(alias | perl -pe 's/^/alias /'; typeset -p |
|
||||
grep -aFvf <(typeset -pr) |
|
||||
egrep -iav 'ZSH_EVAL_CONTEXT|LINENO=| _=|aliases|^typeset [a-z_]+$'|
|
||||
egrep -av '^(typeset -A (commands|functions|options)|typeset IFS=|..$)|cyan';
|
||||
|
|
|
|||
|
|
@ -7491,7 +7491,6 @@ sub start {
|
|||
$ENV{'PARALLEL_TMP'} = ::tmpname("par");
|
||||
::debug("run", $Global::total_running, " processes . Starting (",
|
||||
$job->seq(), "): $command\n");
|
||||
|
||||
if($opt::pipe) {
|
||||
my ($stdin_fh) = ::gensym();
|
||||
$pid = open3_setpgrp($stdin_fh,$stdout_fh,$stderr_fh,$command);
|
||||
|
|
@ -9166,6 +9165,8 @@ sub get {
|
|||
my $ret = $self->{'arg_sub_queue'}->get();
|
||||
if($ret and
|
||||
grep { index($_->orig(),"\0") > 0 } map { @$_ } $ret) {
|
||||
# Allow for \0 in position 0 because GNU Parallel uses "\0"
|
||||
# to mean no-string
|
||||
::warning("a NUL character occurred in the input.",
|
||||
"It cannot be passed through in the argument list.",
|
||||
"Did you mean to use the --null option?");
|
||||
|
|
|
|||
|
|
@ -11,8 +11,8 @@ parallel - build and execute shell command lines from standard input in parallel
|
|||
|
||||
B<parallel> [options] [I<command> [arguments]] < list_of_arguments
|
||||
|
||||
B<parallel> [options] [I<command> [arguments]] ( B<:::> arguments |
|
||||
B<::::> argfile(s) ) ...
|
||||
B<parallel> [options] [I<command> [arguments]] ( B<:::> arguments | B<:::+> arguments |
|
||||
B<::::> argfile(s) | B<::::+> argfile(s) ) ...
|
||||
|
||||
B<parallel> --semaphore [options] I<command>
|
||||
|
||||
|
|
@ -326,6 +326,18 @@ B<:::> and B<::::> can be mixed. So these are equivalent:
|
|||
::: 1 2 3
|
||||
|
||||
|
||||
=item B<:::+> I<arguments> (alpha testing)
|
||||
|
||||
Like B<:::> but linked like B<--xapply> to the previous input source.
|
||||
|
||||
Contrary to B<--xapply> values do not wrap: The shortest input source
|
||||
determines the length.
|
||||
|
||||
Example:
|
||||
|
||||
parallel echo ::: a b c :::+ 1 2 3 ::: X Y :::+ 11 22
|
||||
|
||||
|
||||
=item B<::::> I<argfiles>
|
||||
|
||||
Another way to write B<-a> I<argfile1> B<-a> I<argfile2> ...
|
||||
|
|
@ -335,6 +347,14 @@ B<:::> and B<::::> can be mixed.
|
|||
See B<-a>, B<:::> and B<--xapply>.
|
||||
|
||||
|
||||
=item B<::::+> I<argfiles> (alpha testing)
|
||||
|
||||
Like B<::::+> but linked like B<--xapply> to the previous input source.
|
||||
|
||||
Contrary to B<--xapply> values do not wrap: The shortest input source
|
||||
determines the length.
|
||||
|
||||
|
||||
=item B<--null>
|
||||
|
||||
=item B<-0>
|
||||
|
|
@ -1665,7 +1685,7 @@ Like this:
|
|||
|
||||
#!/usr/bin/parallel --shebang -r traceroute
|
||||
|
||||
foss.org.my
|
||||
qubes-os.org
|
||||
debian.org
|
||||
freenetproject.org
|
||||
|
||||
|
|
@ -1675,7 +1695,7 @@ On FreeBSD B<env> is needed:
|
|||
|
||||
#!/usr/bin/env -S parallel --shebang -r traceroute
|
||||
|
||||
foss.org.my
|
||||
qubes-os.org
|
||||
debian.org
|
||||
freenetproject.org
|
||||
|
||||
|
|
@ -2770,9 +2790,9 @@ printed as soon as possible you can use B<-u>.
|
|||
|
||||
Compare the output of:
|
||||
|
||||
parallel traceroute ::: foss.org.my debian.org freenetproject.org
|
||||
parallel --line-buffer traceroute ::: foss.org.my debian.org freenetproject.org
|
||||
parallel -u traceroute ::: foss.org.my debian.org freenetproject.org
|
||||
parallel traceroute ::: qubes-os.org debian.org freenetproject.org
|
||||
parallel --line-buffer traceroute ::: qubes-os.org debian.org freenetproject.org
|
||||
parallel -u traceroute ::: qubes-os.org debian.org freenetproject.org
|
||||
|
||||
|
||||
=head1 EXAMPLE: Tag output lines
|
||||
|
|
@ -2781,12 +2801,12 @@ GNU B<parallel> groups the output lines, but it can be hard to see
|
|||
where the different jobs begin. B<--tag> prepends the argument to make
|
||||
that more visible:
|
||||
|
||||
parallel --tag traceroute ::: foss.org.my debian.org freenetproject.org
|
||||
parallel --tag traceroute ::: qubes-os.org debian.org freenetproject.org
|
||||
|
||||
B<--tag> works with B<--line-buffer> but not with B<-u>:
|
||||
|
||||
parallel --tag --line-buffer traceroute \
|
||||
::: foss.org.my debian.org freenetproject.org
|
||||
::: qubes-os.org debian.org freenetproject.org
|
||||
|
||||
Check the uptime of the servers in I<~/.parallel/sshloginfile>:
|
||||
|
||||
|
|
@ -2810,17 +2830,17 @@ If you remove B<-k> some of the lines may come out in the wrong order.
|
|||
|
||||
Another example is B<traceroute>:
|
||||
|
||||
parallel traceroute ::: foss.org.my debian.org freenetproject.org
|
||||
parallel traceroute ::: qubes-os.org debian.org freenetproject.org
|
||||
|
||||
will give traceroute of foss.org.my, debian.org and
|
||||
will give traceroute of qubes-os.org, debian.org and
|
||||
freenetproject.org, but it will be sorted according to which job
|
||||
completed first.
|
||||
|
||||
To keep the order the same as input run:
|
||||
|
||||
parallel -k traceroute ::: foss.org.my debian.org freenetproject.org
|
||||
parallel -k traceroute ::: qubes-os.org debian.org freenetproject.org
|
||||
|
||||
This will make sure the traceroute to foss.org.my will be printed
|
||||
This will make sure the traceroute to qubes-os.org will be printed
|
||||
first.
|
||||
|
||||
A bit more complex example is downloading a huge file in chunks in
|
||||
|
|
@ -4730,7 +4750,7 @@ Copyright (C) 2007-10-18 Ole Tange, http://ole.tange.dk
|
|||
|
||||
Copyright (C) 2008,2009,2010 Ole Tange, http://ole.tange.dk
|
||||
|
||||
Copyright (C) 2010,2011,2012,2013,2014,2015 Ole Tange,
|
||||
Copyright (C) 2010,2011,2012,2013,2014,2015,2016 Ole Tange,
|
||||
http://ole.tange.dk and Free Software Foundation, Inc.
|
||||
|
||||
Parts of the manual concerning B<xargs> compatibility is inspired by
|
||||
|
|
@ -4739,7 +4759,7 @@ the manual of B<xargs> from GNU findutils 4.4.2.
|
|||
|
||||
=head1 LICENSE
|
||||
|
||||
Copyright (C) 2007,2008,2009,2010,2011,2012,2013,2014,2015 Free
|
||||
Copyright (C) 2007,2008,2009,2010,2011,2012,2013,2014,2015,2016 Free
|
||||
Software Foundation, Inc.
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
|
|
|
|||
|
|
@ -6,8 +6,8 @@
|
|||
|
||||
This document describes design decisions made in the development of
|
||||
GNU B<parallel> and the reasoning behind them. It will give an
|
||||
overview of why some of the code looks like it does, and help new
|
||||
maintainers understand the code better.
|
||||
overview of why some of the code looks the way it does, and will help
|
||||
new maintainers understand the code better.
|
||||
|
||||
|
||||
=head2 One file program
|
||||
|
|
@ -30,10 +30,10 @@ is CentOS 3.9 and Perl 5.8.0.
|
|||
=head2 Exponentially back off
|
||||
|
||||
GNU B<parallel> busy waits. This is because the reason why a job is
|
||||
not started may be due to load average, and thus it will not make
|
||||
sense to wait for a job to finish. Instead the load average must be
|
||||
checked again. Load average is not the only reason: B<--timeout> has a
|
||||
similar problem.
|
||||
not started may be due to load average (when using B<--load>), and
|
||||
thus it will not make sense to wait for a job to finish. Instead the
|
||||
load average must be checked again. Load average is not the only
|
||||
reason: B<--timeout> has a similar problem.
|
||||
|
||||
To not burn up too much CPU GNU B<parallel> sleeps exponentially
|
||||
longer and longer if nothing happens, maxing out at 1 second.
|
||||
|
|
@ -169,6 +169,20 @@ to complete; remove {} and exit with the exit code from I<input>.
|
|||
|
||||
It is made this way to be compatible with B<*csh>.
|
||||
|
||||
=item --pipepart
|
||||
|
||||
< file perl -e 'while(@ARGV) {
|
||||
sysseek(STDIN,shift,0) || die;
|
||||
$left = shift;
|
||||
while($read = sysread(STDIN,$buf, ($left > 131072 ? 131072 : $left))){
|
||||
$left -= $read;
|
||||
syswrite(STDOUT,$buf);
|
||||
}
|
||||
}' startposition length
|
||||
|
||||
This will read I<length> bytes from I<file> starting at
|
||||
I<startposition> and send it to STDOUT.
|
||||
|
||||
=item --sshlogin I<sln>
|
||||
|
||||
ssh I<sln> I<shell quoted input>
|
||||
|
|
@ -446,14 +460,13 @@ The wrapper looks like this:
|
|||
|
||||
Transferring of variables and functions given by B<--env> is done by
|
||||
running a Perl script remotely that calls the actual command. The Perl
|
||||
script sets B<$ENV{>I<variable>B<}> to the correct value before exec'ing the a
|
||||
shell that runs the function definition followed by the actual
|
||||
command.
|
||||
script sets B<$ENV{>I<variable>B<}> to the correct value before
|
||||
exec'ing the a shell that runs the function definition followed by the
|
||||
actual command.
|
||||
|
||||
B<env_parallel> (mentioned in the man page) copies the full current
|
||||
environment into the environment variable B<PARALLEL_ENV>. This
|
||||
variable is picked up by GNU B<parallel> and used to create the Perl
|
||||
script mentioned above.
|
||||
The function B<env_parallel> copies the full current environment into
|
||||
the environment variable B<PARALLEL_ENV>. This variable is picked up
|
||||
by GNU B<parallel> and used to create the Perl script mentioned above.
|
||||
|
||||
|
||||
=head2 Base64 encoded bzip2
|
||||
|
|
@ -747,7 +760,8 @@ What can be done about it?
|
|||
|
||||
=back
|
||||
|
||||
Unfortunately it is not always possible to predict the root cause of the error.
|
||||
Unfortunately it is not always possible to predict the root cause of
|
||||
the error.
|
||||
|
||||
|
||||
=head2 Computation of load
|
||||
|
|
@ -766,8 +780,8 @@ B<ps> and put the result in a file, which is then used next time.
|
|||
GNU B<parallel> kills jobs. It can be due to B<--memfree>, B<--halt>,
|
||||
or when GNU B<parallel> meets a condition from which it cannot
|
||||
recover. Every job is started as its own process group. This way any
|
||||
(grand)*children will get killed, too. The process group is killed with
|
||||
the specification mentioned in B<--termseq>.
|
||||
(grand)*children will get killed, too. The process group is killed
|
||||
with the specification mentioned in B<--termseq>.
|
||||
|
||||
|
||||
=head2 SQL interface
|
||||
|
|
@ -789,6 +803,10 @@ respectively.
|
|||
The Signal column has been renamed to _Signal due to Signal being a
|
||||
reserved word in MySQL.
|
||||
|
||||
=head2 Logo
|
||||
|
||||
The logo is inspired by the Cafe Wall illusion. The font is DejaVu
|
||||
Sans.
|
||||
|
||||
|
||||
=head1 Ideas for new design
|
||||
|
|
@ -816,16 +834,13 @@ Will that require 2x record size memory?
|
|||
|
||||
Will that require 2x block size memory?
|
||||
|
||||
=head2 Logo
|
||||
|
||||
The logo is inpired by the Cafe Wall illusion. The font is DejaVu Sans.
|
||||
|
||||
|
||||
=head1 Historical decisions
|
||||
|
||||
=head2 --tollef
|
||||
|
||||
You can read about the history of GNU B<parallel> on https://www.gnu.org/software/parallel/history.html
|
||||
You can read about the history of GNU B<parallel> on
|
||||
https://www.gnu.org/software/parallel/history.html
|
||||
|
||||
B<--tollef> was included to make GNU B<parallel> switch compatible
|
||||
with the parallel from moreutils (which is made by Tollef Fog
|
||||
|
|
@ -833,11 +848,12 @@ Heen). This was done so that users of that parallel easily could port
|
|||
their use to GNU B<parallel>: Simply set B<PARALLEL="--tollef"> and
|
||||
that would be it.
|
||||
|
||||
But several distributions chose to make B<--tollef> global (by putting it
|
||||
into /etc/parallel/config), and that caused much confusion when people
|
||||
tried out the examples from GNU B<parallel>'s man page and these did
|
||||
not work. The users became frustrated because the distribution did
|
||||
not make it clear to them that it has made B<--tollef> global.
|
||||
But several distributions chose to make B<--tollef> global (by putting
|
||||
it into /etc/parallel/config) without making the users aware of this,
|
||||
and that caused much confusion when people tried out the examples from
|
||||
GNU B<parallel>'s man page and these did not work. The users became
|
||||
frustrated because the distribution did not make it clear to them that
|
||||
it has made B<--tollef> global.
|
||||
|
||||
So to lessen the frustration and the resulting support, B<--tollef>
|
||||
was obsoleted 20130222 and removed one year later.
|
||||
|
|
|
|||
|
|
@ -1746,7 +1746,25 @@
|
|||
|
||||
<h2 id="Transferring-environment-variables-and-functions">Transferring environment variables and functions</h2>
|
||||
|
||||
<p>Using <b>--env</b> GNU <b>parallel</b> can transfer an environment variable to the remote system.</p>
|
||||
<p><b>env_parallel</b> is a shell function that transfers all aliases, functions, variables, and arrays. You active it by running:</p>
|
||||
|
||||
<pre><code> source `which env_parallel.bash`</code></pre>
|
||||
|
||||
<p>Replace <b>bash</b> with the shell you use.</p>
|
||||
|
||||
<p>Now you can use <b>env_parallel</b> instead of <b>parallel</b> and still have your environment:</p>
|
||||
|
||||
<pre><code> alias myecho=echo
|
||||
myvar="Joe's var is"
|
||||
env_parallel -S $SERVER1 'myecho $myvar' ::: green</code></pre>
|
||||
|
||||
<p>Output:</p>
|
||||
|
||||
<pre><code> Joe's var is green</code></pre>
|
||||
|
||||
<p>The disadvantage is that if your environment is huge <b>env_parallel</b> will fail.</p>
|
||||
|
||||
<p>When <b>env_parallel</b> fails, you can still use <b>--env</b> to tell GNU <b>parallel</b> to transfer an environment variable to the remote system.</p>
|
||||
|
||||
<pre><code> MYVAR='foo bar'
|
||||
export MYVAR
|
||||
|
|
@ -1797,26 +1815,19 @@
|
|||
|
||||
<h2 id="Showing-what-is-actually-run">Showing what is actually run</h2>
|
||||
|
||||
<p><b>--verbose</b> will show the command that would be run on the local machine. When a job is run on a remote machine, this is wrapped with <b>ssh</b> and possibly transferring files and environment variables, setting the workdir, and setting <b>--nice</b> value. <b>-vv</b> shows all of this.</p>
|
||||
<p><b>--verbose</b> will show the command that would be run on the local machine.</p>
|
||||
|
||||
<pre><code> parallel -vv -S $SERVER1 echo ::: bar</code></pre>
|
||||
<p>When using <b>--cat</b>, <b>--pipepart</b>, or when a job is run on a remote machine, the command is wrapped with helper scripts. <b>-vv</b> shows all of this.</p>
|
||||
|
||||
<pre><code> parallel -vv --pipepart wc :::: num30000</code></pre>
|
||||
|
||||
<p>Output:</p>
|
||||
|
||||
<pre><code> ssh lo -- exec perl -e \''@GNU_Parallel=("use","IPC::Open3;","use","MIME::Base64");
|
||||
eval"@GNU_Parallel";my$eval=decode_base64(join"",@ARGV);eval$eval;'\'
|
||||
JEVOVnsiUEFSQUxMRUxfUElEIn09IjI3MzQiOyRFTlZ7IlBBUkFMTEVMX1NFUSJ9PSIx
|
||||
IjskYmFzaGZ1bmMgPSAiIjtAQVJHVj0iZWNobyBiYXIiOyRzaGVsbD0iJEVOVntTSEVM
|
||||
TH0iOyR0bXBkaXI9Ii90bXAiOyRuaWNlPTA7ZG97JEVOVntQQVJBTExFTF9UTVB9PSR0
|
||||
bXBkaXIuIi9wYXIiLmpvaW4iIixtYXB7KDAuLjksImEiLi4ieiIsIkEiLi4iWiIpW3Jh
|
||||
bmQoNjIpXX0oMS4uNSk7fXdoaWxlKC1lJEVOVntQQVJBTExFTF9UTVB9KTskU0lHe0NI
|
||||
TER9PXN1YnskZG9uZT0xO307JHBpZD1mb3JrO3VubGVzcygkcGlkKXtzZXRwZ3JwO2V2
|
||||
YWx7c2V0cHJpb3JpdHkoMCwwLCRuaWNlKX07ZXhlYyRzaGVsbCwiLWMiLCgkYmFzaGZ1
|
||||
bmMuIkBBUkdWIik7ZGllImV4ZWM6JCFcbiI7fWRveyRzPSRzPDE/MC4wMDErJHMqMS4w
|
||||
MzokcztzZWxlY3QodW5kZWYsdW5kZWYsdW5kZWYsJHMpO311bnRpbCgkZG9uZXx8Z2V0
|
||||
cHBpZD09MSk7a2lsbChTSUdIVVAsLSR7cGlkfSl1bmxlc3MkZG9uZTt3YWl0O2V4aXQo
|
||||
JD8mMTI3PzEyOCsoJD8mMTI3KToxKyQ/Pj44KQ==;
|
||||
bar</code></pre>
|
||||
<pre><code> <num30000 perl -e 'while(@ARGV) { sysseek(STDIN,shift,0) || die;
|
||||
$left = shift; while($read = sysread(STDIN,$buf, ($left > 131072
|
||||
? 131072 : $left))){ $left -= $read; syswrite(STDOUT,$buf); } }'
|
||||
0 0 0 168894 | (wc)
|
||||
30000 30000 168894</code></pre>
|
||||
|
||||
<p>When the command gets more complex, the output is so hard to read, that it is only useful for debugging:</p>
|
||||
|
||||
|
|
@ -1828,44 +1839,40 @@
|
|||
|
||||
<p>Output will be similar to:</p>
|
||||
|
||||
<pre><code> ( ssh lo -- mkdir -p ./.parallel/tmp/hk-3492-1;rsync --protocol 30
|
||||
-rlDzR -essh ./abc-file lo:./.parallel/tmp/hk-3492-1 );ssh lo --
|
||||
exec perl -e \''@GNU_Parallel=("use","IPC::Open3;","use","MIME::Base64");
|
||||
<pre><code> ( ssh server -- mkdir -p ./.parallel/tmp/aspire-1928520-1;rsync
|
||||
--protocol 30 -rlDzR -essh ./abc-file
|
||||
server:./.parallel/tmp/aspire-1928520-1 );ssh server -- exec perl -e
|
||||
\''@GNU_Parallel=("use","IPC::Open3;","use","MIME::Base64");
|
||||
eval"@GNU_Parallel";my$eval=decode_base64(join"",@ARGV);eval$eval;'\'
|
||||
c3lzdGVtKCJta2RpciIsIi1wIiwiLS0iLCIucGFyYWxsZWwvdG1wL2hrLTM0OTItMSIp
|
||||
OyBjaGRpciAiLnBhcmFsbGVsL3RtcC9oay0zNDkyLTEiIHx8cHJpbnQoU1RERVJSICJw
|
||||
YXJhbGxlbDogQ2Fubm90IGNoZGlyIHRvIC5wYXJhbGxlbC90bXAvaGstMzQ5Mi0xXG4i
|
||||
KSAmJiBleGl0IDI1NTskRU5WeyJHUEdfQUdFTlRfSU5GTyJ9PSIvdG1wL2dwZy10WjVI
|
||||
U0QvUy5ncGctYWdlbnQ6MjM5NzoxIjskRU5WeyJQQVJBTExFTF9TRVEifT0iMSI7JEVO
|
||||
VnsiU1FMSVRFVEJMIn09InNxbGl0ZTM6Ly8vJTJGdG1wJTJGcGFyYWxsZWwuZGIyL3Bh
|
||||
cnNxbDIiOyRFTlZ7IlBBUkFMTEVMX1BJRCJ9PSIzNDkyIjskRU5WeyJTUUxJVEUifT0i
|
||||
c3FsaXRlMzovLy8lMkZ0bXAlMkZwYXJhbGxlbC5kYjIiOyRFTlZ7IlBBUkFMTEVMX1BJ
|
||||
RCJ9PSIzNDkyIjskRU5WeyJQQVJBTExFTF9TRVEifT0iMSI7QGJhc2hfZnVuY3Rpb25z
|
||||
PXF3KG15X2Z1bmMzKTsgaWYoJEVOVnsiU0hFTEwifT1+L2NzaC8pIHsgcHJpbnQgU1RE
|
||||
RVJSICJDU0gvVENTSCBETyBOT1QgU1VQUE9SVCBuZXdsaW5lcyBJTiBWQVJJQUJMRVMv
|
||||
RlVOQ1RJT05TLiBVbnNldCBAYmFzaF9mdW5jdGlvbnNcbiI7IGV4ZWMgImZhbHNlIjsg
|
||||
fSAKJGJhc2hmdW5jID0gIm15X2Z1bmMzKCkgeyAgZWNobyBpbiBteV9mdW5jIFwkMSA+
|
||||
IFwkMS5vdXQKfTtleHBvcnQgLWYgbXlfZnVuYzMgPi9kZXYvbnVsbDsiO0BBUkdWPSJt
|
||||
eV9mdW5jMyBhYmMtZmlsZSI7JHNoZWxsPSIkRU5We1NIRUxM
|
||||
fSI7JHRtcGRpcj0iL3RtcCI7JG5pY2U9MTc7ZG97JEVOVntQQVJBTExFTF9UTVB9PSR0
|
||||
bXBkaXIuIi9wYXIiLmpvaW4iIixtYXB7KDAuLjksImEiLi4ieiIsIkEiLi4iWiIpW3Jh
|
||||
bmQoNjIpXX0oMS4uNSk7fXdoaWxlKC1lJEVOVntQQVJBTExFTF9UTVB9KTskU0lHe0NI
|
||||
TER9PXN1YnskZG9uZT0xO307JHBpZD1mb3JrO3VubGVzcygkcGlkKXtzZXRwZ3JwO2V2
|
||||
YWx7c2V0cHJpb3JpdHkoMCwwLCRuaWNlKX07ZXhlYyRzaGVsbCwiLWMiLCgkYmFzaGZ1
|
||||
bmMuIkBBUkdWIik7ZGllImV4ZWM6JCFcbiI7fWRveyRzPSRzPDE/MC4wMDErJHMqMS4w
|
||||
MzokcztzZWxlY3QodW5kZWYsdW5kZWYsdW5kZWYsJHMpO311bnRpbCgkZG9uZXx8Z2V0
|
||||
cHBpZD09MSk7a2lsbChTSUdIVVAsLSR7cGlkfSl1bmxlc3MkZG9uZTt3YWl0O2V4aXQo
|
||||
JD8mMTI3PzEyOCsoJD8mMTI3KToxKyQ/Pj44KQ==;_EXIT_status=$?;
|
||||
mkdir -p ./.; rsync --protocol 30 --rsync-path=cd\
|
||||
./.parallel/tmp/hk-3492-1/./.\;\ rsync -rlDzR -essh
|
||||
lo:./abc-file.out ./.;ssh lo -- \(rm\ -f\
|
||||
./.parallel/tmp/hk-3492-1/abc-file\;\ sh\ -c\ \'rmdir\
|
||||
./.parallel/tmp/hk-3492-1/\ ./.parallel/tmp/\ ./.parallel/\
|
||||
2\>/dev/null\'\;rm\ -rf\ ./.parallel/tmp/hk-3492-1\;\);ssh lo --
|
||||
\(rm\ -f\ ./.parallel/tmp/hk-3492-1/abc-file.out\;\ sh\ -c\ \'rmdir\
|
||||
./.parallel/tmp/hk-3492-1/\ ./.parallel/tmp/\ ./.parallel/\
|
||||
2\>/dev/null\'\;rm\ -rf\ ./.parallel/tmp/hk-3492-1\;\);ssh lo -- rm
|
||||
-rf .parallel/tmp/hk-3492-1; exit $_EXIT_status;</code></pre>
|
||||
c3lzdGVtKCJta2RpciIsIi1wIiwiLS0iLCIucGFyYWxsZWwvdG1wL2FzcGlyZS0xOTI4N
|
||||
TsgY2hkaXIgIi5wYXJhbGxlbC90bXAvYXNwaXJlLTE5Mjg1MjAtMSIgfHxwcmludChTVE
|
||||
BhcmFsbGVsOiBDYW5ub3QgY2hkaXIgdG8gLnBhcmFsbGVsL3RtcC9hc3BpcmUtMTkyODU
|
||||
iKSAmJiBleGl0IDI1NTskRU5WeyJPTERQV0QifT0iL2hvbWUvdGFuZ2UvcHJpdmF0L3Bh
|
||||
IjskRU5WeyJQQVJBTExFTF9QSUQifT0iMTkyODUyMCI7JEVOVnsiUEFSQUxMRUxfU0VRI
|
||||
0BiYXNoX2Z1bmN0aW9ucz1xdyhteV9mdW5jMyk7IGlmKCRFTlZ7IlNIRUxMIn09fi9jc2
|
||||
ByaW50IFNUREVSUiAiQ1NIL1RDU0ggRE8gTk9UIFNVUFBPUlQgbmV3bGluZXMgSU4gVkF
|
||||
TL0ZVTkNUSU9OUy4gVW5zZXQgQGJhc2hfZnVuY3Rpb25zXG4iOyBleGVjICJmYWxzZSI7
|
||||
YXNoZnVuYyA9ICJteV9mdW5jMygpIHsgIGVjaG8gaW4gbXlfZnVuYyBcJDEgPiBcJDEub
|
||||
Xhwb3J0IC1mIG15X2Z1bmMzID4vZGV2L251bGw7IjtAQVJHVj0ibXlfZnVuYzMgYWJjLW
|
||||
RzaGVsbD0iJEVOVntTSEVMTH0iOyR0bXBkaXI9Ii90bXAiOyRuaWNlPTE3O2RveyRFTlZ
|
||||
MRUxfVE1QfT0kdG1wZGlyLiIvcGFyIi5qb2luIiIsbWFweygwLi45LCJhIi4uInoiLCJB
|
||||
KVtyYW5kKDYyKV19KDEuLjUpO313aGlsZSgtZSRFTlZ7UEFSQUxMRUxfVE1QfSk7JFNJ
|
||||
fT1zdWJ7JGRvbmU9MTt9OyRwaWQ9Zm9yazt1bmxlc3MoJHBpZCl7c2V0cGdycDtldmFse
|
||||
W9yaXR5KDAsMCwkbmljZSl9O2V4ZWMkc2hlbGwsIi1jIiwoJGJhc2hmdW5jLiJAQVJHVi
|
||||
JleGVjOiQhXG4iO31kb3skcz0kczwxPzAuMDAxKyRzKjEuMDM6JHM7c2VsZWN0KHVuZGV
|
||||
mLHVuZGVmLCRzKTt9dW50aWwoJGRvbmV8fGdldHBwaWQ9PTEpO2tpbGwoU0lHSFVQLC0k
|
||||
dW5sZXNzJGRvbmU7d2FpdDtleGl0KCQ/JjEyNz8xMjgrKCQ/JjEyNyk6MSskPz4+OCk=;
|
||||
_EXIT_status=$?; mkdir -p ./.; rsync --protocol 30 --rsync-path=cd\
|
||||
./.parallel/tmp/aspire-1928520-1/./.\;\ rsync -rlDzR -essh
|
||||
server:./abc-file.out ./.;ssh server -- \(rm\ -f\
|
||||
./.parallel/tmp/aspire-1928520-1/abc-file\;\ sh\ -c\ \'rmdir\
|
||||
./.parallel/tmp/aspire-1928520-1/\ ./.parallel/tmp/\ ./.parallel/\
|
||||
2\>/dev/null\'\;rm\ -rf\ ./.parallel/tmp/aspire-1928520-1\;\);ssh
|
||||
server -- \(rm\ -f\ ./.parallel/tmp/aspire-1928520-1/abc-file.out\;\
|
||||
sh\ -c\ \'rmdir\ ./.parallel/tmp/aspire-1928520-1/\ ./.parallel/tmp/\
|
||||
./.parallel/\ 2\>/dev/null\'\;rm\ -rf\
|
||||
./.parallel/tmp/aspire-1928520-1\;\);ssh server -- rm -rf
|
||||
.parallel/tmp/aspire-1928520-1; exit $_EXIT_status;</code></pre>
|
||||
|
||||
<h1 id="Saving-to-an-SQL-base-advanced">Saving to an SQL base (advanced)</h1>
|
||||
|
||||
|
|
@ -1901,6 +1908,18 @@
|
|||
|
||||
<p>A proper CSV reader (like LibreOffice or R's read.csv) will read this format correctly - even with fields containing newlines as above.</p>
|
||||
|
||||
<p>If the output is big you may want to put it into files using <b>--results</b>:</p>
|
||||
|
||||
<pre><code> parallel --results outdir --sqlandworker csv:////%2Ftmp%2Flog2.csv seq ::: 10 ::: 12 13 14
|
||||
cat /tmp/log2.csv</code></pre>
|
||||
|
||||
<p>Output will be similar to:</p>
|
||||
|
||||
<pre><code> Seq,Host,Starttime,JobRuntime,Send,Receive,Exitval,_Signal,Command,V1,V2,Stdout,Stderr
|
||||
1,:,1458824738.287,0.029,0,9,0,0,"seq 10 12",10,12,outdir/1/10/2/12/stdout,outdir/1/10/2/12/stderr
|
||||
2,:,1458824738.298,0.025,0,12,0,0,"seq 10 13",10,13,outdir/1/10/2/13/stdout,outdir/1/10/2/13/stderr
|
||||
3,:,1458824738.309,0.026,0,15,0,0,"seq 10 14",10,14,outdir/1/10/2/14/stdout,outdir/1/10/2/14/stderr</code></pre>
|
||||
|
||||
<h2 id="DBURL-as-table">DBURL as table</h2>
|
||||
|
||||
<p>The CSV file is an example of a DBURL.</p>
|
||||
|
|
@ -1984,7 +2003,7 @@
|
|||
149796 149796 1048572
|
||||
85349 85349 597444</code></pre>
|
||||
|
||||
<p>The size of the chunk is not exactly 1 MB because GNU <b>parallel</b> only passes full lines - never half a line, thus the blocksize is only average 1 MB. You can change the block size to 2 MB with <b>--block</b>:</p>
|
||||
<p>The size of the chunk is not exactly 1 MB because GNU <b>parallel</b> only passes full lines - never half a line, thus the blocksize is only 1 MB on average. You can change the block size to 2 MB with <b>--block</b>:</p>
|
||||
|
||||
<pre><code> cat num1000000 | parallel --pipe --block 2M wc</code></pre>
|
||||
|
||||
|
|
@ -1995,7 +2014,7 @@
|
|||
299593 299593 2097151
|
||||
85349 85349 597444</code></pre>
|
||||
|
||||
<p>GNU <b>parallel</b> treats each line as a record. If the order of record is unimportant (e.g. you need all lines processed, but you do not care which is processed first), then you can use <b>--round-robin</b>. Without <b>--round-robin</b> GNU <b>parallel</b> will start a command per block; with <b>--round-robin</b> only the requested number of jobs will be started (<b>--jobs</b>). The records will then be distributed between the running jobs:</p>
|
||||
<p>GNU <b>parallel</b> treats each line as a record. If the order of records is unimportant (e.g. you need all lines processed, but you do not care which is processed first), then you can use <b>--round-robin</b>. Without <b>--round-robin</b> GNU <b>parallel</b> will start a command per block; with <b>--round-robin</b> only the requested number of jobs will be started (<b>--jobs</b>). The records will then be distributed between the running jobs:</p>
|
||||
|
||||
<pre><code> cat num1000000 | parallel --pipe -j4 --round-robin wc</code></pre>
|
||||
|
||||
|
|
@ -2027,7 +2046,7 @@
|
|||
140000 140000 980000
|
||||
20000 20000 140001</code></pre>
|
||||
|
||||
<p>Notice that the last job could not get the full 140000 lines, but only 20000 lines.</p>
|
||||
<p>Note how that the last job could not get the full 140000 lines, but only 20000 lines.</p>
|
||||
|
||||
<p>If a record is 75 lines <b>-L</b> can be used:</p>
|
||||
|
||||
|
|
@ -2044,7 +2063,7 @@
|
|||
85350 85350 597450
|
||||
25 25 176</code></pre>
|
||||
|
||||
<p>Notice GNU <b>parallel</b> still reads a block of around 1 MB; but instead of passing full lines to <b>wc</b> it passes full 75 lines at a time. This of course does not hold for the last job (which in this case got 25 lines).</p>
|
||||
<p>Note how GNU <b>parallel</b> still reads a block of around 1 MB; but instead of passing full lines to <b>wc</b> it passes full 75 lines at a time. This of course does not hold for the last job (which in this case got 25 lines).</p>
|
||||
|
||||
<h2 id="Record-separators">Record separators</h2>
|
||||
|
||||
|
|
@ -2386,7 +2405,7 @@
|
|||
|
||||
<p>GNU <b>parallel</b> can work as a counting semaphore. This is slower and less efficient than its normal mode.</p>
|
||||
|
||||
<p>A counting semaphore is like a row of toilets. People needing a toilet can use any toilet, but if there are more people than toilets, they will have to wait for one of the toilets to be available.</p>
|
||||
<p>A counting semaphore is like a row of toilets. People needing a toilet can use any toilet, but if there are more people than toilets, they will have to wait for one of the toilets to become available.</p>
|
||||
|
||||
<p>An alias for <b>parallel --semaphore</b> is <b>sem</b>.</p>
|
||||
|
||||
|
|
@ -2427,7 +2446,7 @@
|
|||
|
||||
<p>The difference between this and just running the command, is that a mutex is set, so if other <b>sem</b>s were running in the background only one would run at a time.</p>
|
||||
|
||||
<p>To tell the difference between which semaphore is used, use <b>--semaphorename</b>/<b>--id</b>. Run this in one terminal:</p>
|
||||
<p>To control which semaphore is used, use <b>--semaphorename</b>/<b>--id</b>. Run this in one terminal:</p>
|
||||
|
||||
<pre><code> sem --id my_id -u 'echo First started; sleep 10; echo The first finished'</code></pre>
|
||||
|
||||
|
|
@ -2490,6 +2509,7 @@
|
|||
<p>Output:</p>
|
||||
|
||||
<pre><code> Usage:
|
||||
|
||||
parallel [options] [command [arguments]] < list_of_arguments
|
||||
parallel [options] [command [arguments]] (::: arguments|:::: argfile(s))...
|
||||
cat ... | parallel --pipe [options] [command [arguments]]
|
||||
|
|
@ -2498,8 +2518,10 @@
|
|||
-k Keep same order
|
||||
-X Multiple arguments with context replace
|
||||
--colsep regexp Split input on regexp for positional replacements
|
||||
{} {.} {/} {/.} {#} Replacement strings
|
||||
{3} {3.} {3/} {3/.} Positional replacement strings
|
||||
{} {.} {/} {/.} {#} {%} {= perl code =} Replacement strings
|
||||
{3} {3.} {3/} {3/.} {=3 perl code =} Positional replacement strings
|
||||
With --plus: {} = {+/}/{/} = {.}.{+.} = {+/}/{/.}.{+.} = {..}.{+..} =
|
||||
{+/}/{/..}.{+..} = {...}.{+...} = {+/}/{/...}.{+...}
|
||||
|
||||
-S sshlogin Example: foo@server.example.com
|
||||
--slf .. Use ~/.parallel/sshloginfile as the list of sshlogins
|
||||
|
|
@ -2513,29 +2535,33 @@
|
|||
|
||||
See 'man parallel' for details
|
||||
|
||||
When using GNU Parallel for a publication please cite:
|
||||
Academic tradition requires you to cite works you base your article on.
|
||||
When using programs that use GNU Parallel to process data for publication
|
||||
please cite:
|
||||
|
||||
O. Tange (2011): GNU Parallel - The Command-Line Power Tool,
|
||||
;login: The USENIX Magazine, February 2011:42-47.
|
||||
|
||||
When asking for help, always report the full output of this:
|
||||
This helps funding further development; AND IT WON'T COST YOU A CENT.
|
||||
If you pay 10000 EUR you should feel free to use GNU Parallel without citing.</code></pre>
|
||||
|
||||
parallel --version</code></pre>
|
||||
<p>When asking for help, always report the full output of this:</p>
|
||||
|
||||
<pre><code> parallel --version</code></pre>
|
||||
|
||||
<p>Output:</p>
|
||||
|
||||
<pre><code> GNU parallel 20130822
|
||||
Copyright (C) 2007,2008,2009,2010,2011,2012,2013 Ole Tange and Free Software Foundation, Inc.
|
||||
<pre><code> GNU parallel 20160323
|
||||
Copyright (C) 2007,2008,2009,2010,2011,2012,2013,2014,2015,2016
|
||||
Ole Tange and Free Software Foundation, Inc.
|
||||
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
|
||||
This is free software: you are free to change and redistribute it.
|
||||
GNU parallel comes with no warranty.
|
||||
|
||||
Web site: http://www.gnu.org/software/parallel
|
||||
|
||||
When using GNU Parallel for a publication please cite:
|
||||
|
||||
O. Tange (2011): GNU Parallel - The Command-Line Power Tool,
|
||||
;login: The USENIX Magazine, February 2011:42-47.</code></pre>
|
||||
When using programs that use GNU Parallel to process data for publication
|
||||
please cite as described in 'parallel --bibtex'.</code></pre>
|
||||
|
||||
<p>In scripts <b>--minversion</b> can be used to ensure the user has at least this version:</p>
|
||||
|
||||
|
|
@ -2543,7 +2569,7 @@
|
|||
|
||||
<p>Output:</p>
|
||||
|
||||
<pre><code> 20130722
|
||||
<pre><code> 20160322
|
||||
Your version is at least 20130722.</code></pre>
|
||||
|
||||
<p>If using GNU <b>parallel</b> for research the BibTeX citation can be generated using <b>--bibtex</b>:</p>
|
||||
|
|
@ -2552,7 +2578,11 @@
|
|||
|
||||
<p>Output:</p>
|
||||
|
||||
<pre><code> @article{Tange2011a,
|
||||
<pre><code> Academic tradition requires you to cite works you base your article on.
|
||||
When using programs that use GNU Parallel to process data for publication
|
||||
please cite:
|
||||
|
||||
@article{Tange2011a,
|
||||
title = {GNU Parallel - The Command-Line Power Tool},
|
||||
author = {O. Tange},
|
||||
address = {Frederiksberg, Denmark},
|
||||
|
|
@ -2562,8 +2592,18 @@
|
|||
volume = {36},
|
||||
url = {http://www.gnu.org/s/parallel},
|
||||
year = {2011},
|
||||
pages = {42-47}
|
||||
}</code></pre>
|
||||
pages = {42-47},
|
||||
doi = {10.5281/zenodo.16303}
|
||||
}
|
||||
|
||||
(Feel free to use \nocite{Tange2011a})
|
||||
|
||||
This helps funding further development; AND IT WON'T COST YOU A CENT.
|
||||
If you pay 10000 EUR you should feel free to use GNU Parallel without citing.
|
||||
|
||||
If you send a copy of your published article to tange@gnu.org, it will be
|
||||
mentioned in the release notes of next version of GNU Parallel.
|
||||
</code></pre>
|
||||
|
||||
<p>With <b>--max-line-length-allowed</b> GNU <b>parallel</b> will report the maximal size of the command line:</p>
|
||||
|
||||
|
|
@ -2606,9 +2646,9 @@
|
|||
|
||||
<p>Output:</p>
|
||||
|
||||
<pre><code> \nice -n17 /bin/bash -c echo\ A
|
||||
\nice -n17 /bin/bash -c echo\ B
|
||||
\nice -n17 /bin/bash -c echo\ C</code></pre>
|
||||
<pre><code> echo A
|
||||
echo B
|
||||
echo C</code></pre>
|
||||
|
||||
<h1 id="Spread-the-word">Spread the word</h1>
|
||||
|
||||
|
|
|
|||
|
|
@ -1731,8 +1731,29 @@ B<--jobs> sets the number of servers to log in to in parallel.
|
|||
|
||||
=head2 Transferring environment variables and functions
|
||||
|
||||
Using B<--env> GNU B<parallel> can transfer an environment variable to the
|
||||
remote system.
|
||||
B<env_parallel> is a shell function that transfers all aliases,
|
||||
functions, variables, and arrays. You active it by running:
|
||||
|
||||
source `which env_parallel.bash`
|
||||
|
||||
Replace B<bash> with the shell you use.
|
||||
|
||||
Now you can use B<env_parallel> instead of B<parallel> and still have
|
||||
your environment:
|
||||
|
||||
alias myecho=echo
|
||||
myvar="Joe's var is"
|
||||
env_parallel -S $SERVER1 'myecho $myvar' ::: green
|
||||
|
||||
Output:
|
||||
|
||||
Joe's var is green
|
||||
|
||||
The disadvantage is that if your environment is huge B<env_parallel>
|
||||
will fail.
|
||||
|
||||
When B<env_parallel> fails, you can still use B<--env> to tell GNU
|
||||
B<parallel> to transfer an environment variable to the remote system.
|
||||
|
||||
MYVAR='foo bar'
|
||||
export MYVAR
|
||||
|
|
@ -1784,31 +1805,26 @@ Output:
|
|||
foo
|
||||
in my_func2 foo bar
|
||||
|
||||
|
||||
|
||||
=head2 Showing what is actually run
|
||||
|
||||
B<--verbose> will show the command that would be run on the local
|
||||
machine. When a job is run on a remote machine, this is wrapped with
|
||||
B<ssh> and possibly transferring files and environment variables, setting
|
||||
the workdir, and setting B<--nice> value. B<-vv> shows all of this.
|
||||
machine.
|
||||
|
||||
parallel -vv -S $SERVER1 echo ::: bar
|
||||
When using B<--cat>, B<--pipepart>, or when a job is run on a remote
|
||||
machine, the command is wrapped with helper scripts. B<-vv> shows all
|
||||
of this.
|
||||
|
||||
parallel -vv --pipepart wc :::: num30000
|
||||
|
||||
Output:
|
||||
|
||||
ssh lo -- exec perl -e \''@GNU_Parallel=("use","IPC::Open3;","use","MIME::Base64");
|
||||
eval"@GNU_Parallel";my$eval=decode_base64(join"",@ARGV);eval$eval;'\'
|
||||
JEVOVnsiUEFSQUxMRUxfUElEIn09IjI3MzQiOyRFTlZ7IlBBUkFMTEVMX1NFUSJ9PSIx
|
||||
IjskYmFzaGZ1bmMgPSAiIjtAQVJHVj0iZWNobyBiYXIiOyRzaGVsbD0iJEVOVntTSEVM
|
||||
TH0iOyR0bXBkaXI9Ii90bXAiOyRuaWNlPTA7ZG97JEVOVntQQVJBTExFTF9UTVB9PSR0
|
||||
bXBkaXIuIi9wYXIiLmpvaW4iIixtYXB7KDAuLjksImEiLi4ieiIsIkEiLi4iWiIpW3Jh
|
||||
bmQoNjIpXX0oMS4uNSk7fXdoaWxlKC1lJEVOVntQQVJBTExFTF9UTVB9KTskU0lHe0NI
|
||||
TER9PXN1YnskZG9uZT0xO307JHBpZD1mb3JrO3VubGVzcygkcGlkKXtzZXRwZ3JwO2V2
|
||||
YWx7c2V0cHJpb3JpdHkoMCwwLCRuaWNlKX07ZXhlYyRzaGVsbCwiLWMiLCgkYmFzaGZ1
|
||||
bmMuIkBBUkdWIik7ZGllImV4ZWM6JCFcbiI7fWRveyRzPSRzPDE/MC4wMDErJHMqMS4w
|
||||
MzokcztzZWxlY3QodW5kZWYsdW5kZWYsdW5kZWYsJHMpO311bnRpbCgkZG9uZXx8Z2V0
|
||||
cHBpZD09MSk7a2lsbChTSUdIVVAsLSR7cGlkfSl1bmxlc3MkZG9uZTt3YWl0O2V4aXQo
|
||||
JD8mMTI3PzEyOCsoJD8mMTI3KToxKyQ/Pj44KQ==;
|
||||
bar
|
||||
<num30000 perl -e 'while(@ARGV) { sysseek(STDIN,shift,0) || die;
|
||||
$left = shift; while($read = sysread(STDIN,$buf, ($left > 131072
|
||||
? 131072 : $left))){ $left -= $read; syswrite(STDOUT,$buf); } }'
|
||||
0 0 0 168894 | (wc)
|
||||
30000 30000 168894
|
||||
|
||||
When the command gets more complex, the output is so hard to read,
|
||||
that it is only useful for debugging:
|
||||
|
|
@ -1821,44 +1837,41 @@ that it is only useful for debugging:
|
|||
|
||||
Output will be similar to:
|
||||
|
||||
( ssh lo -- mkdir -p ./.parallel/tmp/hk-3492-1;rsync --protocol 30
|
||||
-rlDzR -essh ./abc-file lo:./.parallel/tmp/hk-3492-1 );ssh lo --
|
||||
exec perl -e \''@GNU_Parallel=("use","IPC::Open3;","use","MIME::Base64");
|
||||
|
||||
( ssh server -- mkdir -p ./.parallel/tmp/aspire-1928520-1;rsync
|
||||
--protocol 30 -rlDzR -essh ./abc-file
|
||||
server:./.parallel/tmp/aspire-1928520-1 );ssh server -- exec perl -e
|
||||
\''@GNU_Parallel=("use","IPC::Open3;","use","MIME::Base64");
|
||||
eval"@GNU_Parallel";my$eval=decode_base64(join"",@ARGV);eval$eval;'\'
|
||||
c3lzdGVtKCJta2RpciIsIi1wIiwiLS0iLCIucGFyYWxsZWwvdG1wL2hrLTM0OTItMSIp
|
||||
OyBjaGRpciAiLnBhcmFsbGVsL3RtcC9oay0zNDkyLTEiIHx8cHJpbnQoU1RERVJSICJw
|
||||
YXJhbGxlbDogQ2Fubm90IGNoZGlyIHRvIC5wYXJhbGxlbC90bXAvaGstMzQ5Mi0xXG4i
|
||||
KSAmJiBleGl0IDI1NTskRU5WeyJHUEdfQUdFTlRfSU5GTyJ9PSIvdG1wL2dwZy10WjVI
|
||||
U0QvUy5ncGctYWdlbnQ6MjM5NzoxIjskRU5WeyJQQVJBTExFTF9TRVEifT0iMSI7JEVO
|
||||
VnsiU1FMSVRFVEJMIn09InNxbGl0ZTM6Ly8vJTJGdG1wJTJGcGFyYWxsZWwuZGIyL3Bh
|
||||
cnNxbDIiOyRFTlZ7IlBBUkFMTEVMX1BJRCJ9PSIzNDkyIjskRU5WeyJTUUxJVEUifT0i
|
||||
c3FsaXRlMzovLy8lMkZ0bXAlMkZwYXJhbGxlbC5kYjIiOyRFTlZ7IlBBUkFMTEVMX1BJ
|
||||
RCJ9PSIzNDkyIjskRU5WeyJQQVJBTExFTF9TRVEifT0iMSI7QGJhc2hfZnVuY3Rpb25z
|
||||
PXF3KG15X2Z1bmMzKTsgaWYoJEVOVnsiU0hFTEwifT1+L2NzaC8pIHsgcHJpbnQgU1RE
|
||||
RVJSICJDU0gvVENTSCBETyBOT1QgU1VQUE9SVCBuZXdsaW5lcyBJTiBWQVJJQUJMRVMv
|
||||
RlVOQ1RJT05TLiBVbnNldCBAYmFzaF9mdW5jdGlvbnNcbiI7IGV4ZWMgImZhbHNlIjsg
|
||||
fSAKJGJhc2hmdW5jID0gIm15X2Z1bmMzKCkgeyAgZWNobyBpbiBteV9mdW5jIFwkMSA+
|
||||
IFwkMS5vdXQKfTtleHBvcnQgLWYgbXlfZnVuYzMgPi9kZXYvbnVsbDsiO0BBUkdWPSJt
|
||||
eV9mdW5jMyBhYmMtZmlsZSI7JHNoZWxsPSIkRU5We1NIRUxM
|
||||
fSI7JHRtcGRpcj0iL3RtcCI7JG5pY2U9MTc7ZG97JEVOVntQQVJBTExFTF9UTVB9PSR0
|
||||
bXBkaXIuIi9wYXIiLmpvaW4iIixtYXB7KDAuLjksImEiLi4ieiIsIkEiLi4iWiIpW3Jh
|
||||
bmQoNjIpXX0oMS4uNSk7fXdoaWxlKC1lJEVOVntQQVJBTExFTF9UTVB9KTskU0lHe0NI
|
||||
TER9PXN1YnskZG9uZT0xO307JHBpZD1mb3JrO3VubGVzcygkcGlkKXtzZXRwZ3JwO2V2
|
||||
YWx7c2V0cHJpb3JpdHkoMCwwLCRuaWNlKX07ZXhlYyRzaGVsbCwiLWMiLCgkYmFzaGZ1
|
||||
bmMuIkBBUkdWIik7ZGllImV4ZWM6JCFcbiI7fWRveyRzPSRzPDE/MC4wMDErJHMqMS4w
|
||||
MzokcztzZWxlY3QodW5kZWYsdW5kZWYsdW5kZWYsJHMpO311bnRpbCgkZG9uZXx8Z2V0
|
||||
cHBpZD09MSk7a2lsbChTSUdIVVAsLSR7cGlkfSl1bmxlc3MkZG9uZTt3YWl0O2V4aXQo
|
||||
JD8mMTI3PzEyOCsoJD8mMTI3KToxKyQ/Pj44KQ==;_EXIT_status=$?;
|
||||
mkdir -p ./.; rsync --protocol 30 --rsync-path=cd\
|
||||
./.parallel/tmp/hk-3492-1/./.\;\ rsync -rlDzR -essh
|
||||
lo:./abc-file.out ./.;ssh lo -- \(rm\ -f\
|
||||
./.parallel/tmp/hk-3492-1/abc-file\;\ sh\ -c\ \'rmdir\
|
||||
./.parallel/tmp/hk-3492-1/\ ./.parallel/tmp/\ ./.parallel/\
|
||||
2\>/dev/null\'\;rm\ -rf\ ./.parallel/tmp/hk-3492-1\;\);ssh lo --
|
||||
\(rm\ -f\ ./.parallel/tmp/hk-3492-1/abc-file.out\;\ sh\ -c\ \'rmdir\
|
||||
./.parallel/tmp/hk-3492-1/\ ./.parallel/tmp/\ ./.parallel/\
|
||||
2\>/dev/null\'\;rm\ -rf\ ./.parallel/tmp/hk-3492-1\;\);ssh lo -- rm
|
||||
-rf .parallel/tmp/hk-3492-1; exit $_EXIT_status;
|
||||
c3lzdGVtKCJta2RpciIsIi1wIiwiLS0iLCIucGFyYWxsZWwvdG1wL2FzcGlyZS0xOTI4N
|
||||
TsgY2hkaXIgIi5wYXJhbGxlbC90bXAvYXNwaXJlLTE5Mjg1MjAtMSIgfHxwcmludChTVE
|
||||
BhcmFsbGVsOiBDYW5ub3QgY2hkaXIgdG8gLnBhcmFsbGVsL3RtcC9hc3BpcmUtMTkyODU
|
||||
iKSAmJiBleGl0IDI1NTskRU5WeyJPTERQV0QifT0iL2hvbWUvdGFuZ2UvcHJpdmF0L3Bh
|
||||
IjskRU5WeyJQQVJBTExFTF9QSUQifT0iMTkyODUyMCI7JEVOVnsiUEFSQUxMRUxfU0VRI
|
||||
0BiYXNoX2Z1bmN0aW9ucz1xdyhteV9mdW5jMyk7IGlmKCRFTlZ7IlNIRUxMIn09fi9jc2
|
||||
ByaW50IFNUREVSUiAiQ1NIL1RDU0ggRE8gTk9UIFNVUFBPUlQgbmV3bGluZXMgSU4gVkF
|
||||
TL0ZVTkNUSU9OUy4gVW5zZXQgQGJhc2hfZnVuY3Rpb25zXG4iOyBleGVjICJmYWxzZSI7
|
||||
YXNoZnVuYyA9ICJteV9mdW5jMygpIHsgIGVjaG8gaW4gbXlfZnVuYyBcJDEgPiBcJDEub
|
||||
Xhwb3J0IC1mIG15X2Z1bmMzID4vZGV2L251bGw7IjtAQVJHVj0ibXlfZnVuYzMgYWJjLW
|
||||
RzaGVsbD0iJEVOVntTSEVMTH0iOyR0bXBkaXI9Ii90bXAiOyRuaWNlPTE3O2RveyRFTlZ
|
||||
MRUxfVE1QfT0kdG1wZGlyLiIvcGFyIi5qb2luIiIsbWFweygwLi45LCJhIi4uInoiLCJB
|
||||
KVtyYW5kKDYyKV19KDEuLjUpO313aGlsZSgtZSRFTlZ7UEFSQUxMRUxfVE1QfSk7JFNJ
|
||||
fT1zdWJ7JGRvbmU9MTt9OyRwaWQ9Zm9yazt1bmxlc3MoJHBpZCl7c2V0cGdycDtldmFse
|
||||
W9yaXR5KDAsMCwkbmljZSl9O2V4ZWMkc2hlbGwsIi1jIiwoJGJhc2hmdW5jLiJAQVJHVi
|
||||
JleGVjOiQhXG4iO31kb3skcz0kczwxPzAuMDAxKyRzKjEuMDM6JHM7c2VsZWN0KHVuZGV
|
||||
mLHVuZGVmLCRzKTt9dW50aWwoJGRvbmV8fGdldHBwaWQ9PTEpO2tpbGwoU0lHSFVQLC0k
|
||||
dW5sZXNzJGRvbmU7d2FpdDtleGl0KCQ/JjEyNz8xMjgrKCQ/JjEyNyk6MSskPz4+OCk=;
|
||||
_EXIT_status=$?; mkdir -p ./.; rsync --protocol 30 --rsync-path=cd\
|
||||
./.parallel/tmp/aspire-1928520-1/./.\;\ rsync -rlDzR -essh
|
||||
server:./abc-file.out ./.;ssh server -- \(rm\ -f\
|
||||
./.parallel/tmp/aspire-1928520-1/abc-file\;\ sh\ -c\ \'rmdir\
|
||||
./.parallel/tmp/aspire-1928520-1/\ ./.parallel/tmp/\ ./.parallel/\
|
||||
2\>/dev/null\'\;rm\ -rf\ ./.parallel/tmp/aspire-1928520-1\;\);ssh
|
||||
server -- \(rm\ -f\ ./.parallel/tmp/aspire-1928520-1/abc-file.out\;\
|
||||
sh\ -c\ \'rmdir\ ./.parallel/tmp/aspire-1928520-1/\ ./.parallel/tmp/\
|
||||
./.parallel/\ 2\>/dev/null\'\;rm\ -rf\
|
||||
./.parallel/tmp/aspire-1928520-1\;\);ssh server -- rm -rf
|
||||
.parallel/tmp/aspire-1928520-1; exit $_EXIT_status;
|
||||
|
||||
=head1 Saving to an SQL base (advanced)
|
||||
|
||||
|
|
@ -1897,6 +1910,19 @@ Output will be similar to:
|
|||
A proper CSV reader (like LibreOffice or R's read.csv) will read this
|
||||
format correctly - even with fields containing newlines as above.
|
||||
|
||||
If the output is big you may want to put it into files using B<--results>:
|
||||
|
||||
parallel --results outdir --sqlandworker csv:////%2Ftmp%2Flog2.csv seq ::: 10 ::: 12 13 14
|
||||
cat /tmp/log2.csv
|
||||
|
||||
Output will be similar to:
|
||||
|
||||
Seq,Host,Starttime,JobRuntime,Send,Receive,Exitval,_Signal,Command,V1,V2,Stdout,Stderr
|
||||
1,:,1458824738.287,0.029,0,9,0,0,"seq 10 12",10,12,outdir/1/10/2/12/stdout,outdir/1/10/2/12/stderr
|
||||
2,:,1458824738.298,0.025,0,12,0,0,"seq 10 13",10,13,outdir/1/10/2/13/stdout,outdir/1/10/2/13/stderr
|
||||
3,:,1458824738.309,0.026,0,15,0,0,"seq 10 14",10,14,outdir/1/10/2/14/stdout,outdir/1/10/2/14/stderr
|
||||
|
||||
|
||||
=head2 DBURL as table
|
||||
|
||||
The CSV file is an example of a DBURL.
|
||||
|
|
@ -1996,7 +2022,7 @@ Output (the order may be different):
|
|||
|
||||
The size of the chunk is not exactly 1 MB because GNU B<parallel> only
|
||||
passes full lines - never half a line, thus the blocksize is only
|
||||
average 1 MB. You can change the block size to 2 MB with B<--block>:
|
||||
1 MB on average. You can change the block size to 2 MB with B<--block>:
|
||||
|
||||
cat num1000000 | parallel --pipe --block 2M wc
|
||||
|
||||
|
|
@ -2007,8 +2033,8 @@ Output (the order may be different):
|
|||
299593 299593 2097151
|
||||
85349 85349 597444
|
||||
|
||||
GNU B<parallel> treats each line as a record. If the order of record is
|
||||
unimportant (e.g. you need all lines processed, but you do not care
|
||||
GNU B<parallel> treats each line as a record. If the order of records
|
||||
is unimportant (e.g. you need all lines processed, but you do not care
|
||||
which is processed first), then you can use B<--round-robin>. Without
|
||||
B<--round-robin> GNU B<parallel> will start a command per block; with
|
||||
B<--round-robin> only the requested number of jobs will be started
|
||||
|
|
@ -2047,8 +2073,8 @@ Output (the order may be different):
|
|||
140000 140000 980000
|
||||
20000 20000 140001
|
||||
|
||||
Notice that the last job could not get the full 140000 lines, but only
|
||||
20000 lines.
|
||||
Note how that the last job could not get the full 140000 lines, but
|
||||
only 20000 lines.
|
||||
|
||||
If a record is 75 lines B<-L> can be used:
|
||||
|
||||
|
|
@ -2065,10 +2091,10 @@ Output (the order may be different):
|
|||
85350 85350 597450
|
||||
25 25 176
|
||||
|
||||
Notice GNU B<parallel> still reads a block of around 1 MB; but instead of
|
||||
passing full lines to B<wc> it passes full 75 lines at a time. This
|
||||
of course does not hold for the last job (which in this case got 25
|
||||
lines).
|
||||
Note how GNU B<parallel> still reads a block of around 1 MB; but
|
||||
instead of passing full lines to B<wc> it passes full 75 lines at a
|
||||
time. This of course does not hold for the last job (which in this
|
||||
case got 25 lines).
|
||||
|
||||
=head2 Record separators
|
||||
|
||||
|
|
@ -2405,7 +2431,7 @@ efficient than its normal mode.
|
|||
|
||||
A counting semaphore is like a row of toilets. People needing a toilet
|
||||
can use any toilet, but if there are more people than toilets, they
|
||||
will have to wait for one of the toilets to be available.
|
||||
will have to wait for one of the toilets to become available.
|
||||
|
||||
An alias for B<parallel --semaphore> is B<sem>.
|
||||
|
||||
|
|
@ -2455,7 +2481,7 @@ The difference between this and just running the command, is that a
|
|||
mutex is set, so if other B<sem>s were running in the background only one
|
||||
would run at a time.
|
||||
|
||||
To tell the difference between which semaphore is used, use
|
||||
To control which semaphore is used, use
|
||||
B<--semaphorename>/B<--id>. Run this in one terminal:
|
||||
|
||||
sem --id my_id -u 'echo First started; sleep 10; echo The first finished'
|
||||
|
|
@ -2525,6 +2551,7 @@ B<--help> will print a summary of the most important options:
|
|||
Output:
|
||||
|
||||
Usage:
|
||||
|
||||
parallel [options] [command [arguments]] < list_of_arguments
|
||||
parallel [options] [command [arguments]] (::: arguments|:::: argfile(s))...
|
||||
cat ... | parallel --pipe [options] [command [arguments]]
|
||||
|
|
@ -2533,8 +2560,10 @@ Output:
|
|||
-k Keep same order
|
||||
-X Multiple arguments with context replace
|
||||
--colsep regexp Split input on regexp for positional replacements
|
||||
{} {.} {/} {/.} {#} Replacement strings
|
||||
{3} {3.} {3/} {3/.} Positional replacement strings
|
||||
{} {.} {/} {/.} {#} {%} {= perl code =} Replacement strings
|
||||
{3} {3.} {3/} {3/.} {=3 perl code =} Positional replacement strings
|
||||
With --plus: {} = {+/}/{/} = {.}.{+.} = {+/}/{/.}.{+.} = {..}.{+..} =
|
||||
{+/}/{/..}.{+..} = {...}.{+...} = {+/}/{/...}.{+...}
|
||||
|
||||
-S sshlogin Example: foo@server.example.com
|
||||
--slf .. Use ~/.parallel/sshloginfile as the list of sshlogins
|
||||
|
|
@ -2548,29 +2577,33 @@ Output:
|
|||
|
||||
See 'man parallel' for details
|
||||
|
||||
When using GNU Parallel for a publication please cite:
|
||||
Academic tradition requires you to cite works you base your article on.
|
||||
When using programs that use GNU Parallel to process data for publication
|
||||
please cite:
|
||||
|
||||
O. Tange (2011): GNU Parallel - The Command-Line Power Tool,
|
||||
;login: The USENIX Magazine, February 2011:42-47.
|
||||
|
||||
When asking for help, always report the full output of this:
|
||||
This helps funding further development; AND IT WON'T COST YOU A CENT.
|
||||
If you pay 10000 EUR you should feel free to use GNU Parallel without citing.
|
||||
|
||||
When asking for help, always report the full output of this:
|
||||
|
||||
parallel --version
|
||||
|
||||
Output:
|
||||
|
||||
GNU parallel 20130822
|
||||
Copyright (C) 2007,2008,2009,2010,2011,2012,2013 Ole Tange and Free Software Foundation, Inc.
|
||||
GNU parallel 20160323
|
||||
Copyright (C) 2007,2008,2009,2010,2011,2012,2013,2014,2015,2016
|
||||
Ole Tange and Free Software Foundation, Inc.
|
||||
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
|
||||
This is free software: you are free to change and redistribute it.
|
||||
GNU parallel comes with no warranty.
|
||||
|
||||
Web site: http://www.gnu.org/software/parallel
|
||||
|
||||
When using GNU Parallel for a publication please cite:
|
||||
|
||||
O. Tange (2011): GNU Parallel - The Command-Line Power Tool,
|
||||
;login: The USENIX Magazine, February 2011:42-47.
|
||||
When using programs that use GNU Parallel to process data for publication
|
||||
please cite as described in 'parallel --bibtex'.
|
||||
|
||||
In scripts B<--minversion> can be used to ensure the user has at least
|
||||
this version:
|
||||
|
|
@ -2579,7 +2612,7 @@ this version:
|
|||
|
||||
Output:
|
||||
|
||||
20130722
|
||||
20160322
|
||||
Your version is at least 20130722.
|
||||
|
||||
If using GNU B<parallel> for research the BibTeX citation can be
|
||||
|
|
@ -2589,6 +2622,10 @@ generated using B<--bibtex>:
|
|||
|
||||
Output:
|
||||
|
||||
Academic tradition requires you to cite works you base your article on.
|
||||
When using programs that use GNU Parallel to process data for publication
|
||||
please cite:
|
||||
|
||||
@article{Tange2011a,
|
||||
title = {GNU Parallel - The Command-Line Power Tool},
|
||||
author = {O. Tange},
|
||||
|
|
@ -2599,9 +2636,18 @@ Output:
|
|||
volume = {36},
|
||||
url = {http://www.gnu.org/s/parallel},
|
||||
year = {2011},
|
||||
pages = {42-47}
|
||||
pages = {42-47},
|
||||
doi = {10.5281/zenodo.16303}
|
||||
}
|
||||
|
||||
(Feel free to use \nocite{Tange2011a})
|
||||
|
||||
This helps funding further development; AND IT WON'T COST YOU A CENT.
|
||||
If you pay 10000 EUR you should feel free to use GNU Parallel without citing.
|
||||
|
||||
If you send a copy of your published article to tange@gnu.org, it will be
|
||||
mentioned in the release notes of next version of GNU Parallel.
|
||||
|
||||
With B<--max-line-length-allowed> GNU B<parallel> will report the maximal
|
||||
size of the command line:
|
||||
|
||||
|
|
@ -2648,9 +2694,9 @@ Profiles can be combined:
|
|||
|
||||
Output:
|
||||
|
||||
\nice -n17 /bin/bash -c echo\ A
|
||||
\nice -n17 /bin/bash -c echo\ B
|
||||
\nice -n17 /bin/bash -c echo\ C
|
||||
echo A
|
||||
echo B
|
||||
echo C
|
||||
|
||||
|
||||
=head1 Spread the word
|
||||
|
|
|
|||
|
|
@ -271,6 +271,11 @@ echo 'bug #47086: [PATCH] Initialize total_completed from joblog'
|
|||
|
||||
echo '**'
|
||||
|
||||
echo 'bug #47290: xargs: Warning: a NUL character occurred in the input'
|
||||
|
||||
perl -e 'print "foo\0not printed"' | parallel echo
|
||||
|
||||
echo '**'
|
||||
|
||||
EOF
|
||||
echo '### 1 .par file from --files expected'
|
||||
|
|
|
|||
|
|
@ -128,16 +128,7 @@ func_echo() {
|
|||
echo ${assocarr[a]}
|
||||
echo Funky-"$funky"-funky
|
||||
}
|
||||
|
||||
env_parallel() {
|
||||
export PARALLEL_ENV="$(echo "shopt -s expand_aliases 2>/dev/null"; alias;typeset -p |
|
||||
grep -vFf <(readonly) |
|
||||
grep -v 'declare .. (GROUPS|FUNCNAME|DIRSTACK|_|PIPESTATUS|USERNAME|BASH_[A-Z_]+) ';
|
||||
typeset -f)";
|
||||
`which parallel` "$@";
|
||||
unset PARALLEL_ENV;
|
||||
}
|
||||
|
||||
. `which env_parallel.bash`
|
||||
env_parallel alias_echo ::: alias_works
|
||||
env_parallel func_echo ::: function_works
|
||||
env_parallel -S lo alias_echo ::: alias_works_over_ssh
|
||||
|
|
@ -165,16 +156,8 @@ func_echo() {
|
|||
echo Funky-"$funky"-funky
|
||||
}
|
||||
|
||||
env_parallel() {
|
||||
export PARALLEL_ENV="$(alias | perl -pe 's/^/alias /';typeset -p |
|
||||
grep -aFvf <(typeset -pr)|egrep -iav 'ZSH_EVAL_CONTEXT|LINENO=| _=|aliases|^typeset [a-z_]+$'|
|
||||
egrep -av '^(typeset IFS=|..$)|cyan';
|
||||
typeset -f)";
|
||||
parallel "$@";
|
||||
unset PARALLEL_ENV;
|
||||
}
|
||||
|
||||
# alias does not work: http://unix.stackexchange.com/questions/223534/defining-an-alias-and-immediately-use-it
|
||||
# alias does not work:
|
||||
# http://unix.stackexchange.com/questions/223534/defining-an-alias-and-immediately-use-it
|
||||
env_parallel alias_echo ::: alias_does_not_work
|
||||
env_parallel func_echo ::: function_works
|
||||
env_parallel -S zsh@lo alias_echo ::: alias_does_not_work_over_ssh
|
||||
|
|
@ -202,11 +185,6 @@ func_echo() {
|
|||
echo Funky-"$funky"-funky
|
||||
}
|
||||
|
||||
env_parallel() {
|
||||
export PARALLEL_ENV="$(alias | perl -pe 's/^/alias /';typeset -p|egrep -v 'typeset( -i)? -r|PIPESTATUS';typeset -f)";
|
||||
`which parallel` "$@";
|
||||
unset PARALLEL_ENV;
|
||||
}
|
||||
env_parallel alias_echo ::: alias_works
|
||||
env_parallel func_echo ::: function_works
|
||||
env_parallel -S ksh@lo alias_echo ::: alias_works_over_ssh
|
||||
|
|
@ -218,9 +196,6 @@ EOS
|
|||
echo
|
||||
echo "### Fish environment"
|
||||
stdout ssh -q fish@lo <<'EOS' | egrep -v 'Welcome to |packages can be updated|security updates'
|
||||
# All variables cannot reliably be exported
|
||||
# perl -e 'print map { "setenv///$_///$ENV{$_}\n"} grep !/^(PWD|SHLVL|PATH)$/, keys %ENV'| sh -c 'parallel --shellquote' | perl -pe 's:///: :g' |fish
|
||||
|
||||
set myvar "myvar works"
|
||||
setenv myenvvar "myenvvar works"
|
||||
|
||||
|
|
@ -238,8 +213,7 @@ function func_echo
|
|||
echo $argv;
|
||||
echo "$myvar"
|
||||
echo "$myenvvar"
|
||||
# Arrays are not exported
|
||||
# echo $myarray[2]
|
||||
echo $myarray[2]
|
||||
# Assoc arrays do not exist in fish
|
||||
# echo ${assocarr[a]}
|
||||
echo
|
||||
|
|
@ -252,12 +226,6 @@ function func_echo
|
|||
echo
|
||||
end
|
||||
|
||||
function env_parallel
|
||||
setenv PARALLEL_ENV (begin; functions -n | perl -pe 's/,/\n/g' | while read d; functions $d; end; perl -e 'print map { "$_///$ENV{$_}\n"} grep !/^(PWD|SHLVL|PATH)$/, keys %ENV'| sh -c 'parallel --shellquote' | perl -pe 's:^([^/]+)///:setenv $1 :'; end |perl -pe 's/\001/\\cb/g;s/\n/\001/')
|
||||
parallel $argv;
|
||||
set -e PARALLEL_ENV
|
||||
end
|
||||
|
||||
env_parallel alias_echo ::: alias_works
|
||||
env_parallel func_echo ::: function_works
|
||||
env_parallel -S fish@lo alias_echo ::: alias_works_over_ssh
|
||||
|
|
@ -322,7 +290,7 @@ alias alias_echo_var 'echo $argv; echo $myvar; echo ${myarray[2]}; echo Funky-"$
|
|||
# s/^/\001alias /;
|
||||
# Quoted: s/\^/\\001alias\ /\;
|
||||
|
||||
alias env_parallel 'setenv PARALLEL_ENV "`alias | perl -pe s/\\047/\\047\\042\\047\\042\\047/g\;s/\^\(\\S+\)\(\\s+\)\\\(\(.\*\)\\\)/\\1\\2\\3/\;s/\^\(\\S+\)\(\\s+\)\(.\*\)/\\1\\2\\047\\3\\047/\;s/\^/\\001alias\ /\;s/\\\!/\\\\\\\!/g;`";parallel \!*; setenv PARALLEL_ENV'
|
||||
#!# alias env_parallel 'setenv PARALLEL_ENV "`alias | perl -pe s/\\047/\\047\\042\\047\\042\\047/g\;s/\^\(\\S+\)\(\\s+\)\\\(\(.\*\)\\\)/\\1\\2\\3/\;s/\^\(\\S+\)\(\\s+\)\(.\*\)/\\1\\2\\047\\3\\047/\;s/\^/\\001alias\ /\;s/\\\!/\\\\\\\!/g;`";parallel \!*; setenv PARALLEL_ENV'
|
||||
|
||||
|
||||
## set tmpfile=`tempfile`
|
||||
|
|
|
|||
|
|
@ -175,7 +175,7 @@ pod2pdf --output-file ./parallel_design.pdf ./parallel_design.pod --title "GNU P
|
|||
Warning: pod2pdf not found. Using old parallel_design.pdf
|
||||
make[0]: Entering directory `/tmp/parallel-00000000/src'
|
||||
/bin/mkdir -p '/usr/local/bin'
|
||||
/usr/bin/install -c parallel sql niceload env_parallel env_parallel.bash env_parallel.zsh env_parallel.fish env_parallel.ksh '/usr/local/bin'
|
||||
/usr/bin/install -c parallel sql niceload env_parallel env_parallel.bash env_parallel.zsh env_parallel.fish env_parallel.ksh env_parallel.pdksh env_parallel.csh '/usr/local/bin'
|
||||
make install-exec-hook
|
||||
make[0]: Entering directory `/tmp/parallel-00000000/src'
|
||||
rm /usr/local/bin/sem || true
|
||||
|
|
|
|||
|
|
@ -588,5 +588,14 @@ parallel: This job failed:
|
|||
echo 3;exit 1
|
||||
echo '**'
|
||||
**
|
||||
echo 'bug #47290: xargs: Warning: a NUL character occurred in the input'
|
||||
bug #47290: xargs: Warning: a NUL character occurred in the input
|
||||
perl -e 'print "foo\0not printed"' | parallel echo
|
||||
foo
|
||||
parallel: Warning: a NUL character occurred in the input.
|
||||
parallel: Warning: It cannot be passed through in the argument list.
|
||||
parallel: Warning: Did you mean to use the --null option?
|
||||
echo '**'
|
||||
**
|
||||
### 1 .par file from --files expected
|
||||
1
|
||||
|
|
|
|||
|
|
@ -143,7 +143,7 @@ Funky-
|
|||
* Documentation: http://www.linuxmint.com
|
||||
|
||||
|
||||
zsh:129: command not found: alias_echo
|
||||
zsh:130: command not found: alias_echo
|
||||
function_works
|
||||
myvar works
|
||||
array_val2
|
||||
|
|
@ -151,7 +151,7 @@ assoc_val_a
|
|||
Funky-
|
||||
|
||||
!"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~€亗儎厗噲墛媽崕彁憭摂晼棙櫄洔潪煚、￥ウЖ┆<D096><E29486><EFBFBD>辈炒刀犯购患骄坷谅媚牌侨墒颂臀闲岩釉罩棕仝圮蒉哙徕沅彐玷殛腱眍镳耱篝貊鼬<E8B28A><E9BCAC><EFBFBD><EFBFBD>-funky
|
||||
zsh:129: command not found: alias_echo
|
||||
zsh:130: command not found: alias_echo
|
||||
function_works_over_ssh
|
||||
myvar works
|
||||
array_val2
|
||||
|
|
@ -191,26 +191,36 @@ Funky-
|
|||
* Documentation: http://www.linuxmint.com
|
||||
|
||||
|
||||
env_parallel: Warning: ASCII value 1 in variables is not supported
|
||||
env_parallel: Warning: ASCII value 1 in variables is not supported
|
||||
3 arg alias_works
|
||||
env_parallel: Warning: ASCII value 1 in variables is not supported
|
||||
env_parallel: Warning: ASCII value 1 in variables is not supported
|
||||
function_works
|
||||
|
||||
myvar works
|
||||
myenvvar works
|
||||
3
|
||||
|
||||
|
||||
|
||||
Funky--funky
|
||||
Funky-
!"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~€亗儎厗噲墛媽崕彁憭摂晼棙櫄洔潪煚、￥ウЖ┆<D096><E29486><EFBFBD>辈炒刀犯购患骄坷谅媚牌侨墒颂臀闲岩釉罩棕仝圮蒉哙徕沅彐玷殛腱眍镳耱篝貊鼬<E8B28A><E9BCAC><EFBFBD><EFBFBD>-funky
|
||||
Funky-
!"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~€亗儎厗噲墛媽崕彁憭摂晼棙櫄洔潪煚、￥ウЖ┆<D096><E29486><EFBFBD>辈炒刀犯购患骄坷谅媚牌侨墒颂臀闲岩釉罩棕仝圮蒉哙徕沅彐玷殛腱眍镳耱篝貊鼬<E8B28A><E9BCAC><EFBFBD><EFBFBD>-funky
|
||||
|
||||
|
||||
|
||||
env_parallel: Warning: ASCII value 1 in variables is not supported
|
||||
env_parallel: Warning: ASCII value 1 in variables is not supported
|
||||
3 arg alias_works_over_ssh
|
||||
env_parallel: Warning: ASCII value 1 in variables is not supported
|
||||
env_parallel: Warning: ASCII value 1 in variables is not supported
|
||||
function_works_over_ssh
|
||||
|
||||
myvar works
|
||||
myenvvar works
|
||||
3
|
||||
|
||||
|
||||
|
||||
Funky--funky
|
||||
Funky-
!"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~€亗儎厗噲墛媽崕彁憭摂晼棙櫄洔潪煚、￥ウЖ┆<D096><E29486><EFBFBD>辈炒刀犯购患骄坷谅媚牌侨墒颂臀闲岩釉罩棕仝圮蒉哙徕沅彐玷殛腱眍镳耱篝貊鼬<E8B28A><E9BCAC><EFBFBD><EFBFBD>-funky
|
||||
Funky-
!"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~€亗儎厗噲墛媽崕彁憭摂晼棙櫄洔潪煚、￥ウЖ┆<D096><E29486><EFBFBD>辈炒刀犯购患骄坷谅媚牌侨墒颂臀闲岩釉罩棕仝圮蒉哙徕沅彐玷殛腱眍镳耱篝貊鼬<E8B28A><E9BCAC><EFBFBD><EFBFBD>-funky
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -704,6 +704,10 @@ bar
|
|||
parallel --nonall --tag -S $SERVER1,$SERVER2 echo foo bar
|
||||
csh@lo foo bar
|
||||
parallel@lo foo bar
|
||||
source `which env_parallel.bash`
|
||||
alias myecho=echo
|
||||
myvar="Joe's var is"
|
||||
env_parallel -S $SERVER1 'myecho $myvar' ::: green
|
||||
MYVAR='foo bar'
|
||||
export MYVAR
|
||||
parallel --env MYVAR -S $SERVER1 echo '$MYVAR' ::: baz
|
||||
|
|
@ -815,9 +819,9 @@ _
|
|||
parallel --env _ -S $SERVER1 'echo $VAR; my_func2' ::: bar
|
||||
|
||||
/bin/bash: my_func2: command not found
|
||||
parallel -vv -S $SERVER1 echo ::: bar
|
||||
ssh -l parallel lo -- exec perl -e \''@GNU_Parallel=("use","IPC::Open3;","use","MIME::Base64");eval"@GNU_Parallel";my$eval=decode_base64(join"",@ARGV);eval$eval;'\' BASE64;
|
||||
bar
|
||||
parallel -vv --pipepart wc :::: num30000
|
||||
<num30000 perl -e 'while(@ARGV) { sysseek(STDIN,shift,0) || die; $left = shift; while($read = sysread(STDIN,$buf, ($left > 131072 ? 131072 : $left))){ $left -= $read; syswrite(STDOUT,$buf); } }' 0 0 0 168894 | (wc)
|
||||
30000 30000 168894
|
||||
my_func3() {
|
||||
echo in my_func $1 > $1.out
|
||||
}
|
||||
|
|
@ -854,6 +858,24 @@ Seq,Host,Starttime,JobRuntime,Send,Receive,Exitval,_Signal,Command,V1,V2,Stdout,
|
|||
13
|
||||
14
|
||||
",
|
||||
parallel --results outdir --sqlandworker csv:////%2Ftmp%2Flog2.csv seq ::: 10 ::: 12 13 14
|
||||
cat /tmp/log2.csv
|
||||
10
|
||||
11
|
||||
12
|
||||
10
|
||||
11
|
||||
12
|
||||
13
|
||||
10
|
||||
11
|
||||
12
|
||||
13
|
||||
14
|
||||
Seq,Host,Starttime,JobRuntime,Send,Receive,Exitval,_Signal,Command,V1,V2,Stdout,Stderr
|
||||
1,:,000000000.000,0.000,0,9,0,0,"seq 10 12",10,12,outdir/1/10/2/12/stdout,outdir/1/10/2/12/stderr
|
||||
2,:,000000000.000,0.000,0,12,0,0,"seq 10 13",10,13,outdir/1/10/2/13/stdout,outdir/1/10/2/13/stderr
|
||||
3,:,000000000.000,0.000,0,15,0,0,"seq 10 14",10,14,outdir/1/10/2/14/stdout,outdir/1/10/2/14/stderr
|
||||
vendor://[[user][:password]@][host][:port]/[database[/table]
|
||||
/bin/bash: vendor://[[user][:password]@][host][:port]/[database[/table]: No such file or directory
|
||||
sqlite3:///%2Ftmp%2Fmydatabase/mytable
|
||||
|
|
@ -1205,6 +1227,18 @@ please cite:
|
|||
This helps funding further development; AND IT WON'T COST YOU A CENT.
|
||||
If you pay 10000 EUR you should feel free to use GNU Parallel without citing.
|
||||
|
||||
parallel --version
|
||||
GNU parallel VERSION
|
||||
Copyright (C) 2007,2008,2009,2010,2011,2012,2013,2014,2015,2016
|
||||
Ole Tange and Free Software Foundation, Inc.
|
||||
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
|
||||
This is free software: you are free to change and redistribute it.
|
||||
GNU parallel comes with no warranty.
|
||||
|
||||
Web site: http://www.gnu.org/software/parallel
|
||||
|
||||
When using programs that use GNU Parallel to process data for publication
|
||||
please cite as described in 'parallel --bibtex'.
|
||||
parallel --minversion VERSION && echo Your version is at least VERSION.
|
||||
VERSION
|
||||
Your version is at least VERSION.
|
||||
|
|
|
|||
Loading…
Reference in a new issue