mirror of
https://git.savannah.gnu.org/git/parallel.git
synced 2024-12-22 12:47:54 +00:00
parallel: Preparations for transfering full environment using $parallel_bash_environment.
Better error message if 'file not found' for -a --pipepart. Allow for forcing 0 CPUs using 0/sshlogin. Round fractional wanted CPUs up.
This commit is contained in:
Ole Tange
parent
fcf1e64438
commit
fe1f91776a
2
NEWS
2
NEWS
|
|
@ -7,7 +7,7 @@
|
|||
* Positional perl expressions (similar to {2}) are given as {=2 perl
|
||||
expression=} where 2 is the position.
|
||||
|
||||
* One small backwards incompatability: {1}_{2} will replace {2} with
|
||||
* One small backwards incompatibility: {1}_{2} will replace {2} with
|
||||
the empty string if there is only one argument. Previously {2} would
|
||||
have been left untouched.
|
||||
|
||||
|
|
|
|||
|
|
@ -182,7 +182,7 @@ http://freshmeat.net/projects/parallel/releases/new
|
|||
|
||||
== Update Diaspora Twitter ==
|
||||
|
||||
New release of #GNU Parallel pi․dk/0 New in this release pi․dk/2 See the intro videos pi․dk/1
|
||||
New release of #GNU Parallel pi․dk/0 New in this release pi․dk/2 See the intro videos pi․dk/1
|
||||
10 secs installation: wget -O - pi․dk/3|bash
|
||||
|
||||
[x] Twitter
|
||||
|
|
@ -217,7 +217,7 @@ cc:Tim Cuthbertson <tim3d.junk@gmail.com>,
|
|||
Ryoichiro Suzuki <ryoichiro.suzuki@gmail.com>,
|
||||
Jesse Alama <jesse.alama@gmail.com>
|
||||
|
||||
Subject: GNU Parallel 20140822 ('Argentina/Gaza') released
|
||||
Subject: GNU Parallel 20140822 ('Argentina/Gaza/Williams') released
|
||||
|
||||
GNU Parallel 20140822 ('Argentina/Gaza') has been released. It is available for download at: http://ftp.gnu.org/gnu/parallel/
|
||||
|
||||
|
|
@ -232,12 +232,16 @@ New in this release:
|
|||
|
||||
* GNU Parallel now uses the same shell it was started from as the command shell for local jobs. So if GNU Parallel is started from tcsh it will use tcsh as its shell even if the login $SHELL is different. For remote jobs the login $SHELL will be used.
|
||||
|
||||
* The whole current environment in bash can be copied by using a shell wrapper function (env_parallel).
|
||||
|
||||
* GNU Parallel was cited in: A Web Service for Scholarly Big Data Information Extraction http://patshih.ist.psu.edu/publications/Williams-CiteSeerExtractor-ICWS14.pdf
|
||||
|
||||
* --plus adds the replacement strings {+/} {+.} {+..} {+...} {..} {...} {/..} {/...}. The idea being that '+foo' matches the opposite of 'foo' and {} = {+/}/{/} = {.}.{+.} = {+/}/{/.}.{+.} = {..}.{+..} = {+/}/{/..}.{+..} = {...}.{+...} = {+/}/{/...}.{+...}
|
||||
|
||||
* GNU Parallel was covered in the webcast 2014-08-20: Data Science at the Command Line http://www.oreilly.com/pub/e/3115
|
||||
|
||||
* A Peek into GNU Parallel http://blog.dataweave.in/post/94238943763/a-peek-into-gnu-parallel
|
||||
|
||||
* Сборка GNU parallel для CentOS/RHEL http://www.stableit.ru/2014/07/gnu-parallel-centosrhel.html
|
||||
|
||||
* Bug fixes and man page updates.
|
||||
|
|
|
|||
44
src/parallel
44
src/parallel
|
|
@ -231,7 +231,7 @@ sub find_split_positions {
|
|||
my @pos;
|
||||
my ($recstart,$recend) = recstartrecend();
|
||||
my $recendrecstart = $recend.$recstart;
|
||||
open (my $fh, "<", $file) || die;
|
||||
my $fh = ::open_or_exit($file);
|
||||
push(@pos,$headerlen);
|
||||
for(my $pos = $block+$headerlen; $pos < $size; $pos += $block) {
|
||||
my $buf;
|
||||
|
|
@ -720,7 +720,7 @@ sub get_options_from_array {
|
|||
sub parse_options {
|
||||
# Returns: N/A
|
||||
# Defaults:
|
||||
$Global::version = 20140722;
|
||||
$Global::version = 20140815;
|
||||
$Global::progname = 'parallel';
|
||||
$Global::infinity = 2**31;
|
||||
$Global::debug = 0;
|
||||
|
|
@ -1035,7 +1035,7 @@ sub parse_env_var {
|
|||
# Returns: N/A
|
||||
$Global::envvar = "";
|
||||
$Global::envwarn = "";
|
||||
my @vars = ();
|
||||
my @vars = ('parallel_bash_environment');
|
||||
for my $varstring (@opt::env) {
|
||||
# Split up --env VAR1,VAR2
|
||||
push @vars, split /,/, $varstring;
|
||||
|
|
@ -1057,7 +1057,8 @@ sub parse_env_var {
|
|||
}
|
||||
# Keep only defined variables
|
||||
@vars = grep { defined($ENV{$_}) } @vars;
|
||||
my @qcsh = map { my $a=$_; "setenv $a " . env_quote($ENV{$a}) } @vars;
|
||||
my @qcsh = (map { my $a=$_; "setenv $a " . env_quote($ENV{$a}) }
|
||||
grep { not /^parallel_bash_environment$/ } @vars);
|
||||
my @qbash = map { my $a=$_; "export $a=" . env_quote($ENV{$a}) } @vars;
|
||||
my @bash_functions = grep { substr($ENV{$_},0,4) eq "() {" } @vars;
|
||||
if(@bash_functions) {
|
||||
|
|
@ -1067,13 +1068,14 @@ sub parse_env_var {
|
|||
}
|
||||
}
|
||||
push @qbash, map { my $a=$_; "eval $a\"\$$a\"" } @bash_functions;
|
||||
|
||||
# Check if any variables contain \n
|
||||
if(grep /\n/, @ENV{@vars}) {
|
||||
if(my @v = grep { $ENV{$_}=~/\n/ } @vars) {
|
||||
# \n is bad for csh and will cause it to fail.
|
||||
$Global::envwarn .= ::shell_quote_scalar(q{echo $SHELL | egrep "/t?csh" > /dev/null && echo CSH/TCSH DO NOT SUPPORT newlines IN VARIABLES/FUNCTIONS && exec false;}."\n");
|
||||
$Global::envwarn = ::shell_quote_scalar(q{echo $SHELL | egrep "/t?csh" > /dev/null && echo CSH/TCSH DO NOT SUPPORT newlines IN VARIABLES/FUNCTIONS. Unset }."@v".q{ && exec false;}."\n\n") . $Global::envwarn;
|
||||
}
|
||||
|
||||
if(not @qcsh) { push @qcsh, "true"; }
|
||||
if(not @qbash) { push @qbash, "true"; }
|
||||
# Create lines like:
|
||||
# echo $SHELL | grep "/t\\{0,1\\}csh" >/dev/null && setenv V1 val1 && setenv V2 val2 || export V1=val1 && export V2=val2 ; echo "$V1$V2"
|
||||
if(@vars) {
|
||||
|
|
@ -1084,6 +1086,9 @@ sub parse_env_var {
|
|||
. q{ || }
|
||||
. join(" && ", @qbash)
|
||||
.q{;});
|
||||
if($ENV{'parallel_bash_environment'}) {
|
||||
$Global::envvar .= "parallel_bash_environment;\n";
|
||||
}
|
||||
}
|
||||
$Global::envvarlen = length $Global::envvar;
|
||||
}
|
||||
|
|
@ -1425,6 +1430,7 @@ sub _maybe_unquote {
|
|||
sub _shellwords {
|
||||
# '"'"'\""'"' foo\ bar\" '\" '\ quux => (q("'""), 'foo bar"', '\" quux');
|
||||
my $s = shift;
|
||||
# TODO: fails on " -v"
|
||||
my (@words);
|
||||
while($s =~ s{^
|
||||
(
|
||||
|
|
@ -2042,6 +2048,7 @@ sub get_job_with_sshlogin {
|
|||
::debug("start", "No more jobs: JobQueue empty\n");
|
||||
return undef;
|
||||
}
|
||||
# if(not $sshlogin->start_more()) { return undef; }
|
||||
|
||||
my $clean_command = $job->replaced();
|
||||
if($clean_command =~ /^\s*$/) {
|
||||
|
|
@ -2867,8 +2874,8 @@ sub which {
|
|||
while($testpid) {
|
||||
::debug("init", "shell? ". $name_of_ref->{$testpid}."\n");
|
||||
if($name_of_ref->{$testpid} =~ /$regexp/o) {
|
||||
::debug("init", "which $3$6 => ");
|
||||
$shellpath = (which($3.$6,@{$fakename{$3.$6}}))[0];
|
||||
::debug("init", "which ".($3||$6)." => ");
|
||||
$shellpath = (which($3 || $6,@{$fakename{$3 || $6}}))[0];
|
||||
::debug("init", "shell path $shellpath\n");
|
||||
$shellpath and last;
|
||||
}
|
||||
|
|
@ -3124,8 +3131,8 @@ sub new {
|
|||
my $class = shift;
|
||||
my $sshlogin_string = shift;
|
||||
my $ncpus;
|
||||
if($sshlogin_string =~ s:^(\d+)/:: and $1) {
|
||||
# Override default autodetected ncpus unless zero or missing
|
||||
if($sshlogin_string =~ s:^(\d+)/:: and defined $1) {
|
||||
# Override default autodetected ncpus unless missing
|
||||
$ncpus = $1;
|
||||
}
|
||||
my $string = $sshlogin_string;
|
||||
|
|
@ -3868,9 +3875,7 @@ sub user_requested_processes {
|
|||
::error("Parsing of --jobs/-j/--max-procs/-P failed.\n");
|
||||
::die_usage();
|
||||
}
|
||||
if($processes < 1) {
|
||||
$processes = 1;
|
||||
}
|
||||
$processes = ::ceil($processes);
|
||||
}
|
||||
return $processes;
|
||||
}
|
||||
|
|
@ -3888,10 +3893,12 @@ sub ncpus {
|
|||
}
|
||||
} else {
|
||||
my $ncpu;
|
||||
my $sqe = ::shell_quote_scalar($Global::envvar);
|
||||
if($opt::use_cpus_instead_of_cores) {
|
||||
$ncpu = qx(echo|$sshcmd $serverlogin $Global::envvar parallel --number-of-cpus);
|
||||
$ncpu = qx(echo|$sshcmd $serverlogin $sqe parallel --number-of-cpus);
|
||||
} else {
|
||||
$ncpu = qx(echo|$sshcmd $serverlogin $Global::envvar parallel --number-of-cores);
|
||||
::debug("init",qq(echo|$sshcmd $serverlogin $sqe parallel --number-of-cores\n));
|
||||
$ncpu = qx(echo|$sshcmd $serverlogin $sqe parallel --number-of-cores);
|
||||
}
|
||||
chomp $ncpu;
|
||||
if($ncpu =~ /^\s*[0-9]+\s*$/s) {
|
||||
|
|
@ -5957,7 +5964,8 @@ sub populate {
|
|||
# Returns: N/A
|
||||
my $self = shift;
|
||||
my $next_arg;
|
||||
my $max_len = $Global::minimal_command_line_length || Limits::Command::max_length();
|
||||
# my $max_len = $Global::minimal_command_line_length || Limits::Command::max_length();
|
||||
my $max_len = Limits::Command::max_length();
|
||||
|
||||
if($opt::cat or $opt::fifo) {
|
||||
# Get a tempfile name
|
||||
|
|
@ -5987,7 +5995,7 @@ sub populate {
|
|||
my $args = join(" ", map { $_->orig() } @$next_arg);
|
||||
::error("Command line too long (",
|
||||
$self->len(), " >= ",
|
||||
Limits::Command::max_length(),
|
||||
$max_len,
|
||||
") at number ",
|
||||
$self->{'arg_queue'}->arg_number(),
|
||||
": ".
|
||||
|
|
|
|||
|
|
@ -97,7 +97,7 @@ B<exportf> to export and to set $SHELL to bash:
|
|||
|
||||
The command cannot contain the character \257 (macron: ¯).
|
||||
|
||||
=item B<{}> (alpha testing)
|
||||
=item B<{}> (beta testing)
|
||||
|
||||
Input line. This replacement string will be replaced by a full line
|
||||
read from the input source. The input source is normally stdin
|
||||
|
|
@ -110,7 +110,7 @@ If the command line contains no replacement strings then B<{}> will be
|
|||
appended to the command line.
|
||||
|
||||
|
||||
=item B<{.}> (alpha testing)
|
||||
=item B<{.}> (beta testing)
|
||||
|
||||
Input line without extension. This replacement string will be replaced
|
||||
by the input with the extension removed. If the input line contains
|
||||
|
|
@ -126,7 +126,7 @@ The replacement string B<{.}> can be changed with B<--er>.
|
|||
To understand replacement strings see B<{}>.
|
||||
|
||||
|
||||
=item B<{/}> (alpha testing)
|
||||
=item B<{/}> (beta testing)
|
||||
|
||||
Basename of input line. This replacement string will be replaced by
|
||||
the input with the directory part removed.
|
||||
|
|
@ -137,7 +137,7 @@ B<--basenamereplace>.
|
|||
To understand replacement strings see B<{}>.
|
||||
|
||||
|
||||
=item B<{//}> (alpha testing)
|
||||
=item B<{//}> (beta testing)
|
||||
|
||||
Dirname of input line. This replacement string will be replaced by the
|
||||
dir of the input line. See B<dirname>(1).
|
||||
|
|
@ -148,7 +148,7 @@ B<--dirnamereplace>.
|
|||
To understand replacement strings see B<{}>.
|
||||
|
||||
|
||||
=item B<{/.}> (alpha testing)
|
||||
=item B<{/.}> (beta testing)
|
||||
|
||||
Basename of input line without extension. This replacement string will
|
||||
be replaced by the input with the directory and extension part
|
||||
|
|
@ -160,7 +160,7 @@ B<--basenameextensionreplace>.
|
|||
To understand replacement strings see B<{}>.
|
||||
|
||||
|
||||
=item B<{#}> (alpha testing)
|
||||
=item B<{#}> (beta testing)
|
||||
|
||||
Sequence number of the job to run. This replacement string will be
|
||||
replaced by the sequence number of the job being run. It contains the
|
||||
|
|
@ -171,7 +171,7 @@ The replacement string B<{#}> can be changed with B<--seqreplace>.
|
|||
To understand replacement strings see B<{}>.
|
||||
|
||||
|
||||
=item B<{%}> (alpha testing)
|
||||
=item B<{%}> (beta testing)
|
||||
|
||||
Job slot number. This replacement string will be replaced by the job's
|
||||
slot number between 1 and number of jobs to run in parallel. There
|
||||
|
|
@ -183,7 +183,7 @@ The replacement string B<{%}> can be changed with B<--slotreplace>.
|
|||
To understand replacement strings see B<{}>.
|
||||
|
||||
|
||||
=item B<{>I<n>B<}>
|
||||
=item B<{>I<n>B<}> (beta testing)
|
||||
|
||||
Argument from input source I<n> or the I<n>'th argument. This
|
||||
positional replacement string will be replaced by the input from input
|
||||
|
|
@ -194,7 +194,7 @@ I<n>'th last argument.
|
|||
To understand replacement strings see B<{}>.
|
||||
|
||||
|
||||
=item B<{>I<n>.B<}>
|
||||
=item B<{>I<n>.B<}> (beta testing)
|
||||
|
||||
Argument from input source I<n> or the I<n>'th argument without
|
||||
extension. It is a combination of B<{>I<n>B<}> and B<{.}>.
|
||||
|
|
@ -207,7 +207,7 @@ extension removed.
|
|||
To understand positional replacement strings see B<{>I<n>B<}>.
|
||||
|
||||
|
||||
=item B<{>I<n>/B<}>
|
||||
=item B<{>I<n>/B<}> (beta testing)
|
||||
|
||||
Basename of argument from input source I<n> or the I<n>'th argument.
|
||||
It is a combination of B<{>I<n>B<}> and B<{/}>.
|
||||
|
|
@ -220,7 +220,7 @@ directory (if any) removed.
|
|||
To understand positional replacement strings see B<{>I<n>B<}>.
|
||||
|
||||
|
||||
=item B<{>I<n>//B<}>
|
||||
=item B<{>I<n>//B<}> (beta testing)
|
||||
|
||||
Dirname of argument from input source I<n> or the I<n>'th argument.
|
||||
It is a combination of B<{>I<n>B<}> and B<{//}>.
|
||||
|
|
@ -232,7 +232,7 @@ the I<n>'th argument (when used with B<-N>). See B<dirname>(1).
|
|||
To understand positional replacement strings see B<{>I<n>B<}>.
|
||||
|
||||
|
||||
=item B<{>I<n>/.B<}>
|
||||
=item B<{>I<n>/.B<}> (beta testing)
|
||||
|
||||
Basename of argument from input source I<n> or the I<n>'th argument
|
||||
without extension. It is a combination of B<{>I<n>B<}>, B<{/}>, and
|
||||
|
|
@ -246,7 +246,7 @@ directory (if any) and extension removed.
|
|||
To understand positional replacement strings see B<{>I<n>B<}>.
|
||||
|
||||
|
||||
=item B<{=>I<perl expression>B<=}> (alpha testing)
|
||||
=item B<{=>I<perl expression>B<=}> (beta testing)
|
||||
|
||||
Replace with calculated I<perl expression>. B<$_> will contain the
|
||||
same as B<{}>. After evaluating I<perl expression> B<$_> will be used
|
||||
|
|
@ -259,7 +259,7 @@ The B<{=>I<perl expression>B<=}> must be given as a single string.
|
|||
See also: B<--rpl> B<--parens>
|
||||
|
||||
|
||||
=item B<{=>I<n> I<perl expression>B<=}> (alpha testing)
|
||||
=item B<{=>I<n> I<perl expression>B<=}> (beta testing)
|
||||
|
||||
Positional equivalent to B<{= perl expression =}>. To understand
|
||||
positional replacement strings see B<{>I<n>B<}>.
|
||||
|
|
@ -321,9 +321,9 @@ Use NUL as delimiter. Normally input lines will end in \n
|
|||
for processing arguments that may contain \n (newline).
|
||||
|
||||
|
||||
=item B<--arg-file> I<input-file>
|
||||
=item B<--arg-file> I<input-file> (alpha testing)
|
||||
|
||||
=item B<-a> I<input-file>
|
||||
=item B<-a> I<input-file> (alpha testing)
|
||||
|
||||
Use I<input-file> as input source. If you use this option, stdin
|
||||
(standard input) is given to the first process run. Otherwise, stdin
|
||||
|
|
@ -360,7 +360,7 @@ string that is not in the command line.
|
|||
See also: B<:::>.
|
||||
|
||||
|
||||
=item B<--bar>
|
||||
=item B<--bar> (alpha testing)
|
||||
|
||||
Show progress as a progress bar. In the bar is shown: % of jobs
|
||||
completed, estimated seconds left, and number of jobs started.
|
||||
|
|
@ -413,9 +413,9 @@ Print the BibTeX entry for GNU B<parallel> and disable citation
|
|||
notice.
|
||||
|
||||
|
||||
=item B<--block> I<size>
|
||||
=item B<--block> I<size> (alpha testing)
|
||||
|
||||
=item B<--block-size> I<size>
|
||||
=item B<--block-size> I<size> (alpha testing)
|
||||
|
||||
Size of block in bytes. The size can be postfixed with K, M, G, T, P,
|
||||
k, m, g, t, or p which would multiply the size with 1024, 1048576,
|
||||
|
|
@ -546,9 +546,9 @@ the job may be scheduled on another computer or the local computer if
|
|||
: is in the list.
|
||||
|
||||
|
||||
=item B<--eof>[=I<eof-str>]
|
||||
=item B<--eof>[=I<eof-str>] (alpha testing)
|
||||
|
||||
=item B<-e>[I<eof-str>]
|
||||
=item B<-e>[I<eof-str>] (alpha testing)
|
||||
|
||||
This option is a synonym for the B<-E> option. Use B<-E> instead,
|
||||
because it is POSIX compliant for B<xargs> while this option is not.
|
||||
|
|
@ -591,7 +591,7 @@ cannot be exported:
|
|||
See also: B<--record-env>.
|
||||
|
||||
|
||||
=item B<--eta>
|
||||
=item B<--eta> (alpha testing)
|
||||
|
||||
Show the estimated number of seconds before finishing. This forces GNU
|
||||
B<parallel> to read all jobs before starting to find the number of
|
||||
|
|
@ -661,9 +661,9 @@ See also: B<--line-buffer> B<--ungroup>
|
|||
Print a summary of the options to GNU B<parallel> and exit.
|
||||
|
||||
|
||||
=item B<--halt-on-error> I<val> (alpha testing)
|
||||
=item B<--halt-on-error> I<val> (beta testing)
|
||||
|
||||
=item B<--halt> I<val> (alpha testing)
|
||||
=item B<--halt> I<val> (beta testing)
|
||||
|
||||
How should GNU B<parallel> terminate if one of more jobs fail?
|
||||
|
||||
|
|
@ -776,13 +776,13 @@ If the evaluated number is less than 1 then 1 will be used. See also
|
|||
B<--use-cpus-instead-of-cores>.
|
||||
|
||||
|
||||
=item B<--jobs> I<N>%
|
||||
=item B<--jobs> I<N>% (alpha testing)
|
||||
|
||||
=item B<-j> I<N>%
|
||||
=item B<-j> I<N>% (alpha testing)
|
||||
|
||||
=item B<--max-procs> I<N>%
|
||||
=item B<--max-procs> I<N>% (alpha testing)
|
||||
|
||||
=item B<-P> I<N>%
|
||||
=item B<-P> I<N>% (alpha testing)
|
||||
|
||||
Multiply N% with the number of CPU cores. Run this many jobs in parallel.
|
||||
If the evaluated number is less than 1 then 1 will be used. See also
|
||||
|
|
@ -987,7 +987,7 @@ B<--files> is often used with B<--pipe>.
|
|||
See also: B<--recstart>, B<--recend>, B<--fifo>, B<--cat>, B<--pipepart>.
|
||||
|
||||
|
||||
=item B<--pipepart> (beta testing)
|
||||
=item B<--pipepart> (alpha testing)
|
||||
|
||||
Pipe parts of a physical file. B<--pipepart> works similar to
|
||||
B<--pipe>, but is much faster. It has a few limitations:
|
||||
|
|
@ -1118,7 +1118,7 @@ from the terminal. Only run the command line if the response starts
|
|||
with 'y' or 'Y'. Implies B<-t>.
|
||||
|
||||
|
||||
=item B<--parens> I<parensstring> (alpha testing)
|
||||
=item B<--parens> I<parensstring> (beta testing)
|
||||
|
||||
Use to define start and end parenthesis for B<{= perl expression =}>. The
|
||||
left and the right parenthesis can be multiple characters and are
|
||||
|
|
@ -1133,9 +1133,9 @@ B<,,>:
|
|||
See also: B<--rpl> B<{= perl expression =}>
|
||||
|
||||
|
||||
=item B<--profile> I<profilename>
|
||||
=item B<--profile> I<profilename> (alpha testing)
|
||||
|
||||
=item B<-J> I<profilename>
|
||||
=item B<-J> I<profilename> (alpha testing)
|
||||
|
||||
Use profile I<profilename> for options. This is useful if you want to
|
||||
have multiple profiles. You could have one profile for running jobs in
|
||||
|
|
@ -1356,7 +1356,7 @@ B<--keep-order> will not work with B<--round-robin> as it is
|
|||
impossible to track which input block corresponds to which output.
|
||||
|
||||
|
||||
=item B<--rpl> 'I<tag> I<perl expression>' (alpha testing)
|
||||
=item B<--rpl> 'I<tag> I<perl expression>' (beta testing)
|
||||
|
||||
Use I<tag> as a replacement string for I<perl expression>. This makes
|
||||
it possible to define your own replacement strings. GNU B<parallel>'s
|
||||
|
|
@ -1461,9 +1461,9 @@ Use the replacement string I<replace-str> instead of B<{#}> for
|
|||
job sequence number.
|
||||
|
||||
|
||||
=item B<--shebang> (alpha testing)
|
||||
=item B<--shebang> (beta testing)
|
||||
|
||||
=item B<--hashbang> (alpha testing)
|
||||
=item B<--hashbang> (beta testing)
|
||||
|
||||
GNU B<parallel> can be called as a shebang (#!) command as the first
|
||||
line of a script. The content of the file will be treated as
|
||||
|
|
@ -1488,7 +1488,7 @@ On FreeBSD B<env> is needed:
|
|||
freenetproject.org
|
||||
|
||||
|
||||
=item B<--shebang-wrap> (alpha testing)
|
||||
=item B<--shebang-wrap> (beta testing)
|
||||
|
||||
GNU B<parallel> can parallelize scripts by wrapping the shebang
|
||||
line. If the program can be run like this:
|
||||
|
|
|
|||
|
|
@ -62,6 +62,8 @@ echo '### bug #42893: --block should not cause decimals in cat_partial'
|
|||
echo '### bug #42902: profiles containing arguments with space'
|
||||
echo "--rpl 'FULLPATH chomp(\$_=\"/bin/bash=\".\`readlink -f \$_\`);' " > ~/.parallel/FULLPATH;
|
||||
parallel -JFULLPATH echo FULLPATH ::: $0
|
||||
PARALLEL="--rpl 'FULLPATH chomp(\$_=\"/bin/bash=\".\`readlink -f \$_\`);' -v" parallel echo FULLPATH ::: $0
|
||||
PARALLEL="--rpl 'FULLPATH chomp(\$_=\"/bin/bash=\".\`readlink -f \$_\`);' perl -e \'print \\\"@ARGV\\\n\\\"\' " parallel With script in \\\$PARALLEL FULLPATH ::: .
|
||||
|
||||
echo '### bug #42892: parallel -a nonexiting --pipepart'
|
||||
parallel --pipepart -a nonexisting wc
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
#!/bin/bash
|
||||
|
||||
SERVER1=parallel-server3
|
||||
SERVER2=parallel-server2
|
||||
SERVER1=redhat9.tange.dk
|
||||
SERVER2=centos3.tange.dk
|
||||
|
||||
echo '### Test $PARALLEL'
|
||||
PARALLEL="-k
|
||||
|
|
@ -26,13 +26,13 @@ PARALLEL="-k --jobs 1 echo" parallel ::: a b c
|
|||
PARALLEL="-k --jobs 1 echo 1" parallel -v echo 2 ::: a b c
|
||||
|
||||
echo '### Test ugly quoting from $PARALLEL'
|
||||
PARALLEL="-k --jobs 1 perl -pe '\$a=1; print\$a'" parallel -v ::: <(echo a) <(echo b)
|
||||
PARALLEL='-k --jobs 1 -S parallel@'$SERVER1' perl -pe "\\$a=1; print\\$a"' parallel -v '<(echo {})' ::: foo
|
||||
PARALLEL="-k --jobs 1 perl -pe \'\$a=1; print\$a\'" parallel -v ::: <(echo a) <(echo b)
|
||||
PARALLEL='-k --jobs 1 -S parallel@'$SERVER1' perl -pe \"\\$a=1; print\\$a\"' parallel -v '<(echo {})' ::: foo
|
||||
|
||||
echo '### Test ugly quoting from profile file'
|
||||
cat <<EOF >~/.parallel/test_profile
|
||||
# testprofile
|
||||
-k --jobs 1 perl -pe '\$a=1; print \$a'
|
||||
-k --jobs 1 perl -pe \'\$a=1; print \$a\'
|
||||
EOF
|
||||
parallel -v -J test_profile ::: <(echo a) <(echo b)
|
||||
|
||||
|
|
@ -40,11 +40,11 @@ echo '### Test ugly quoting from profile file --plain'
|
|||
parallel -v -J test_profile --plain echo ::: <(echo a) <(echo b)
|
||||
|
||||
PARALLEL='-k --jobs 1 echo' parallel -S ssh\ parallel\@$SERVER1\ ssh\ parallel@$SERVER2 -v ::: foo
|
||||
PARALLEL='-k --jobs 1 perl -pe "\\$a=1; print \\$a"' parallel -S ssh\ parallel\@$SERVER1\ ssh\ $SERVER2 -vv '<(echo {})' ::: foo
|
||||
PARALLEL='-k --jobs 1 perl -pe \"\\$a=1; print \\$a\"' parallel -S ssh\ parallel\@$SERVER1\ ssh\ $SERVER2 -vv '<(echo {})' ::: foo
|
||||
|
||||
echo '### Test quoting of $ in command from profile file'
|
||||
cat <<EOF >~/.parallel/test_profile
|
||||
-k --jobs 1 perl -pe '\\\$a=1; print \\\$a'
|
||||
-k --jobs 1 perl -pe \'\\\$a=1; print \\\$a\'
|
||||
EOF
|
||||
parallel -v -J test_profile -S ssh\ parallel\@$SERVER1\ ssh\ $SERVER2 '<(echo {})' ::: foo
|
||||
|
||||
|
|
@ -52,14 +52,14 @@ echo '### Test quoting of $ in command from profile file --plain'
|
|||
parallel -v -J test_profile --plain -S ssh\ parallel\@$SERVER1\ ssh\ $SERVER2 'cat <(echo {})' ::: foo
|
||||
|
||||
echo '### Test quoting of $ in command from $PARALLEL'
|
||||
PARALLEL='-k --jobs 1 perl -pe "\\$a=1; print \\$a" ' parallel -S ssh\ parallel\@$SERVER1\ ssh\ $SERVER2 -v '<(echo {})' ::: foo
|
||||
PARALLEL='-k --jobs 1 perl -pe \"\\$a=1; print \\$a\" ' parallel -S ssh\ parallel\@$SERVER1\ ssh\ $SERVER2 -v '<(echo {})' ::: foo
|
||||
|
||||
echo '### Test quoting of $ in command from $PARALLEL --plain'
|
||||
PARALLEL='-k --jobs 1 perl -pe "\\$a=1; print \\$a" ' parallel --plain -S ssh\ parallel\@$SERVER1\ ssh\ $SERVER2 -v 'cat <(echo {})' ::: foo
|
||||
PARALLEL='-k --jobs 1 perl -pe \"\\$a=1; print \\$a\" ' parallel --plain -S ssh\ parallel\@$SERVER1\ ssh\ $SERVER2 -v 'cat <(echo {})' ::: foo
|
||||
|
||||
echo '### Test quoting of space in arguments (-S) from profile file'
|
||||
cat <<EOF >~/.parallel/test_profile
|
||||
-k --jobs 1 -S ssh\ parallel\@$SERVER1\ ssh\ parallel@$SERVER2 perl -pe '\$a=1; print \$a'
|
||||
-k --jobs 1 -S ssh\ parallel\@$SERVER1\ ssh\ parallel@$SERVER2 perl -pe \'\$a=1; print \$a\'
|
||||
EOF
|
||||
parallel -v -J test_profile '<(echo {})' ::: foo
|
||||
|
||||
|
|
@ -67,17 +67,17 @@ echo '### Test quoting of space in arguments (-S) from profile file --plain'
|
|||
parallel -v -J test_profile --plain 'cat <(echo {})' ::: foo
|
||||
|
||||
echo '### Test quoting of space in arguments (-S) from $PARALLEL'
|
||||
PARALLEL='-k --jobs 1 -S ssh\ parallel@'$SERVER1'\ ssh\ parallel@'$SERVER2' perl -pe "\\$a=1; print \\$a" ' parallel -v '<(echo {})' ::: foo
|
||||
PARALLEL='-k --jobs 1 -S ssh\ parallel@'$SERVER1'\ ssh\ parallel@'$SERVER2' perl -pe \"\\$a=1; print \\$a\" ' parallel -v '<(echo {})' ::: foo
|
||||
|
||||
echo '### Test quoting of space in long arguments (--sshlogin) from profile file'
|
||||
cat <<EOF >~/.parallel/test_profile
|
||||
# testprofile
|
||||
-k --jobs 1 --sshlogin ssh\ parallel\@$SERVER1\ ssh\ parallel@$SERVER2 perl -pe '\$a=1; print \$a'
|
||||
-k --jobs 1 --sshlogin ssh\ parallel\@$SERVER1\ ssh\ parallel@$SERVER2 perl -pe \'\$a=1; print \$a\'
|
||||
EOF
|
||||
parallel -v -J test_profile '<(echo {})' ::: foo
|
||||
|
||||
echo '### Test quoting of space in arguments (-S) from $PARALLEL'
|
||||
PARALLEL='-k --jobs 1 --sshlogin ssh\ parallel\@'$SERVER1'\ ssh\ parallel@'$SERVER2' perl -pe "\\$a=1; print \\$a" ' parallel -v '<(echo {})' ::: foo
|
||||
PARALLEL='-k --jobs 1 --sshlogin ssh\ parallel\@'$SERVER1'\ ssh\ parallel@'$SERVER2' perl -pe \"\\$a=1; print \\$a\" ' parallel -v '<(echo {})' ::: foo
|
||||
|
||||
echo '### Test merging of profiles - sort needed because -k only works on the single machine'
|
||||
echo --tag > ~/.parallel/test_tag
|
||||
|
|
|
|||
|
|
@ -23,6 +23,14 @@ Exit code 1
|
|||
echo '### bug #42725: csh with \n in variables'
|
||||
### bug #42725: csh with \n in variables
|
||||
not_csh() { echo This is not csh/tcsh; }; export -f not_csh; parallel --env not_csh -S csh@lo not_csh ::: 1; parallel --env not_csh -S tcsh@lo not_csh ::: 1; parallel --env not_csh -S parallel@lo not_csh ::: 1;
|
||||
CSH/TCSH DO NOT SUPPORT newlines IN VARIABLES/FUNCTIONS
|
||||
CSH/TCSH DO NOT SUPPORT newlines IN VARIABLES/FUNCTIONS
|
||||
CSH/TCSH DO NOT SUPPORT newlines IN VARIABLES/FUNCTIONS. Unset not_csh
|
||||
CSH/TCSH DO NOT SUPPORT newlines IN VARIABLES/FUNCTIONS. Unset not_csh
|
||||
This is not csh/tcsh
|
||||
Unmatched ".
|
||||
\} || export not_csh=\(\)\ \{\ \ echo\ This\ is\ not\ csh/tcsh: Command not found.
|
||||
Unmatched ".
|
||||
parallel: Warning: Could not figure out number of cpus on csh@lo (). Using 1.
|
||||
Unmatched ".
|
||||
\} || export not_csh=\(\)\ \{\ \ echo\ This\ is\ not\ csh/tcsh: Command not found.
|
||||
Unmatched ".
|
||||
parallel: Warning: Could not figure out number of cpus on tcsh@lo (). Using 1.
|
||||
|
|
|
|||
|
|
@ -81,6 +81,11 @@ echo '### bug #42902: profiles containing arguments with space'
|
|||
### bug #42902: profiles containing arguments with space
|
||||
echo "--rpl 'FULLPATH chomp(\$_=\"/bin/bash=\".\`readlink -f \$_\`);' " > ~/.parallel/FULLPATH; parallel -JFULLPATH echo FULLPATH ::: $0
|
||||
/bin/bash=/bin/bash
|
||||
PARALLEL="--rpl 'FULLPATH chomp(\$_=\"/bin/bash=\".\`readlink -f \$_\`);' -v" parallel echo FULLPATH ::: $0
|
||||
echo /bin/bash=/bin/bash
|
||||
/bin/bash=/bin/bash
|
||||
PARALLEL="--rpl 'FULLPATH chomp(\$_=\"/bin/bash=\".\`readlink -f \$_\`);' perl -e \'print \\\"@ARGV\\\n\\\"\' " parallel With script in \\\$PARALLEL FULLPATH ::: .
|
||||
With script in $PARALLEL /bin/bash=/home/tange/privat/parallel/testsuite
|
||||
echo '### bug #42892: parallel -a nonexiting --pipepart'
|
||||
### bug #42892: parallel -a nonexiting --pipepart
|
||||
parallel --pipepart -a nonexisting wc
|
||||
|
|
|
|||
|
|
@ -108,7 +108,6 @@ parallel: Warning: $HOME not set. Using /tmp
|
|||
echo '### How do we deal with missing $SHELL'
|
||||
### How do we deal with missing $SHELL
|
||||
unset SHELL; stdout perl -w $(which parallel) -k echo ::: 1 2 3
|
||||
parallel: Warning: $SHELL not set. Using /bin/sh.
|
||||
1
|
||||
2
|
||||
3
|
||||
|
|
|
|||
|
|
@ -77,35 +77,35 @@ OK
|
|||
Input for ssh
|
||||
parallel@parallel-server1 mkdir -p ./.
|
||||
-l parallel parallel-server1 rsync --server -lDrRze.iLs . ./.
|
||||
-tt -oLogLevel=quiet parallel@parallel-server1 eval `echo $SHELL | grep "/t\{0,1\}csh" > /dev/null && echo setenv PARALLEL_SEQ 2\; setenv PARALLEL_PID 00000 || echo PARALLEL_SEQ=2\;export PARALLEL_SEQ\; PARALLEL_PID=00000\;export PARALLEL_PID` ; tty >/dev/null && stty isig -onlcr -echo;cat tmp/parallel.file.'
|
||||
'newline2 > tmp/parallel.file.'
|
||||
'newline2.out;cat tmp/parallel.file.'
|
||||
'newline2 > tmp/parallel.file.'
|
||||
'newline2.out2
|
||||
-l parallel parallel-server1 cd ././tmp; rsync --server --sender -lDrRze.iLs . ./parallel.file.'
|
||||
'newline2.out
|
||||
-l parallel parallel-server1 cd ././tmp; rsync --server --sender -lDrRze.iLs . ./parallel.file.'
|
||||
'newline2.out2
|
||||
parallel@parallel-server1 (rm -f ./tmp/parallel.file.'
|
||||
'newline2; rmdir ./tmp/ ./ 2>/dev/null;)
|
||||
parallel@parallel-server1 (rm -f ./tmp/parallel.file.'
|
||||
'newline2.out; rmdir ./tmp/ ./ 2>/dev/null;)
|
||||
parallel@parallel-server1 (rm -f ./tmp/parallel.file.'
|
||||
'newline2.out2; rmdir ./tmp/ ./ 2>/dev/null;)
|
||||
parallel@parallel-server2 mkdir -p ./.
|
||||
-l parallel parallel-server2 rsync --server -lDrRze.iLs . ./.
|
||||
-tt -oLogLevel=quiet parallel@parallel-server2 eval `echo $SHELL | grep "/t\{0,1\}csh" > /dev/null && echo setenv PARALLEL_SEQ 1\; setenv PARALLEL_PID 00000 || echo PARALLEL_SEQ=1\;export PARALLEL_SEQ\; PARALLEL_PID=00000\;export PARALLEL_PID` ; tty >/dev/null && stty isig -onlcr -echo;cat tmp/parallel.file.'
|
||||
-tt -oLogLevel=quiet parallel@parallel-server1 eval `echo $SHELL | grep "/t\{0,1\}csh" > /dev/null && echo setenv PARALLEL_SEQ 1\; setenv PARALLEL_PID 00000 || echo PARALLEL_SEQ=1\;export PARALLEL_SEQ\; PARALLEL_PID=00000\;export PARALLEL_PID` ; tty >/dev/null && stty isig -onlcr -echo;cat tmp/parallel.file.'
|
||||
'newline1 > tmp/parallel.file.'
|
||||
'newline1.out;cat tmp/parallel.file.'
|
||||
'newline1 > tmp/parallel.file.'
|
||||
'newline1.out2
|
||||
-l parallel parallel-server2 cd ././tmp; rsync --server --sender -lDrRze.iLs . ./parallel.file.'
|
||||
-l parallel parallel-server1 cd ././tmp; rsync --server --sender -lDrRze.iLs . ./parallel.file.'
|
||||
'newline1.out
|
||||
-l parallel parallel-server2 cd ././tmp; rsync --server --sender -lDrRze.iLs . ./parallel.file.'
|
||||
-l parallel parallel-server1 cd ././tmp; rsync --server --sender -lDrRze.iLs . ./parallel.file.'
|
||||
'newline1.out2
|
||||
parallel@parallel-server2 (rm -f ./tmp/parallel.file.'
|
||||
parallel@parallel-server1 (rm -f ./tmp/parallel.file.'
|
||||
'newline1; rmdir ./tmp/ ./ 2>/dev/null;)
|
||||
parallel@parallel-server2 (rm -f ./tmp/parallel.file.'
|
||||
parallel@parallel-server1 (rm -f ./tmp/parallel.file.'
|
||||
'newline1.out; rmdir ./tmp/ ./ 2>/dev/null;)
|
||||
parallel@parallel-server2 (rm -f ./tmp/parallel.file.'
|
||||
parallel@parallel-server1 (rm -f ./tmp/parallel.file.'
|
||||
'newline1.out2; rmdir ./tmp/ ./ 2>/dev/null;)
|
||||
parallel@parallel-server2 mkdir -p ./.
|
||||
-l parallel parallel-server2 rsync --server -lDrRze.iLs . ./.
|
||||
-tt -oLogLevel=quiet parallel@parallel-server2 eval `echo $SHELL | grep "/t\{0,1\}csh" > /dev/null && echo setenv PARALLEL_SEQ 2\; setenv PARALLEL_PID 00000 || echo PARALLEL_SEQ=2\;export PARALLEL_SEQ\; PARALLEL_PID=00000\;export PARALLEL_PID` ; tty >/dev/null && stty isig -onlcr -echo;cat tmp/parallel.file.'
|
||||
'newline2 > tmp/parallel.file.'
|
||||
'newline2.out;cat tmp/parallel.file.'
|
||||
'newline2 > tmp/parallel.file.'
|
||||
'newline2.out2
|
||||
-l parallel parallel-server2 cd ././tmp; rsync --server --sender -lDrRze.iLs . ./parallel.file.'
|
||||
'newline2.out
|
||||
-l parallel parallel-server2 cd ././tmp; rsync --server --sender -lDrRze.iLs . ./parallel.file.'
|
||||
'newline2.out2
|
||||
parallel@parallel-server2 (rm -f ./tmp/parallel.file.'
|
||||
'newline2; rmdir ./tmp/ ./ 2>/dev/null;)
|
||||
parallel@parallel-server2 (rm -f ./tmp/parallel.file.'
|
||||
'newline2.out; rmdir ./tmp/ ./ 2>/dev/null;)
|
||||
parallel@parallel-server2 (rm -f ./tmp/parallel.file.'
|
||||
'newline2.out2; rmdir ./tmp/ ./ 2>/dev/null;)
|
||||
|
|
|
|||
|
|
@ -42,7 +42,7 @@ echo /dev/fd/62
|
|||
/dev/fd/62
|
||||
echo foo
|
||||
foo
|
||||
ssh parallel@parallel-server3 ssh -tt -oLogLevel=quiet parallel-server2 'eval `echo $SHELL | grep "/t\{0,1\}csh" > /dev/null && echo setenv PARALLEL_SEQ '$PARALLEL_SEQ'\; setenv PARALLEL_PID '$PARALLEL_PID' || echo PARALLEL_SEQ='$PARALLEL_SEQ'\;export PARALLEL_SEQ\; PARALLEL_PID='$PARALLEL_PID'\;export PARALLEL_PID` ;' tty\ \>/dev/null\ \&\&\ stty\ isig\ -onlcr\ -echo\;perl\ -pe\ \"\\\$a=1\;\ print\ \\\$a\"\ \<\(echo\ foo\);
|
||||
ssh parallel@redhat9.tange.dk ssh -tt -oLogLevel=quiet centos3.tange.dk 'eval `echo $SHELL | grep "/t\{0,1\}csh" > /dev/null && echo setenv PARALLEL_SEQ '$PARALLEL_SEQ'\; setenv PARALLEL_PID '$PARALLEL_PID' || echo PARALLEL_SEQ='$PARALLEL_SEQ'\;export PARALLEL_SEQ\; PARALLEL_PID='$PARALLEL_PID'\;export PARALLEL_PID` ;' tty\ \>/dev/null\ \&\&\ stty\ isig\ -onlcr\ -echo\;perl\ -pe\ \"\\\$a=1\;\ print\ \\\$a\"\ \<\(echo\ foo\);
|
||||
1foo
|
||||
### Test quoting of $ in command from profile file
|
||||
perl -pe '$a=1; print $a' <(echo foo)
|
||||
|
|
|
|||
Loading…
Reference in a new issue