mirror of
https://git.savannah.gnu.org/git/parallel.git
synced 2024-11-22 14:07:55 +00:00
parallel: Kill children if receving TERM or INT.
This commit is contained in:
Ole Tange
parent
98c4da806f
commit
d8ec46d2d8
4
NEWS
4
NEWS
|
|
@ -105,10 +105,6 @@ New in this release:
|
|||
Performance Stream Parallel Processing
|
||||
http://www.cs.wustl.edu/~lip/pubs/pmam15_jbeard.pdf
|
||||
|
||||
* GNU Parallel was cited in: Towards Collaborative Exploration and
|
||||
Analysis of Big Data from Mars: A Noachis Terra Case Study
|
||||
http://link.springer.com/chapter/10.1007/978-3-319-13865-7_25
|
||||
|
||||
* GNU Parallel was cited in: Quantifying properties of hot and dense
|
||||
QCD matter through systematic model-to-data comparison
|
||||
http://arxiv.org/pdf/1502.00339.pdf
|
||||
|
|
|
|||
|
|
@ -209,9 +209,9 @@ cc:Tim Cuthbertson <tim3d.junk@gmail.com>,
|
|||
Ryoichiro Suzuki <ryoichiro.suzuki@gmail.com>,
|
||||
Jesse Alama <jesse.alama@gmail.com>
|
||||
|
||||
Subject: GNU Parallel 20150422 ('') released
|
||||
Subject: GNU Parallel 20150422 ('Germanwings') released
|
||||
|
||||
GNU Parallel 20150422 ('') has been released. It is available for download at: http://ftp.gnu.org/gnu/parallel/
|
||||
GNU Parallel 20150422 ('Germanwings') has been released. It is available for download at: http://ftp.gnu.org/gnu/parallel/
|
||||
|
||||
Haiku of the month:
|
||||
|
||||
|
|
@ -221,6 +221,8 @@ New in this release:
|
|||
|
||||
* GNU Parallel now has a DOI: https://dx.doi.org/10.5281/zenodo.16303
|
||||
|
||||
* GNU Parallel was cited in: Scaling Machine Learning for Target Prediction in Drug Discovery using Apache Spark https://cris.cumulus.vub.ac.be/portal/files/5147244/spark.pdf
|
||||
|
||||
* GNU Parallel was cited in: Bayesian inference of protein structure from chemical shift data https://peerj.com/articles/861/
|
||||
|
||||
* GNU Parallel was cited in: Toward Enhanced Metadata Quality of Large-Scale Digital Libraries: Estimating Volume Time Range https://www.ideals.illinois.edu/bitstream/handle/2142/73656/186_ready.pdf
|
||||
|
|
@ -255,6 +257,8 @@ taxator-tk http://algbio.cs.uni-duesseldorf.de/webapps/wa-download/ (check it)
|
|||
|
||||
* GNU Parallel (Sebuah Uji Coba) http://kaka.prakasa.my.id/2014/09/04/gnu-parallel-sebuah-uji-coba/
|
||||
|
||||
* 你见过的最牛逼的命令行程序是什么？ http://www.zhihu.com/question/29257300
|
||||
|
||||
* Bug fixes and man page updates.
|
||||
|
||||
GNU Parallel - For people who live life in the parallel lane.
|
||||
|
|
|
|||
134
src/parallel
134
src/parallel
|
|
@ -49,29 +49,28 @@ if($Global::max_number_of_args) {
|
|||
|
||||
my @command = @ARGV;
|
||||
|
||||
my @fhlist;
|
||||
my @input_source_fh;
|
||||
if($opt::pipepart) {
|
||||
@fhlist = map { open_or_exit($_) } "/dev/null";
|
||||
@input_source_fh = map { open_or_exit($_) } "/dev/null";
|
||||
} else {
|
||||
@fhlist = map { open_or_exit($_) } @opt::a;
|
||||
if(not @fhlist and not $opt::pipe) {
|
||||
@fhlist = (*STDIN);
|
||||
@input_source_fh = map { open_or_exit($_) } @opt::a;
|
||||
if(not @input_source_fh and not $opt::pipe) {
|
||||
@input_source_fh = (*STDIN);
|
||||
}
|
||||
}
|
||||
|
||||
if($opt::skip_first_line) {
|
||||
# Skip the first line for the first file handle
|
||||
my $fh = $fhlist[0];
|
||||
my $fh = $input_source_fh[0];
|
||||
<$fh>;
|
||||
}
|
||||
if($opt::header and not $opt::pipe) {
|
||||
my $fh = $fhlist[0];
|
||||
# split with colsep or \t
|
||||
# $header force $colsep = \t if undef?
|
||||
my $delimiter = $opt::colsep;
|
||||
$delimiter ||= "\$";
|
||||
$delimiter ||= "\t";
|
||||
my $id = 1;
|
||||
for my $fh (@fhlist) {
|
||||
for my $fh (@input_source_fh) {
|
||||
my $line = <$fh>;
|
||||
chomp($line);
|
||||
::debug("init", "Delimiter: '$delimiter'");
|
||||
|
|
@ -89,7 +88,7 @@ if($opt::header and not $opt::pipe) {
|
|||
}
|
||||
} else {
|
||||
my $id = 1;
|
||||
for my $fh (@fhlist) {
|
||||
for my $fh (@input_source_fh) {
|
||||
$Global::input_source_header{$id} = $id;
|
||||
$id++;
|
||||
}
|
||||
|
|
@ -101,7 +100,7 @@ if($opt::filter_hosts and (@opt::sshlogin or @opt::sshloginfile)) {
|
|||
}
|
||||
|
||||
if($opt::nonall or $opt::onall) {
|
||||
onall(@command);
|
||||
onall(\@input_source_fh,@command);
|
||||
wait_and_exit(min(undef_as_zero($Global::exitstatus),254));
|
||||
}
|
||||
|
||||
|
|
@ -109,7 +108,7 @@ if($opt::nonall or $opt::onall) {
|
|||
# multiple --transfer and --basefile with different /./
|
||||
|
||||
$Global::JobQueue = JobQueue->new(
|
||||
\@command,\@fhlist,$Global::ContextReplace,$number_of_args,\@Global::ret_files);
|
||||
\@command,\@input_source_fh,$Global::ContextReplace,$number_of_args,\@Global::ret_files);
|
||||
|
||||
if($opt::eta or $opt::bar) {
|
||||
# Count the number of jobs before starting any
|
||||
|
|
@ -1052,7 +1051,7 @@ sub parse_options {
|
|||
|
||||
sub init_globals {
|
||||
# Defaults:
|
||||
$Global::version = 20150329;
|
||||
$Global::version = 20150403;
|
||||
$Global::progname = 'parallel';
|
||||
$Global::infinity = 2**31;
|
||||
$Global::debug = 0;
|
||||
|
|
@ -1086,7 +1085,6 @@ sub init_globals {
|
|||
);
|
||||
# Modifiable copy of %Global::replace
|
||||
%Global::rpl = %Global::replace;
|
||||
$Global::parens = "{==}";
|
||||
$/ = "\n";
|
||||
$Global::ignore_empty = 0;
|
||||
$Global::interactive = 0;
|
||||
|
|
@ -1109,6 +1107,24 @@ sub init_globals {
|
|||
|
||||
sub parse_replacement_string_options {
|
||||
# Deal with --rpl
|
||||
# Uses:
|
||||
# %Global::rpl
|
||||
# $Global::parensleft
|
||||
# $Global::parensright
|
||||
# $opt::parens
|
||||
# $Global::parensleft
|
||||
# $Global::parensright
|
||||
# $opt::plus
|
||||
# %Global::plus
|
||||
# $opt::I
|
||||
# $opt::U
|
||||
# $opt::i
|
||||
# $opt::basenamereplace
|
||||
# $opt::dirnamereplace
|
||||
# $opt::seqreplace
|
||||
# $opt::slotreplace
|
||||
# $opt::basenameextensionreplace
|
||||
|
||||
sub rpl {
|
||||
# Modify %Global::rpl
|
||||
# Replace $old with $new
|
||||
|
|
@ -1118,10 +1134,11 @@ sub parse_replacement_string_options {
|
|||
delete $Global::rpl{$old};
|
||||
}
|
||||
}
|
||||
if(defined $opt::parens) { $Global::parens = $opt::parens; }
|
||||
my $parenslen = 0.5*length $Global::parens;
|
||||
$Global::parensleft = substr($Global::parens,0,$parenslen);
|
||||
$Global::parensright = substr($Global::parens,$parenslen);
|
||||
my $parens = "{==}";
|
||||
if(defined $opt::parens) { $parens = $opt::parens; }
|
||||
my $parenslen = 0.5*length $parens;
|
||||
$Global::parensleft = substr($parens,0,$parenslen);
|
||||
$Global::parensright = substr($parens,$parenslen);
|
||||
if(defined $opt::plus) { %Global::rpl = (%Global::plus,%Global::rpl); }
|
||||
if(defined $opt::I) { rpl('{}',$opt::I); }
|
||||
if(defined $opt::U) { rpl('{.}',$opt::U); }
|
||||
|
|
@ -1145,6 +1162,23 @@ sub parse_semaphore {
|
|||
# Semaphore defaults
|
||||
# Must be done before computing number of processes and max_line_length
|
||||
# because when running as a semaphore GNU Parallel does not read args
|
||||
# Uses:
|
||||
# $opt::semaphore
|
||||
# $Global::semaphore
|
||||
# $opt::semaphoretimeout
|
||||
# $Semaphore::timeout
|
||||
# $opt::semaphorename
|
||||
# $Semaphore::name
|
||||
# $opt::fg
|
||||
# $Semaphore::fg
|
||||
# $opt::wait
|
||||
# $Semaphore::wait
|
||||
# $opt::bg
|
||||
# @opt::a
|
||||
# @Global::unget_argv
|
||||
# $Global::default_simultaneous_sshlogins
|
||||
# $opt::jobs
|
||||
# $Global::interactive
|
||||
$Global::semaphore ||= ($0 =~ m:(^|/)sem$:); # called as 'sem'
|
||||
if(defined $opt::semaphore) { $Global::semaphore = 1; }
|
||||
if(defined $opt::semaphoretimeout) { $Global::semaphore = 1; }
|
||||
|
|
@ -1177,18 +1211,6 @@ sub parse_semaphore {
|
|||
}
|
||||
}
|
||||
|
||||
sub env_quote {
|
||||
# Input:
|
||||
# $v = value to quote
|
||||
# Returns:
|
||||
# $v = value quoted as environment variable
|
||||
my $v = $_[0];
|
||||
$v =~ s/([\\])/\\$1/g;
|
||||
$v =~ s/([\[\] \#\'\&\<\>\(\)\;\{\}\t\"\$\`\*\174\!\?\~])/\\$1/g;
|
||||
$v =~ s/\n/"\n"/g;
|
||||
return $v;
|
||||
}
|
||||
|
||||
sub record_env {
|
||||
# Record current %ENV-keys in ~/.parallel/ignored_vars
|
||||
# Returns: N/A
|
||||
|
|
@ -1495,8 +1517,8 @@ sub read_args_from_command_line {
|
|||
|
||||
sub cleanup {
|
||||
# Returns: N/A
|
||||
if(@opt::basefile) { cleanup_basefile(); }
|
||||
unlink keys %Global::unlink;
|
||||
if(@opt::basefile) { cleanup_basefile(); }
|
||||
}
|
||||
|
||||
sub __QUOTING_ARGUMENTS_FOR_SHELL__ {}
|
||||
|
|
@ -2798,14 +2820,14 @@ sub onall {
|
|||
close $fh;
|
||||
return $tmpfile;
|
||||
}
|
||||
my @command = @_;
|
||||
my ($input_source_fh_ref,@command) = @_;
|
||||
if($Global::quoting) {
|
||||
@command = shell_quote_empty(@command);
|
||||
}
|
||||
|
||||
# Copy all @fhlist (-a and :::) into tempfiles
|
||||
# Copy all @input_source_fh (-a and :::) into tempfiles
|
||||
my @argfiles = ();
|
||||
for my $fh (@fhlist) {
|
||||
for my $fh (@$input_source_fh_ref) {
|
||||
my ($outfh, $name) = ::tmpfile(SUFFIX => ".all", UNLINK => 1);
|
||||
print $outfh (<$fh>);
|
||||
close $outfh;
|
||||
|
|
@ -2893,10 +2915,14 @@ sub save_original_signal_handler {
|
|||
# Uses:
|
||||
# %Global::original_sig
|
||||
# Returns: N/A
|
||||
$SIG{INT} = sub { if($opt::tmux) { qx { tmux kill-session -t p$$ }; }
|
||||
unlink keys %Global::unlink; exit -1 };
|
||||
$SIG{TERM} = sub { if($opt::tmux) { qx { tmux kill-session -t p$$ }; }
|
||||
unlink keys %Global::unlink; exit -1 };
|
||||
$SIG{INT} = sub {
|
||||
if($opt::tmux) { qx { tmux kill-session -t p$$ }; }
|
||||
wait_and_exit(255);
|
||||
};
|
||||
$SIG{TERM} = sub {
|
||||
if($opt::tmux) { qx { tmux kill-session -t p$$ }; }
|
||||
wait_and_exit(255);
|
||||
};
|
||||
%Global::original_sig = %SIG;
|
||||
$SIG{TERM} = sub {}; # Dummy until jobs really start
|
||||
$SIG{ALRM} = 'IGNORE';
|
||||
|
|
@ -3009,11 +3035,11 @@ sub wait_and_exit {
|
|||
# If we do not wait, we sometimes get segfault
|
||||
# Returns: N/A
|
||||
my $error = shift;
|
||||
unlink keys %Global::unlink;
|
||||
if($error) {
|
||||
# Kill all without printing
|
||||
for my $job (values %Global::running) {
|
||||
$job->kill("TERM");
|
||||
$job->kill("TERM");
|
||||
$job->kill();
|
||||
}
|
||||
}
|
||||
for (keys %Global::unkilled_children) {
|
||||
|
|
@ -3193,10 +3219,16 @@ sub bibtex {
|
|||
if(open (my $fh, ">", $ENV{'HOME'}."/.parallel/will-cite")) {
|
||||
close $fh;
|
||||
print "\nThank you for your support. It is much appreciated. The citation\n",
|
||||
"notice is now silenced. You may also use '--will-cite'.\n";
|
||||
"notice is now silenced. You may also use '--will-cite'.\n",
|
||||
"If you use '--will-cite' in scripts you are expected to pay\n",
|
||||
"the 10000 EUR, because you are making it harder to see the\n",
|
||||
"citation notice.\n\n";
|
||||
} else {
|
||||
print "\nThank you for your support. It is much appreciated. The citation\n",
|
||||
"cannot permanently be disabled. Use --will-cite instead.\n";
|
||||
"cannot permanently be silenced. Use '--will-cite' instead.\n",
|
||||
"If you use '--will-cite' in scripts you are expected to pay\n",
|
||||
"the 10000 EUR, because you are making it harder to see the\n",
|
||||
"citation notice.\n\n";
|
||||
last;
|
||||
}
|
||||
}
|
||||
|
|
@ -5637,12 +5669,13 @@ sub empty_input_detector {
|
|||
# If some input: Pass input as input to pipe
|
||||
# This avoids starting the $read command if there is no input.
|
||||
# Returns:
|
||||
# $cmd = script to prepend to '($real command)'
|
||||
# $cmd = script to prepend to '| ($real command)'
|
||||
|
||||
# The $tmpfile might exist if run on a remote system - we accept that risk
|
||||
my ($dummy_fh, $tmpfile) = ::tmpfile(SUFFIX => ".chr");
|
||||
# Unlink to avoid leaving files if --dry-run or --sshlogin
|
||||
unlink $tmpfile;
|
||||
$Global::unlink{$tmpfile} = 1;
|
||||
my $cmd =
|
||||
# Exit value:
|
||||
# empty input = true
|
||||
|
|
@ -5650,7 +5683,7 @@ sub empty_input_detector {
|
|||
# sh -c needed as csh cannot hide stderr
|
||||
qq{ sh -c 'dd bs=1 count=1 of=$tmpfile 2>/dev/null'; }.
|
||||
qq{ test \! -s "$tmpfile" && rm -f "$tmpfile" && exec true; }.
|
||||
qq{ (cat $tmpfile; rm $tmpfile; cat - ) | };
|
||||
qq{ (cat $tmpfile; rm $tmpfile; cat - ) };
|
||||
return $cmd;
|
||||
}
|
||||
|
||||
|
|
@ -5663,7 +5696,7 @@ sub filter_through_compress {
|
|||
|
||||
for my $fdno (1,2) {
|
||||
my $wpid = open(my $fdw,"|-", empty_input_detector().
|
||||
"($opt::compress_program) >>".
|
||||
"| ($opt::compress_program) >>".
|
||||
$self->fh($fdno,'name')) || die $?;
|
||||
$self->set_fh($fdno,'w',$fdw);
|
||||
$self->set_fh($fdno,'wpid',$wpid);
|
||||
|
|
@ -5890,7 +5923,7 @@ sub kill {
|
|||
# Record this jobs as failed
|
||||
$self->set_exitstatus(-1);
|
||||
# Send two TERMs to give time to clean up
|
||||
::debug("run", "Kill seq ", $self->seq(), "\n");
|
||||
::debug("run", "Kill seq ", $self->seq(), " signal '@signals'\n");
|
||||
my @send_signals = @signals || ("TERM", "TERM", "KILL");
|
||||
for my $signal (@send_signals) {
|
||||
my $alive = 0;
|
||||
|
|
@ -5899,6 +5932,7 @@ sub kill {
|
|||
# The job still running
|
||||
kill $signal, $pid;
|
||||
$alive = 1;
|
||||
::debug("run","$pid is alive\n");
|
||||
}
|
||||
}
|
||||
# If a signal was given as input, do not do the sleep below
|
||||
|
|
@ -6119,7 +6153,7 @@ sub wrapped {
|
|||
$command = (shift @Global::cat_partials). " | ($command)";
|
||||
} elsif($opt::pipe) {
|
||||
# Prepend EOF-detector to avoid starting $command if EOF.
|
||||
$command = empty_input_detector(). "($command);";
|
||||
$command = empty_input_detector(). "| ($command);";
|
||||
}
|
||||
if($opt::tmux) {
|
||||
# Wrap command with 'tmux'
|
||||
|
|
@ -6761,7 +6795,7 @@ sub print_dryrun_and_verbose {
|
|||
my $actual_command = shift;
|
||||
# Temporary file name. Used for fifo to communicate exit val
|
||||
my ($fh, $tmpfile) = ::tmpfile(SUFFIX => ".tmx");
|
||||
$Global::unlink{$tmpfile}=1;
|
||||
$Global::unlink{$tmpfile} = 1;
|
||||
close $fh;
|
||||
unlink $tmpfile;
|
||||
my $visual_command = $self->replaced();
|
||||
|
|
@ -8213,11 +8247,11 @@ sub tmux_length {
|
|||
for my $l (1, 2020, 16320, 100000, $len) {
|
||||
my ($fh, $tmpfile) = ::tmpfile(SUFFIX => ".tmb");
|
||||
close $fh;
|
||||
$Global::unlink{$tmpfile} = 1;
|
||||
unlink $tmpfile;
|
||||
my $tmuxcmd = "sh -c '".$ENV{'TMUX'}." -S $tmpfile new-session -d -n echo $l".
|
||||
("x"x$l). " 2>/dev/null' && echo $l";
|
||||
("x"x$l). " 2>/dev/null' && echo $l; rm -f $tmpfile";
|
||||
push @out, qx{ $tmuxcmd };
|
||||
unlink $tmpfile;
|
||||
}
|
||||
::debug("tmux","tmux-length ",@out);
|
||||
chomp @out;
|
||||
|
|
|
|||
|
|
@ -32,7 +32,8 @@ is CentOS 3.9 and Perl 5.8.0.
|
|||
GNU B<parallel> busy waits. This is because the reason why a job is
|
||||
not started may be due to load average, and thus it will not make
|
||||
sense to wait for a job to finish. Instead the load average must be
|
||||
checked again. Load average is not the only reason.
|
||||
checked again. Load average is not the only reason: --timeout has a
|
||||
similar problem.
|
||||
|
||||
To not burn up too up too much CPU GNU B<parallel> sleeps
|
||||
exponentially longer and longer if nothing happens, maxing out at 1
|
||||
|
|
@ -282,17 +283,17 @@ too. Take B<--nice> as an example:
|
|||
will work just fine. But when run remotely, you need to move the nice
|
||||
command so it is being run on the server:
|
||||
|
||||
parallel --S server nice command ...
|
||||
parallel -S server nice command ...
|
||||
|
||||
And this will again work just fine, as long as you are running a
|
||||
single command. When you are running a composed command you need nice
|
||||
to apply to the whole command, and it gets harder still:
|
||||
|
||||
parallel --S server --q nice bash -c 'command1 ...; command2 | command3'
|
||||
parallel -S server -q nice bash -c 'command1 ...; command2 | command3'
|
||||
|
||||
It is not impossible, but by using B<--nice> GNU B<parallel> will do
|
||||
the right thing for you. Similarly when transferring files: It starts
|
||||
to get hard when the file names contain space, :, ‘, *, or other
|
||||
to get hard when the file names contain space, :, `, *, or other
|
||||
special characters.
|
||||
|
||||
To run the commands in a B<tmux> session you basically just need to
|
||||
|
|
@ -324,7 +325,7 @@ post-shellshock versions of B<bash>.
|
|||
|
||||
=head2 Remote Ctrl-C and standard error (stderr)
|
||||
|
||||
If the user presses Ctrl-C the user expect jobs to stop. This works
|
||||
If the user presses Ctrl-C the user expects jobs to stop. This works
|
||||
out of the box if the jobs are run locally. Unfortunately it is not so
|
||||
simple if the jobs are run remotely.
|
||||
|
||||
|
|
@ -366,7 +367,7 @@ stderr. The wrapper looks like this:
|
|||
|
||||
=head2 Transferring of variables and functions
|
||||
|
||||
Transferring of variables and functions given by B<-env> is done by
|
||||
Transferring of variables and functions given by B<--env> is done by
|
||||
running a Perl script remotely that calls the actual command. The Perl
|
||||
script sets $ENV{variable} to the correct value before exec'ing the a
|
||||
shell that runs the function definition followed by the actual
|
||||
|
|
@ -378,7 +379,7 @@ B<parallel_bash_environment>. This variable is picked up by GNU
|
|||
B<parallel> and used to create the Perl script mentioned above.
|
||||
|
||||
|
||||
=head2 Base64 encode bzip2
|
||||
=head2 Base64 encoded bzip2
|
||||
|
||||
B<csh> limits words of commands to 1024 chars. This is often too little
|
||||
when GNU B<parallel> encodes environment variables and wraps the
|
||||
|
|
|
|||
|
|
@ -553,27 +553,27 @@
|
|||
|
||||
<p>When predefined replacement strings are not flexible enough a perl expression can be used instead. One example is to remove two extensions: foo.tar.gz -> foo</p>
|
||||
|
||||
<pre><code> parallel echo '{= s:\.[^.]+$::;s:\.[^.]+$::; =}' ::: foo.tar.gz</code></pre>
|
||||
<pre><code> parallel echo '{= s:\.[^.]+$::;s:\.[^.]+$::; =}' ::: foo.tar.gz</code></pre>
|
||||
|
||||
<p>Output:</p>
|
||||
|
||||
<pre><code> foo</code></pre>
|
||||
<pre><code> foo</code></pre>
|
||||
|
||||
<p>If the strings <b>{=</b> and <b>=}</b> cause problems they can be replaced with --parens:</p>
|
||||
|
||||
<pre><code> parallel --parens ,,,, echo ',, s:\.[^.]+$::;s:\.[^.]+$::; ,,' ::: foo.tar.gz</code></pre>
|
||||
<pre><code> parallel --parens ,,,, echo ',, s:\.[^.]+$::;s:\.[^.]+$::; ,,' ::: foo.tar.gz</code></pre>
|
||||
|
||||
<p>Output: Same as above.</p>
|
||||
|
||||
<p>To define a short hand replacement string use <b>--rpl</b>:</p>
|
||||
|
||||
<pre><code> parallel --rpl '.. s:\.[^.]+$::;s:\.[^.]+$::;' echo '..' ::: foo.tar.gz</code></pre>
|
||||
<pre><code> parallel --rpl '.. s:\.[^.]+$::;s:\.[^.]+$::;' echo '..' ::: foo.tar.gz</code></pre>
|
||||
|
||||
<p>Output: Same as above.</p>
|
||||
|
||||
<p>If the short hand starts with '{' it can be used as a positional replacement string, too:</p>
|
||||
|
||||
<pre><code> parallel --rpl '{..} s:\.[^.]+$::;s:\.[^.]+$::;' echo '{..}' ::: foo.tar.gz</code></pre>
|
||||
<pre><code> parallel --rpl '{..} s:\.[^.]+$::;s:\.[^.]+$::;' echo '{..}' ::: foo.tar.gz</code></pre>
|
||||
|
||||
<p>Output: Same as above.</p>
|
||||
|
||||
|
|
@ -628,15 +628,15 @@
|
|||
|
||||
<p>To use a perl expression as a positional replacement string simply prepend the perl expression with number and space:</p>
|
||||
|
||||
<pre><code> parallel echo '{=2 s:\.[^.]+$::;s:\.[^.]+$::; =} {1}' ::: bar ::: foo.tar.gz</code></pre>
|
||||
<pre><code> parallel echo '{=2 s:\.[^.]+$::;s:\.[^.]+$::; =} {1}' ::: bar ::: foo.tar.gz</code></pre>
|
||||
|
||||
<p>Output:</p>
|
||||
|
||||
<pre><code> foo bar</code></pre>
|
||||
<pre><code> foo bar</code></pre>
|
||||
|
||||
<p>If a defined short hand starts with '{' it can be used as a positional replacement string, too:</p>
|
||||
|
||||
<pre><code> parallel --rpl '{..} s:\.[^.]+$::;s:\.[^.]+$::;' echo '{2..} {1}' ::: bar ::: foo.tar.gz</code></pre>
|
||||
<pre><code> parallel --rpl '{..} s:\.[^.]+$::;s:\.[^.]+$::;' echo '{2..} {1}' ::: bar ::: foo.tar.gz</code></pre>
|
||||
|
||||
<p>Output: Same as above.</p>
|
||||
|
||||
|
|
@ -918,7 +918,7 @@
|
|||
|
||||
<p>To get the output immediately use --ungroup:</p>
|
||||
|
||||
<pre><code> parallel -j2 --ungroup 'printf "%s-start\n%s" {} {};sleep {};printf "%s\n" -middle;echo {}-end' ::: 4 2 1 </code></pre>
|
||||
<pre><code> parallel -j2 --ungroup 'printf "%s-start\n%s" {} {};sleep {};printf "%s\n" -middle;echo {}-end' ::: 4 2 1</code></pre>
|
||||
|
||||
<p>Output:</p>
|
||||
|
||||
|
|
@ -1117,14 +1117,14 @@
|
|||
Starting 3
|
||||
Thu Aug 15 16:24:38 CEST 2013</code></pre>
|
||||
|
||||
<p>If jobs taking more than a certain amount of time are known to fail, they can be stopped with --timeout:</p>
|
||||
<p>If jobs taking more than a certain amount of time are known to fail, they can be stopped with --timeout. The accuracy of --timeout is 2 seconds:</p>
|
||||
|
||||
<pre><code> parallel --timeout 2.1 sleep {}\; echo {} ::: 1 2 3 4</code></pre>
|
||||
<pre><code> parallel --timeout 4.1 sleep {}\; echo {} ::: 2 4 6 8</code></pre>
|
||||
|
||||
<p>Output:</p>
|
||||
|
||||
<pre><code> 1
|
||||
2</code></pre>
|
||||
<pre><code> 2
|
||||
4</code></pre>
|
||||
|
||||
<p>GNU Parallel can compute the median runtime for jobs and kill those that take more than 200% of the median runtime:</p>
|
||||
|
||||
|
|
|
|||
|
|
@ -452,28 +452,28 @@ When predefined replacement strings are not flexible enough a perl
|
|||
expression can be used instead. One example is to remove two
|
||||
extensions: foo.tar.gz -> foo
|
||||
|
||||
parallel echo '{= s:\.[^.]+$::;s:\.[^.]+$::; =}' ::: foo.tar.gz
|
||||
parallel echo '{= s:\.[^.]+$::;s:\.[^.]+$::; =}' ::: foo.tar.gz
|
||||
|
||||
Output:
|
||||
|
||||
foo
|
||||
foo
|
||||
|
||||
If the strings B<{=> and B<=}> cause problems they can be replaced with --parens:
|
||||
|
||||
parallel --parens ,,,, echo ',, s:\.[^.]+$::;s:\.[^.]+$::; ,,' ::: foo.tar.gz
|
||||
parallel --parens ,,,, echo ',, s:\.[^.]+$::;s:\.[^.]+$::; ,,' ::: foo.tar.gz
|
||||
|
||||
Output: Same as above.
|
||||
|
||||
To define a short hand replacement string use B<--rpl>:
|
||||
|
||||
parallel --rpl '.. s:\.[^.]+$::;s:\.[^.]+$::;' echo '..' ::: foo.tar.gz
|
||||
parallel --rpl '.. s:\.[^.]+$::;s:\.[^.]+$::;' echo '..' ::: foo.tar.gz
|
||||
|
||||
Output: Same as above.
|
||||
|
||||
If the short hand starts with '{' it can be used as a positional
|
||||
replacement string, too:
|
||||
|
||||
parallel --rpl '{..} s:\.[^.]+$::;s:\.[^.]+$::;' echo '{..}' ::: foo.tar.gz
|
||||
parallel --rpl '{..} s:\.[^.]+$::;s:\.[^.]+$::;' echo '{..}' ::: foo.tar.gz
|
||||
|
||||
Output: Same as above.
|
||||
|
||||
|
|
@ -532,16 +532,16 @@ Output (the order may be different):
|
|||
To use a perl expression as a positional replacement string simply
|
||||
prepend the perl expression with number and space:
|
||||
|
||||
parallel echo '{=2 s:\.[^.]+$::;s:\.[^.]+$::; =} {1}' ::: bar ::: foo.tar.gz
|
||||
parallel echo '{=2 s:\.[^.]+$::;s:\.[^.]+$::; =} {1}' ::: bar ::: foo.tar.gz
|
||||
|
||||
Output:
|
||||
|
||||
foo bar
|
||||
foo bar
|
||||
|
||||
If a defined short hand starts with '{' it can be used as a positional
|
||||
replacement string, too:
|
||||
|
||||
parallel --rpl '{..} s:\.[^.]+$::;s:\.[^.]+$::;' echo '{2..} {1}' ::: bar ::: foo.tar.gz
|
||||
parallel --rpl '{..} s:\.[^.]+$::;s:\.[^.]+$::;' echo '{2..} {1}' ::: bar ::: foo.tar.gz
|
||||
|
||||
Output: Same as above.
|
||||
|
||||
|
|
@ -838,10 +838,10 @@ Output:
|
|||
|
||||
To get the output immediately use --ungroup:
|
||||
|
||||
parallel -j2 --ungroup 'printf "%s-start\n%s" {} {};sleep {};printf "%s\n" -middle;echo {}-end' ::: 4 2 1
|
||||
parallel -j2 --ungroup 'printf "%s-start\n%s" {} {};sleep {};printf "%s\n" -middle;echo {}-end' ::: 4 2 1
|
||||
|
||||
Output:
|
||||
|
||||
|
||||
4-start
|
||||
42-start
|
||||
2-middle
|
||||
|
|
@ -1052,14 +1052,15 @@ Output:
|
|||
|
||||
|
||||
If jobs taking more than a certain amount of time are known to fail,
|
||||
they can be stopped with --timeout:
|
||||
they can be stopped with --timeout. The accuracy of --timeout is 2
|
||||
seconds:
|
||||
|
||||
parallel --timeout 2.1 sleep {}\; echo {} ::: 1 2 3 4
|
||||
parallel --timeout 4.1 sleep {}\; echo {} ::: 2 4 6 8
|
||||
|
||||
Output:
|
||||
|
||||
1
|
||||
2
|
||||
4
|
||||
|
||||
GNU Parallel can compute the median runtime for jobs and kill those
|
||||
that take more than 200% of the median runtime:
|
||||
|
|
@ -1484,7 +1485,7 @@ arguments. --nonall is a no argument --onall:
|
|||
parallel --nonall -S $SERVER1,$SERVER2 echo foo bar
|
||||
|
||||
Output:
|
||||
|
||||
|
||||
foo bar
|
||||
foo bar
|
||||
|
||||
|
|
|
|||
|
|
@ -2,28 +2,28 @@ testsuite: 3
|
|||
true
|
||||
|
||||
3: ../src/parallel tests-to-run/* wanted-results/* startdb prereqlocal prereqremote
|
||||
TRIES=3 time sh Start.sh - mem || true
|
||||
TRIES=3 time bash Start.sh - mem || true
|
||||
touch ~/.parallel/will-cite
|
||||
make stopvm
|
||||
|
||||
1: ../src/parallel tests-to-run/* wanted-results/* prereqlocal startdb prereqremote
|
||||
time sh Start.sh - mem || true
|
||||
time bash Start.sh - mem || true
|
||||
touch ~/.parallel/will-cite
|
||||
make stopvm
|
||||
|
||||
mem: ../src/parallel tests-to-run/*mem* wanted-results/*mem* prereqlocal
|
||||
time sh Start.sh mem NONE || true
|
||||
time bash Start.sh mem NONE || true
|
||||
touch ~/.parallel/will-cite
|
||||
make stopvm
|
||||
|
||||
testdb: ../src/parallel tests-to-run/*sql* wanted-results/*sql* prereqdb
|
||||
time sh Start.sh sql NONE
|
||||
time bash Start.sh sql NONE
|
||||
|
||||
local: testlocal
|
||||
true
|
||||
|
||||
testlocal: ../src/parallel tests-to-run/*local* wanted-results/*local* prereqlocal installparallel
|
||||
time sh Start.sh local NONE
|
||||
time bash Start.sh local NONE
|
||||
|
||||
prereqlocal: installparallel
|
||||
tcsh -c echo tcsh installed || (echo tcsh is required for testsuite; /bin/false)
|
||||
|
|
@ -40,7 +40,7 @@ prereqlocal: installparallel
|
|||
which timeout || (echo timeout is required for testsuite; /bin/false)
|
||||
|
||||
prereqremote: installparallel startvm
|
||||
parallel --tag ssh parallel@parallel-server{} parallel --minversion 20121021 ::: 1 2 3 || (echo parallel on remote required for testsuite; /bin/true)
|
||||
parallel --timeout 5 --tag ssh parallel@parallel-server{} parallel --minversion 20121021 ::: 1 2 3 || (echo parallel on remote required for testsuite; /bin/true)
|
||||
|
||||
startvm:
|
||||
# Make sure we can reach the virtual machines
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
#!/bin/bash
|
||||
#!/bin/bash -x
|
||||
|
||||
# Argument can be substring of tests (such as 'local')
|
||||
|
||||
|
|
@ -7,26 +7,37 @@ SHFILE=/tmp/unittest-parallel.sh
|
|||
MAX_SEC_PER_TEST=900
|
||||
export TIMEOUT=$MAX_SEC_PER_TEST
|
||||
|
||||
if [ "$TRIES" = "3" ] ; then
|
||||
# Try a failing test thrice
|
||||
echo Retrying 3 times
|
||||
ls -t tests-to-run/*${1}*.sh |
|
||||
grep -v ${2} |
|
||||
perl -pe 's:(.*/(.*)).sh:bash $1.sh > actual-results/$2; diff -Naur wanted-results/$2 actual-results/$2 >/dev/null || bash $1.sh > actual-results/$2; diff -Naur wanted-results/$2 actual-results/$2 >/dev/null || bash $1.sh > actual-results/$2; diff -Naur wanted-results/$2 actual-results/$2 || touch $1.sh: ' \
|
||||
>$SHFILE
|
||||
else
|
||||
# Run a failing test once
|
||||
echo Not retrying
|
||||
ls -t tests-to-run/*${1}*.sh |
|
||||
grep -v ${2} |
|
||||
perl -pe 's:(.*/(.*)).sh:bash $1.sh > actual-results/$2; diff -Naur wanted-results/$2 actual-results/$2 || touch $1.sh:' \
|
||||
>$SHFILE
|
||||
fi
|
||||
run_test() {
|
||||
script="$1"
|
||||
base=`basename "$script" .sh`
|
||||
if [ "$TRIES" = "3" ] ; then
|
||||
# Try 3 times
|
||||
bash $script > actual-results/$base
|
||||
diff -Naur wanted-results/$base actual-results/$base >/dev/null ||
|
||||
bash $script > actual-results/$base
|
||||
diff -Naur wanted-results/$base actual-results/$base >/dev/null ||
|
||||
bash $script > actual-results/$base
|
||||
diff -Naur wanted-results/$base actual-results/$base ||
|
||||
(touch $script && echo touch $script)
|
||||
else
|
||||
# Run only once
|
||||
bash $script > actual-results/$base
|
||||
diff -Naur wanted-results/$base actual-results/$base ||
|
||||
(touch $script && echo touch $script)
|
||||
fi
|
||||
|
||||
# Check if it was cleaned up
|
||||
find /tmp -maxdepth 1 |
|
||||
perl -ne '/\.(tmb|chr|tms|par)$/ and ++$a and print "TMP NOT CLEAN. FOUND: $_".`touch '$script'`;'
|
||||
# May be owned by other users
|
||||
sudo rm -f /tmp/*.{tmb,chr,tms,par}
|
||||
}
|
||||
export -f run_test
|
||||
|
||||
date
|
||||
mkdir -p actual-results
|
||||
stdout sh -x $SHFILE | tee testsuite.log
|
||||
rm $SHFILE
|
||||
ls -t tests-to-run/*${1}*.sh | grep -v ${2} |
|
||||
stdout parallel --tty -tj1 run_test | tee testsuite.log
|
||||
# If testsuite.log contains @@ then there is a diff
|
||||
if grep -q '@@' testsuite.log ; then
|
||||
false
|
||||
|
|
|
|||
|
|
@ -94,3 +94,6 @@ echo '### TMUX not found'
|
|||
TMUX=not-existing parallel --tmux echo ::: 1
|
||||
|
||||
EOF
|
||||
echo '### 1 .par file from --files expected'
|
||||
ls /tmp/par*.par /var/tmp/par*.par /tmp/*.tms /tmp/*.tmx 2>/dev/null | wc -l
|
||||
find /tmp/par*.par /var/tmp/par*.par /tmp/*.tms /tmp/*.tmx -mmin -10 2>/dev/null | parallel rm
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
# Simple jobs that never fails
|
||||
# Each should be taking 3-10s and be possible to run in parallel
|
||||
# I.e.: No race conditions, no logins
|
||||
cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj4 -k --joblog /tmp/jl-`basename $0` -L1
|
||||
cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj0 -k --joblog /tmp/jl-`basename $0` -L1
|
||||
echo '### bug #42089: --results with arg > 256 chars (should be 1 char shorter)'
|
||||
parallel --results parallel_test_dir echo ::: 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456;
|
||||
ls parallel_test_dir/1/
|
||||
|
|
@ -72,4 +72,31 @@ echo '**'
|
|||
echo '### Test slow arguments generation - https://savannah.gnu.org/bugs/?32834';
|
||||
seq 1 3 | parallel -j1 "sleep 2; echo {}" | parallel -kj2 echo
|
||||
|
||||
echo '**'
|
||||
|
||||
echo '### Are children killed if GNU Parallel receives TERM twice? There should be no sleep at the end'
|
||||
|
||||
parallel -q bash -c 'sleep 120 & pid=$!; wait $pid' ::: 1 &
|
||||
T=$!;
|
||||
sleep 1;
|
||||
pstree $$;
|
||||
kill -TERM $T;
|
||||
sleep 1;
|
||||
pstree $$;
|
||||
kill -TERM $T;
|
||||
sleep 1;
|
||||
pstree $$;
|
||||
|
||||
echo '**'
|
||||
|
||||
echo '### Are children killed if GNU Parallel receives INT twice? There should be no sleep at the end'
|
||||
|
||||
parallel -q bash -c 'sleep 120 & pid=$!; wait $pid' ::: 1 &
|
||||
T=$!;
|
||||
sleep 1;
|
||||
pstree $$;
|
||||
kill -INT $T;
|
||||
sleep 1;
|
||||
pstree $$;
|
||||
|
||||
EOF
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
# SSH only allowed to localhost/lo
|
||||
# --retries if ssh dies
|
||||
cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | parallel -vj7 --retries 2 -k --joblog /tmp/jl-`basename $0` -L1
|
||||
cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | parallel -vj1 --retries 2 -k --joblog /tmp/jl-`basename $0` -L1
|
||||
echo '### zsh'
|
||||
ssh zsh@lo 'fun="() { echo function from zsh to zsh \$*; }";
|
||||
export fun;
|
||||
|
|
|
|||
|
|
@ -9,8 +9,8 @@ par_tmux() {
|
|||
(stdout parallel --timeout 3 --tmux --delay .3 echo '{}{=$_="\\"x$_=}'; echo $?) | par_tmux_filter
|
||||
}
|
||||
export -f par_tmux
|
||||
|
||||
cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj8 --retries 2 -k --joblog /tmp/jl-`basename $0` -L1
|
||||
#j8
|
||||
cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj0 --retries 2 -k --joblog /tmp/jl-`basename $0` -L1
|
||||
echo '### tmux1.9'
|
||||
seq 000 100 | TMUX=tmux1.9 par_tmux
|
||||
seq 100 200 | TMUX=tmux1.9 par_tmux
|
||||
|
|
@ -67,3 +67,5 @@ echo '### Test critical lengths. Must not block'
|
|||
seq 280 425 | TMUX=tmux1.9 stdout parallel --tmux echo '{}{=$_="a"x$_=}' | par_tmux_filter
|
||||
|
||||
EOF
|
||||
|
||||
rm -f /tmp/paralocal7*;
|
||||
|
|
@ -1,7 +1,8 @@
|
|||
#!/bin/bash
|
||||
|
||||
cd testsuite 2>/dev/null
|
||||
mkdir -p tmp
|
||||
rm -rf tmp
|
||||
mkdir tmp
|
||||
cd tmp
|
||||
echo '### test parallel_tutorial'
|
||||
rm -f /tmp/runs
|
||||
|
|
@ -14,8 +15,10 @@ perl -ne '$/="\n\n"; /^Output/../^[^O]\S/ and next; /^ / and print;' ../../src/
|
|||
s/zenity/zenity --timeout=12/;
|
||||
s:/usr/bin/time:/usr/bin/time -f %e:;
|
||||
s:ignored_vars:ignored_vars|sort:;
|
||||
# When parallelized: Sleep to make sure the abc-files are made
|
||||
/%head1/ and $_.="sleep .3\n\n"x10;
|
||||
' |
|
||||
stdout bash -x |
|
||||
stdout parallel -j7 -vd'\n\n' |
|
||||
perl -pe '$|=1;
|
||||
# --files and --tmux
|
||||
s:/tmp/par......(...):/tmp/parXXXXX.$1:;
|
||||
|
|
@ -42,4 +45,6 @@ perl -ne '$/="\n\n"; /^Output/../^[^O]\S/ and next; /^ / and print;' ../../src/
|
|||
# + command_X | (Bash outputs these in random order)
|
||||
s/.*command_[ABC].*\n//;
|
||||
'
|
||||
# parallel -d'\n\n'
|
||||
# 3+3 .par files (from --files), 1 .tms-file from tmux attach
|
||||
ls /tmp/par*.par /var/tmp/par*.par /tmp/*.tms /tmp/*.tmx 2>/dev/null | wc -l
|
||||
find /tmp/par*.par /var/tmp/par*.par /tmp/*.tms /tmp/*.tmx -mmin -10 2>/dev/null | parallel rm
|
||||
|
|
|
|||
|
|
@ -161,3 +161,5 @@ echo '### TMUX not found'
|
|||
### TMUX not found
|
||||
TMUX=not-existing parallel --tmux echo ::: 1
|
||||
parallel: Error: not-existing not found in $PATH.
|
||||
### 1 .par file from --files expected
|
||||
1
|
||||
|
|
|
|||
|
|
@ -142,3 +142,23 @@ echo '### Test slow arguments generation - https://savannah.gnu.org/bugs/?32834'
|
|||
1
|
||||
2
|
||||
3
|
||||
echo '**'
|
||||
**
|
||||
echo '### Are children killed if GNU Parallel receives TERM twice? There should be no sleep at the end'
|
||||
### Are children killed if GNU Parallel receives TERM twice? There should be no sleep at the end
|
||||
parallel -q bash -c 'sleep 120 & pid=$!; wait $pid' ::: 1 & T=$!; sleep 1; pstree $$; kill -TERM $T; sleep 1; pstree $$; kill -TERM $T; sleep 1; pstree $$; echo '**'
|
||||
bash-+-perl---bash---sleep
|
||||
`-pstree
|
||||
bash-+-perl---bash---sleep
|
||||
`-pstree
|
||||
bash---pstree
|
||||
**
|
||||
parallel: SIGTERM received. No new jobs will be started.
|
||||
parallel: Waiting for these 1 jobs to finish. Send SIGTERM again to stop now.
|
||||
parallel: bash -c sleep\ 120\ \&\ pid=\$\!\;\ wait\ \$pid 1
|
||||
echo '### Are children killed if GNU Parallel receives INT twice? There should be no sleep at the end'
|
||||
### Are children killed if GNU Parallel receives INT twice? There should be no sleep at the end
|
||||
parallel -q bash -c 'sleep 120 & pid=$!; wait $pid' ::: 1 & T=$!; sleep 1; pstree $$; kill -INT $T; sleep 1; pstree $$;
|
||||
bash-+-perl---bash---sleep
|
||||
`-pstree
|
||||
bash---pstree
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
Loading…
Reference in a new issue