mirror of
https://git.savannah.gnu.org/git/parallel.git
synced 2024-10-18 12:59:39 +00:00
Fixed --header bug for multiline headers. +testsuite.
Allow for --timeout 200.5% (float). --workdir . --transfer fixed.
This commit is contained in:
Ole Tange
parent
be6fe3846f
commit
6961852117
6
README
6
README
|
|
@ -30,7 +30,11 @@ The 10 seconds installation will try do to a full installation; if
|
|||
that fails, a personal installation; if that fails, a minimal
|
||||
installation.
|
||||
|
||||
wget -O - pi.dk/3 | sh
|
||||
wget -O - pi.dk/3 | bash
|
||||
|
||||
or:
|
||||
|
||||
curl pi.dk/3/ | bash
|
||||
|
||||
This will literally install faster than reading the rest of this
|
||||
document.
|
||||
|
|
|
|||
|
|
@ -209,6 +209,9 @@ New in this release:
|
|||
http://www.keylength.com/en/4/ the signing key was changed from
|
||||
1024D/ID:FFFFFFF1 to 9888R/ID:88888888.
|
||||
|
||||
* Job ad asking for GNU Parallel expertise
|
||||
http://seattle.craigslist.org/est/sof/4006079352.html
|
||||
|
||||
* Agalma: an automated phylogenomics workflow
|
||||
http://arxiv.org/pdf/1307.6432
|
||||
|
||||
|
|
|
|||
35
src/parallel
35
src/parallel
|
|
@ -305,12 +305,13 @@ sub spreadstdin {
|
|||
my $buf = "";
|
||||
my $header = "";
|
||||
if($opt::header) {
|
||||
my $non_greedy_regexp = $opt::header;
|
||||
# ? , * , + , {} => ?? , *? , +? , {}?
|
||||
$non_greedy_regexp =~ s/(\?|\*|\+|\})/$1\?/g;
|
||||
if($opt::header eq ":") { $opt::header = "(.*\n)"; }
|
||||
# Number = number of lines
|
||||
$opt::header =~ s/^(\d+)$/"(.*\n)"x$1/e;
|
||||
while(read(STDIN,substr($buf,length $buf,0),$opt::blocksize)) {
|
||||
if($buf=~s/^(.*?$non_greedy_regexp)//) {
|
||||
$header = $1; last;
|
||||
if($buf=~s/^($opt::header)//) {
|
||||
$header = $1;
|
||||
last;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -426,7 +427,6 @@ sub spreadstdin {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
# If there is anything left in the buffer write it
|
||||
substr($buf,0,0) = "";
|
||||
write_record_to_pipe($chunk_number++,\$header,\$buf,$recstart,$recend,length $buf);
|
||||
|
|
@ -482,7 +482,7 @@ sub round_robin_write {
|
|||
if($job->stdin_buffer_length() > 0) {
|
||||
$something_written += $job->non_block_write();
|
||||
} else {
|
||||
$job->set_stdin_buffer($block_ref,$endpos,$recstart,$recend);
|
||||
$job->set_stdin_buffer($header_ref,$block_ref,$endpos,$recstart,$recend);
|
||||
$block_passed = 1;
|
||||
$job->set_virgin(0);
|
||||
$something_written += $job->non_block_write();
|
||||
|
|
@ -626,7 +626,7 @@ sub options_hash {
|
|||
"B=s" => \$opt::retired,
|
||||
"ctrlc|ctrl-c" => \$opt::ctrlc,
|
||||
"noctrlc|no-ctrlc|no-ctrl-c" => \$opt::noctrlc,
|
||||
"workdir|wd=s" => \$opt::workdir,
|
||||
"workdir|work-dir|wd=s" => \$opt::workdir,
|
||||
"W=s" => \$opt::retired,
|
||||
"tmpdir=s" => \$opt::tmpdir,
|
||||
"tempdir=s" => \$opt::tmpdir,
|
||||
|
|
@ -820,8 +820,9 @@ sub parse_options {
|
|||
if(defined $opt::fg) { $Global::semaphore = 1; }
|
||||
if(defined $opt::bg) { $Global::semaphore = 1; }
|
||||
if(defined $opt::wait) { $Global::semaphore = 1; }
|
||||
if(defined $opt::timeout and $opt::timeout !~ /^\d+%?$/) {
|
||||
if(defined $opt::timeout and $opt::timeout !~ /^\d+(\.\d+)?%?$/) {
|
||||
::error("--timeout must be seconds or percentage\n");
|
||||
wait_and_exit(255);
|
||||
}
|
||||
if(defined $opt::minversion) {
|
||||
print $Global::version,"\n";
|
||||
|
|
@ -3739,8 +3740,8 @@ sub write {
|
|||
|
||||
sub set_stdin_buffer {
|
||||
my $self = shift;
|
||||
my ($block_ref,$endpos,$recstart,$recend) = @_;
|
||||
$self->{'stdin_buffer'} = substr($$block_ref,0,$endpos);
|
||||
my ($header_ref,$block_ref,$endpos,$recstart,$recend) = @_;
|
||||
$self->{'stdin_buffer'} = ($self->virgin() ? $$header_ref : "").substr($$block_ref,0,$endpos);
|
||||
if($opt::remove_rec_sep) {
|
||||
remove_rec_sep(\$self->{'stdin_buffer'},$recstart,$recend);
|
||||
}
|
||||
|
|
@ -3786,10 +3787,11 @@ sub non_block_write {
|
|||
} else {
|
||||
# successfully wrote everything
|
||||
my $a="";
|
||||
$self->set_stdin_buffer(\$a,0,"","");
|
||||
$self->set_stdin_buffer(\$a,\$a,"","");
|
||||
$something_written = $rv;
|
||||
}
|
||||
}
|
||||
|
||||
::debug("Non-block: $something_written");
|
||||
return $something_written;
|
||||
}
|
||||
|
|
@ -4231,7 +4233,7 @@ sub workdir {
|
|||
my $home = $ENV{'HOME'};
|
||||
eval 'use Cwd';
|
||||
my $cwd = cwd();
|
||||
$opt::workdir = $cwd;
|
||||
$workdir = $cwd;
|
||||
if($home) {
|
||||
# If homedir exists: remove the homedir from
|
||||
# workdir if cwd starts with homedir
|
||||
|
|
@ -4247,7 +4249,7 @@ sub workdir {
|
|||
my ($parent_dev, $parent_ino) = (stat($parent))[0,1];
|
||||
if($parent_dev == $home_dev and $parent_ino == $home_ino) {
|
||||
# dev and ino is the same: We found the homedir.
|
||||
$opt::workdir = join("/",@dir_parts);
|
||||
$workdir = join("/",@dir_parts);
|
||||
last;
|
||||
}
|
||||
}
|
||||
|
|
@ -5842,12 +5844,11 @@ package TimeoutQueue;
|
|||
sub new {
|
||||
my $class = shift;
|
||||
my $delta_time = shift;
|
||||
my ($pct,$avg_damper);
|
||||
if($delta_time =~ /(\d+)%/) {
|
||||
my ($pct);
|
||||
if($delta_time =~ /(\d+(\.\d+)?)%/) {
|
||||
# Timeout in percent
|
||||
$pct = $1/100;
|
||||
$delta_time = 1_000_000;
|
||||
$avg_damper = (1-0.001)/0.9;
|
||||
}
|
||||
return bless {
|
||||
'queue' => [],
|
||||
|
|
|
|||
|
|
@ -553,15 +553,18 @@ status will be the exit status from the failing job.
|
|||
=back
|
||||
|
||||
|
||||
=item B<--header> I<regexp>
|
||||
=item B<--header> I<regexp> (alpha testing)
|
||||
|
||||
Use upto regexp as header. For normal usage the matched header
|
||||
(typically the first line: B<--header '\n'>) will be split using
|
||||
B<--colsep> (which will default to '\t') and column names can be used
|
||||
as replacement variables: B<{column name}>. For B<--pipe> the matched
|
||||
header will be prepended to each output.
|
||||
Use regexp as header. For normal usage the matched header (typically
|
||||
the first line: B<--header '.*\n'>) will be split using B<--colsep>
|
||||
(which will default to '\t') and column names can be used as
|
||||
replacement variables: B<{column name}>.
|
||||
|
||||
B<--header :> is an alias for B<--header '\n'>.
|
||||
For B<--pipe> the matched header will be prepended to each output.
|
||||
|
||||
B<--header :> is an alias for B<--header '.*\n'>.
|
||||
|
||||
If I<regexp> is a number, it will match that many lines.
|
||||
|
||||
|
||||
=item B<-I> I<replace-str>
|
||||
|
|
|
|||
|
|
@ -584,16 +584,19 @@ status will be the exit status from the failing job.
|
|||
|
||||
@end table
|
||||
|
||||
@item @strong{--header} @emph{regexp}
|
||||
@anchor{@strong{--header} @emph{regexp}}
|
||||
@item @strong{--header} @emph{regexp} (alpha testing)
|
||||
@anchor{@strong{--header} @emph{regexp} (alpha testing)}
|
||||
|
||||
Use upto regexp as header. For normal usage the matched header
|
||||
(typically the first line: @strong{--header '\n'}) will be split using
|
||||
@strong{--colsep} (which will default to '\t') and column names can be used
|
||||
as replacement variables: @strong{@{column name@}}. For @strong{--pipe} the matched
|
||||
header will be prepended to each output.
|
||||
Use regexp as header. For normal usage the matched header (typically
|
||||
the first line: @strong{--header '.*\n'}) will be split using @strong{--colsep}
|
||||
(which will default to '\t') and column names can be used as
|
||||
replacement variables: @strong{@{column name@}}.
|
||||
|
||||
@strong{--header :} is an alias for @strong{--header '\n'}.
|
||||
For @strong{--pipe} the matched header will be prepended to each output.
|
||||
|
||||
@strong{--header :} is an alias for @strong{--header '.*\n'}.
|
||||
|
||||
If @emph{regexp} is a number, it will match that many lines.
|
||||
|
||||
@item @strong{-I} @emph{replace-str}
|
||||
@anchor{@strong{-I} @emph{replace-str}}
|
||||
|
|
|
|||
1881
src/parallel_tutorial.html
Normal file
1881
src/parallel_tutorial.html
Normal file
File diff suppressed because it is too large
Load diff
|
|
@ -33,6 +33,18 @@ stdout /usr/bin/time -f %e parallel --delay 2 true ::: 1 2 3 | perl -ne '$_ >= 4
|
|||
echo '### Exit value should not be affected if an earlier job times out'
|
||||
parallel -j2 --timeout 1 --joblog - -k ::: "sleep 10" "exit 255" | field 7
|
||||
|
||||
echo '### --header regexp'
|
||||
(echo %head1; echo %head2; seq 5) | nice parallel -kj2 --pipe -N2 --header '(%.*\n)*' echo JOB{#}\;cat
|
||||
|
||||
echo '### --header num'
|
||||
(echo %head1; echo %head2; seq 5) | nice parallel -kj2 --pipe -N2 --header 2 echo JOB{#}\;cat
|
||||
|
||||
echo '### --header regexp --round-robin'
|
||||
(echo %head1; echo %head2; seq 5) | nice parallel -kj2 --pipe -N2 --round --header '(%.*\n)*' echo JOB{#}\;cat
|
||||
|
||||
echo '### --header num --round-robin'
|
||||
(echo %head1; echo %head2; seq 5) | nice parallel -kj2 --pipe -N2 --round --header 2 echo JOB{#}\;cat
|
||||
|
||||
EOF
|
||||
|
||||
rm -rf tmp
|
||||
|
|
|
|||
|
|
@ -14,3 +14,7 @@ seq 1 100 | parallel --sshlogin "/tmp/myssh1 $SSHLOGIN1, /tmp/myssh2 $SSHLOGIN2"
|
|||
|
||||
echo '### --filter-hosts - OK, non-such-user, connection refused, wrong host'
|
||||
parallel --nonall --filter-hosts -S localhost,NoUser@localhost,154.54.72.206,"ssh 5.5.5.5" hostname
|
||||
|
||||
echo '### test --workdir . in $HOME'
|
||||
cd && mkdir parallel-test && cd parallel-test &&
|
||||
echo OK > testfile && parallel --workdir . --transfer -S $SSHLOGIN1 cat {} ::: testfile
|
||||
|
|
|
|||
|
|
@ -52,3 +52,57 @@ OK
|
|||
Exitval
|
||||
-1
|
||||
255
|
||||
### --header regexp
|
||||
JOB1
|
||||
%head1
|
||||
%head2
|
||||
1
|
||||
2
|
||||
JOB2
|
||||
%head1
|
||||
%head2
|
||||
3
|
||||
4
|
||||
JOB3
|
||||
%head1
|
||||
%head2
|
||||
5
|
||||
### --header num
|
||||
JOB1
|
||||
%head1
|
||||
%head2
|
||||
1
|
||||
2
|
||||
JOB2
|
||||
%head1
|
||||
%head2
|
||||
3
|
||||
4
|
||||
JOB3
|
||||
%head1
|
||||
%head2
|
||||
5
|
||||
### --header regexp --round-robin
|
||||
JOB1
|
||||
%head1
|
||||
%head2
|
||||
3
|
||||
4
|
||||
JOB2
|
||||
%head1
|
||||
%head2
|
||||
1
|
||||
2
|
||||
5
|
||||
### --header num --round-robin
|
||||
JOB1
|
||||
%head1
|
||||
%head2
|
||||
1
|
||||
2
|
||||
5
|
||||
JOB2
|
||||
%head1
|
||||
%head2
|
||||
3
|
||||
4
|
||||
|
|
|
|||
Loading…
Reference in a new issue