mirror of
https://git.savannah.gnu.org/git/parallel.git
synced 2024-11-25 23:47:53 +00:00
Fixed bug #46630: --tf/--transferfile like --basefile but for each job.
Passes testsuite.
This commit is contained in:
Ole Tange
parent
84e1c33182
commit
c1d01e800e
59
src/parallel
59
src/parallel
|
|
@ -78,7 +78,7 @@ if($opt::header and not $opt::pipe) {
|
|||
for my $s (split /$delimiter/o, $line) {
|
||||
::debug("init", "Colname: '$s'");
|
||||
# Replace {colname} with {2}
|
||||
for(@command,@Global::ret_files) {
|
||||
for(@command,@Global::ret_files,@Global::transfer_files) {
|
||||
s:\{$s(|/|//|\.|/\.)\}:\{$id$1\}:g;
|
||||
}
|
||||
$Global::input_source_header{$id} = $s;
|
||||
|
|
@ -107,7 +107,8 @@ if($opt::nonall or $opt::onall) {
|
|||
# multiple --transfer and --basefile with different /./
|
||||
|
||||
$Global::JobQueue = JobQueue->new(
|
||||
\@command,\@input_source_fh,$Global::ContextReplace,$number_of_args,\@Global::ret_files);
|
||||
\@command,\@input_source_fh,$Global::ContextReplace,
|
||||
$number_of_args,\@Global::transfer_files,\@Global::ret_files);
|
||||
|
||||
if($opt::pipepart) {
|
||||
if($opt::roundrobin) {
|
||||
|
|
@ -797,6 +798,8 @@ sub options_hash {
|
|||
"sshloginfile|slf=s" => \@opt::sshloginfile,
|
||||
"controlmaster|M" => \$opt::controlmaster,
|
||||
"ssh=s" => \$opt::ssh,
|
||||
"transfer-file|transferfile|transfer-files|transferfiles|tf=s"
|
||||
=> \@opt::transfer_files,
|
||||
"return=s" => \@opt::return,
|
||||
"trc=s" => \@opt::trc,
|
||||
"transfer" => \$opt::transfer,
|
||||
|
|
@ -976,6 +979,8 @@ sub parse_options {
|
|||
if(@opt::sshlogin) { @Global::sshlogin = @opt::sshlogin; }
|
||||
if(@opt::sshloginfile) { read_sshloginfiles(@opt::sshloginfile); }
|
||||
if(@opt::return) { push @Global::ret_files, @opt::return; }
|
||||
if($opt::transfer) { push @Global::transfer_files, $opt::i || $opt::I || "{}"; }
|
||||
if(@opt::transfer_files) { push @Global::transfer_files, @opt::transfer_files; }
|
||||
if(not defined $opt::recstart and
|
||||
not defined $opt::recend) { $opt::recend = "\n"; }
|
||||
if(not defined $opt::blocksize) { $opt::blocksize = "1M"; }
|
||||
|
|
@ -1028,7 +1033,10 @@ sub parse_options {
|
|||
}
|
||||
if(@opt::trc) {
|
||||
push @Global::ret_files, @opt::trc;
|
||||
$opt::transfer = 1;
|
||||
if(not @Global::transfer_files) {
|
||||
# Defaults to --transferfile {}
|
||||
push @Global::transfer_files, $opt::i || $opt::I || "{}";
|
||||
}
|
||||
$opt::cleanup = 1;
|
||||
}
|
||||
if(defined $opt::max_lines) {
|
||||
|
|
@ -1293,8 +1301,8 @@ sub parse_replacement_string_options {
|
|||
$Global::parensright = substr($parens,$parenslen);
|
||||
if(defined $opt::plus) { %Global::rpl = (%Global::plus,%Global::rpl); }
|
||||
if(defined $opt::I) { rpl('{}',$opt::I); }
|
||||
if(defined $opt::U) { rpl('{.}',$opt::U); }
|
||||
if(defined $opt::i and $opt::i) { rpl('{}',$opt::i); }
|
||||
if(defined $opt::U) { rpl('{.}',$opt::U); }
|
||||
if(defined $opt::basenamereplace) { rpl('{/}',$opt::basenamereplace); }
|
||||
if(defined $opt::dirnamereplace) { rpl('{//}',$opt::dirnamereplace); }
|
||||
if(defined $opt::seqreplace) { rpl('{#}',$opt::seqreplace); }
|
||||
|
|
@ -2788,13 +2796,15 @@ sub parse_sshlogin {
|
|||
}
|
||||
|
||||
# debug("start", "sshlogin: ", my_dump(%Global::host),"\n");
|
||||
if($opt::transfer or @opt::return or $opt::cleanup or @opt::basefile) {
|
||||
if(@Global::transfer_files or @opt::return or $opt::cleanup or @opt::basefile) {
|
||||
if(not remote_hosts()) {
|
||||
# There are no remote hosts
|
||||
if(@opt::trc) {
|
||||
::warning("--trc ignored as there are no remote --sshlogin.");
|
||||
} elsif (defined $opt::transfer) {
|
||||
::warning("--transfer ignored as there are no remote --sshlogin.");
|
||||
} elsif (@opt::transfer_files) {
|
||||
::warning("--transferfile ignored as there are no remote --sshlogin.");
|
||||
} elsif (@opt::return) {
|
||||
::warning("--return ignored as there are no remote --sshlogin.");
|
||||
} elsif (defined $opt::cleanup) {
|
||||
|
|
@ -3650,7 +3660,7 @@ sub tmpfifo {
|
|||
sub qqx {
|
||||
# Like qx but with clean environment (except for $PATH)
|
||||
# and STDERR ignored
|
||||
# This is needed if the environment contains functions
|
||||
# This is needed if the environment contains functions
|
||||
# that /bin/sh does not understand
|
||||
my $devnull = $Global::debug ? "" : "exec 2>/dev/null;";
|
||||
return qx{ $devnull @_ };
|
||||
|
|
@ -5819,10 +5829,11 @@ sub new {
|
|||
my $read_from = shift;
|
||||
my $context_replace = shift;
|
||||
my $max_number_of_args = shift;
|
||||
my $transfer_files = shift;
|
||||
my $return_files = shift;
|
||||
my $commandlinequeue = CommandLineQueue->new
|
||||
($commandref, $read_from, $context_replace, $max_number_of_args,
|
||||
$return_files);
|
||||
$transfer_files, $return_files);
|
||||
my @unget = ();
|
||||
return bless {
|
||||
'unget' => \@unget,
|
||||
|
|
@ -6949,21 +6960,18 @@ sub sshlogin_wrap {
|
|||
|
||||
sub transfer {
|
||||
# Files to transfer
|
||||
# Non-quoted and with {...} substituted
|
||||
# Returns:
|
||||
# @transfer - File names of files to transfer
|
||||
my $self = shift;
|
||||
my @transfer = ();
|
||||
$self->{'transfersize'} = 0;
|
||||
if($opt::transfer) {
|
||||
for my $record (@{$self->{'commandline'}{'arg_list'}}) {
|
||||
# Merge arguments from records into args
|
||||
for my $arg (@$record) {
|
||||
CORE::push @transfer, $arg->orig();
|
||||
# filesize
|
||||
if(-e $arg->orig()) {
|
||||
$self->{'transfersize'} += (stat($arg->orig()))[7];
|
||||
}
|
||||
}
|
||||
|
||||
my @transfer = $self->{'commandline'}->
|
||||
replace_placeholders($self->{'commandline'}{'transfer_files'},0,0);
|
||||
for(@transfer) {
|
||||
# filesize
|
||||
if(-e $_) {
|
||||
$self->{'transfersize'} += (stat($_))[7];
|
||||
}
|
||||
}
|
||||
return @transfer;
|
||||
|
|
@ -6988,7 +6996,7 @@ sub sshtransfer {
|
|||
my $sshlogin = $self->sshlogin();
|
||||
my $workdir = $self->workdir();
|
||||
for my $file ($self->transfer()) {
|
||||
push @pre, $sshlogin->rsync_transfer_cmd($file,$workdir).";";
|
||||
push @pre, $sshlogin->rsync_transfer_cmd($file,$workdir).";";
|
||||
}
|
||||
return join("",@pre);
|
||||
}
|
||||
|
|
@ -7936,6 +7944,7 @@ sub new {
|
|||
my $arg_queue = shift;
|
||||
my $context_replace = shift;
|
||||
my $max_number_of_args = shift; # for -N and normal (-n1)
|
||||
my $transfer_files = shift;
|
||||
my $return_files = shift;
|
||||
my $replacecount_ref = shift;
|
||||
my $len_ref = shift;
|
||||
|
|
@ -7956,6 +7965,7 @@ sub new {
|
|||
'max_number_of_args' => $max_number_of_args,
|
||||
'replacecount' => \%replacecount,
|
||||
'context_replace' => $context_replace,
|
||||
'transfer_files' => $transfer_files,
|
||||
'return_files' => $return_files,
|
||||
'replaced' => undef,
|
||||
}, ref($class) || $class;
|
||||
|
|
@ -8466,6 +8476,7 @@ sub new {
|
|||
my $read_from = shift;
|
||||
my $context_replace = shift || 0;
|
||||
my $max_number_of_args = shift;
|
||||
my $transfer_files = shift;
|
||||
my $return_files = shift;
|
||||
my @unget = ();
|
||||
my ($count,$posrpl,$perlexpr);
|
||||
|
|
@ -8488,7 +8499,7 @@ sub new {
|
|||
# Protect matching inside {= perl expr =}
|
||||
# by replacing {= and =} with \257< and \257>
|
||||
# in @command, --return and --tagstring (if used)
|
||||
for(@command,@$return_files,
|
||||
for(@command,@$transfer_files,@$return_files,
|
||||
(defined $opt::tagstring ? $opt::tagstring : $dummy)) {
|
||||
# Disallow \257 to avoid nested {= {= =} =}
|
||||
if(/\257/) {
|
||||
|
|
@ -8525,7 +8536,7 @@ sub new {
|
|||
|
||||
# Add {} if no replacement strings in @command
|
||||
($replacecount_ref, $len_ref, @command) =
|
||||
replacement_counts_and_lengths($return_files,@command);
|
||||
replacement_counts_and_lengths($transfer_files,$return_files,@command);
|
||||
if("@command" =~ /^[^ \t\n=]*\257</) {
|
||||
# Replacement string is (part of) the command (and not just
|
||||
# argument or variable definition V1={})
|
||||
|
|
@ -8543,6 +8554,7 @@ sub new {
|
|||
'len' => $len_ref,
|
||||
'max_number_of_args' => $max_number_of_args,
|
||||
'size' => undef,
|
||||
'transfer_files' => $transfer_files,
|
||||
'return_files' => $return_files,
|
||||
'seq' => 1,
|
||||
}, ref($class) || $class;
|
||||
|
|
@ -8590,10 +8602,12 @@ sub replacement_counts_and_lengths {
|
|||
# If no {} found in @command: add it to @command
|
||||
#
|
||||
# Input:
|
||||
# \@transfer_files = array of filenames to transfer
|
||||
# \@return_files = array of filenames to return
|
||||
# @command = command template
|
||||
# Output:
|
||||
# \%replacecount, \%len, @command
|
||||
my $transfer_files = shift;
|
||||
my $return_files = shift;
|
||||
my @command = @_;
|
||||
my (%replacecount,%len);
|
||||
|
|
@ -8623,7 +8637,7 @@ sub replacement_counts_and_lengths {
|
|||
# All {= perl expr =} have been removed: The rest is non-context
|
||||
$noncontextlen += length $c;
|
||||
}
|
||||
for(@$return_files) {
|
||||
for(@$transfer_files,@$return_files) {
|
||||
my $t = $_;
|
||||
while($t =~ s/ \257<([^\257]*)\257> //x) {
|
||||
# %replacecount = { "perlexpr" => number of times seen }
|
||||
|
|
@ -8682,6 +8696,7 @@ sub get {
|
|||
$self->{'arg_queue'},
|
||||
$self->{'context_replace'},
|
||||
$self->{'max_number_of_args'},
|
||||
$self->{'transfer_files'},
|
||||
$self->{'return_files'},
|
||||
$self->{'replacecount'},
|
||||
$self->{'len'},
|
||||
|
|
|
|||
|
|
@ -445,7 +445,7 @@ I<file> will be transferred to each sshlogin before a jobs is
|
|||
started. It will be removed if B<--cleanup> is active. The file may be
|
||||
a script to run or some common base data needed for the jobs.
|
||||
Multiple B<--bf> can be specified to transfer more basefiles. The
|
||||
I<file> will be transferred the same way as B<--transfer>.
|
||||
I<file> will be transferred the same way as B<--transferfile>.
|
||||
|
||||
|
||||
=item B<--basenamereplace> I<replace-str>
|
||||
|
|
@ -527,22 +527,23 @@ See also B<--fifo>.
|
|||
|
||||
=item B<--cleanup>
|
||||
|
||||
Remove transferred files. B<--cleanup> will remove the transferred files
|
||||
on the remote computer after processing is done.
|
||||
Remove transferred files. B<--cleanup> will remove the transferred
|
||||
files on the remote computer after processing is done.
|
||||
|
||||
find log -name '*gz' | parallel \
|
||||
--sshlogin server.example.com --transfer \
|
||||
--sshlogin server.example.com --transferfile {} \
|
||||
--return {.}.bz2 --cleanup "zcat {} | bzip -9 >{.}.bz2"
|
||||
|
||||
With B<--transfer> the file transferred to the remote computer will be
|
||||
removed on the remote computer. Directories created will not be removed
|
||||
- even if they are empty.
|
||||
With B<--transferfile {}> the file transferred to the remote computer
|
||||
will be removed on the remote computer. Directories created will not
|
||||
be removed - even if they are empty.
|
||||
|
||||
With B<--return> the file transferred from the remote computer will be
|
||||
removed on the remote computer. Directories created will not be removed
|
||||
- even if they are empty.
|
||||
removed on the remote computer. Directories created will not be
|
||||
removed - even if they are empty.
|
||||
|
||||
B<--cleanup> is ignored when not used with B<--transfer> or B<--return>.
|
||||
B<--cleanup> is ignored when not used with B<--transferfile> or
|
||||
B<--return>.
|
||||
|
||||
|
||||
=item B<--colsep> I<regexp>
|
||||
|
|
@ -1541,7 +1542,7 @@ times:
|
|||
--sshlogin server.example.com \
|
||||
--return {.}.out --return {.}.out2 touch {.}.out {.}.out2
|
||||
|
||||
B<--return> is often used with B<--transfer> and B<--cleanup>.
|
||||
B<--return> is often used with B<--transferfile> and B<--cleanup>.
|
||||
|
||||
B<--return> is ignored when used with B<--sshlogin :> or when not used
|
||||
with B<--sshlogin>.
|
||||
|
|
@ -1824,7 +1825,7 @@ The remote host must have GNU B<parallel> installed.
|
|||
|
||||
B<--sshlogin> is known to cause problems with B<-m> and B<-X>.
|
||||
|
||||
B<--sshlogin> is often used with B<--transfer>, B<--return>,
|
||||
B<--sshlogin> is often used with B<--transferfile>, B<--return>,
|
||||
B<--cleanup>, and B<--trc>.
|
||||
|
||||
|
||||
|
|
@ -1979,38 +1980,46 @@ Print the job to be run on stderr (standard error).
|
|||
See also B<-v>, B<-p>.
|
||||
|
||||
|
||||
=item B<--transfer>
|
||||
=item B<--transfer> (alpha testing)
|
||||
|
||||
Transfer files to remote computers. B<--transfer> is used with
|
||||
B<--sshlogin> when the arguments are files and should be transferred
|
||||
to the remote computers. The files will be transferred using B<rsync>
|
||||
and will be put relative to the default work dir. If the path contains
|
||||
/./ the remaining path will be relative to the work dir. E.g.
|
||||
Transfer files to remote computers. Shorthand for: B<--transferfile {}>.
|
||||
|
||||
|
||||
=item B<--transferfile> I<filename> (alpha testing)
|
||||
|
||||
=item B<--tf> I<filename> (alpha testing)
|
||||
|
||||
B<--transferfile> is used with B<--sshlogin> to transfer files to the
|
||||
remote computers. The files will be transferred using B<rsync> and
|
||||
will be put relative to the default work dir. If the path contains /./
|
||||
the remaining path will be relative to the work dir. E.g.
|
||||
|
||||
echo foo/bar.txt | parallel \
|
||||
--sshlogin server.example.com --transfer wc
|
||||
--sshlogin server.example.com --transferfile {} wc
|
||||
|
||||
This will transfer the file I<foo/bar.txt> to the computer
|
||||
I<server.example.com> to the file I<$HOME/foo/bar.txt> before running
|
||||
B<wc foo/bar.txt> on I<server.example.com>.
|
||||
|
||||
echo /tmp/foo/bar.txt | parallel \
|
||||
--sshlogin server.example.com --transfer wc
|
||||
--sshlogin server.example.com --transferfile {} wc
|
||||
|
||||
This will transfer the file I<foo/bar.txt> to the computer
|
||||
I<server.example.com> to the file I</tmp/foo/bar.txt> before running
|
||||
B<wc /tmp/foo/bar.txt> on I<server.example.com>.
|
||||
|
||||
B<--transfer> is often used with B<--return> and B<--cleanup>.
|
||||
B<--transferfile> is often used with B<--return> and B<--cleanup>. A
|
||||
shorthand for B<--transferfile {}> is B<--transfer>.
|
||||
|
||||
B<--transfer> is ignored when used with B<--sshlogin :> or when not used with B<--sshlogin>.
|
||||
B<--transferfile> is ignored when used with B<--sshlogin :> or when
|
||||
not used with B<--sshlogin>.
|
||||
|
||||
|
||||
=item B<--trc> I<filename>
|
||||
|
||||
Transfer, Return, Cleanup. Short hand for:
|
||||
Transfer, Return, Cleanup. Shorthand for:
|
||||
|
||||
B<--transfer> B<--return> I<filename> B<--cleanup>
|
||||
B<--transferfile {}> B<--return> I<filename> B<--cleanup>
|
||||
|
||||
|
||||
=item B<--trim> <n|l|r|lr|rl>
|
||||
|
|
@ -2105,9 +2114,9 @@ Print the version GNU B<parallel> and exit.
|
|||
|
||||
=item B<--wd> I<mydir>
|
||||
|
||||
Files transferred using B<--transfer> and B<--return> will be relative
|
||||
to I<mydir> on remote computers, and the command will be executed in
|
||||
the dir I<mydir>.
|
||||
Files transferred using B<--transferfile> and B<--return> will be
|
||||
relative to I<mydir> on remote computers, and the command will be
|
||||
executed in the dir I<mydir>.
|
||||
|
||||
The special I<mydir> value B<...> will create working dirs under
|
||||
B<~/.parallel/tmp/> on the remote computers. If B<--cleanup> is given
|
||||
|
|
|
|||
|
|
@ -18,7 +18,6 @@ echo '**'
|
|||
echo '### Test of --retries on unreachable host'
|
||||
### Test of --retries on unreachable host
|
||||
seq 2 | stdout parallel -k --retries 2 -v -S 4.3.2.1,: echo
|
||||
ssh: connect to host 4.3.2.1 port 22: Connection timed out
|
||||
parallel: Warning: Could not figure out number of cpus on 4.3.2.1 (). Using 1.
|
||||
echo 1
|
||||
1
|
||||
|
|
|
|||
Loading…
Reference in a new issue