mirror of
https://git.savannah.gnu.org/git/parallel.git
synced 2024-12-22 20:57:53 +00:00
--basefile no longer breaks old tests
This commit is contained in:
Ole Tange
parent
c01590571d
commit
db1f767749
89
src/parallel
89
src/parallel
|
|
@ -2336,36 +2336,18 @@ sub next_command_line_with_sshlogin {
|
|||
my ($sshcmd,$serverlogin) = sshcommand_of_sshlogin($sshlogin);
|
||||
my ($pre,$post)=("","");
|
||||
if($next_command_line and $serverlogin ne ":") {
|
||||
my $rsync_opt = "-rlDzR -e".shell_quote($sshcmd);
|
||||
for my $file (@$args_ref) {
|
||||
$file =~ s:/\./:/:g; # Rsync treats /./ special. We dont want that
|
||||
my $noext = no_extension($file); # Remove .ext before prepending ./
|
||||
my $relpath = ($file !~ m:^/:); # Is the path relative?
|
||||
# If relative path: prepend ./ (to avoid problems with ':')
|
||||
$noext = ($relpath ? "./".$noext : $noext);
|
||||
# Use different subdirs depending on abs or rel path
|
||||
my $rsync_destdir = ($relpath ? "./" : "/");
|
||||
if($::opt_transfer) {
|
||||
# --transfer
|
||||
# Abs path: rsync -rlDzR /home/tange/dir/subdir/file.gz server:/
|
||||
# Rel path: rsync -rlDzR ./subdir/file.gz server:./
|
||||
if(-r shell_unquote($file)) {
|
||||
$pre .= "rsync $rsync_opt $file $serverlogin:$rsync_destdir ;";
|
||||
} else {
|
||||
print STDERR "Warning: $file is not readable and will not be transferred\n";
|
||||
}
|
||||
$pre .= sshtransfer($sshlogin,$file).";";
|
||||
}
|
||||
for my $ret_file (@Global::ret_files) {
|
||||
my $remove = $::opt_cleanup ? "--remove-source-files" : "";
|
||||
my $replaced = context_replace($ret_file,[$file],[$noext]);
|
||||
# --return
|
||||
# Abs path: rsync -rlDzR server:/home/tange/dir/subdir/file.gz /
|
||||
# Rel path: rsync -rlDzR server:./subsir/file.gz ./
|
||||
$post .= "rsync $rsync_opt $remove $serverlogin:"
|
||||
.shell_quote($replaced)." $rsync_destdir ;";
|
||||
if(@Global::ret_files) {
|
||||
# --return or --trc
|
||||
$post .= sshreturn($sshlogin,$file).";";
|
||||
}
|
||||
if($::opt_cleanup) {
|
||||
$post .= "$sshcmd $serverlogin rm -f ".shell_quote($file).";";
|
||||
# --cleanup
|
||||
$post .= sshcleanup($sshlogin,$file).";";
|
||||
}
|
||||
}
|
||||
if($post) {
|
||||
|
|
@ -2692,6 +2674,61 @@ sub control_path_dir {
|
|||
return $Global::control_path_dir;
|
||||
}
|
||||
|
||||
sub sshtransfer {
|
||||
# Return the sshcommand needed to transfer the file
|
||||
return sshtransferreturn(@_,1,0);
|
||||
}
|
||||
|
||||
sub sshreturn {
|
||||
# Return the sshcommand needed to returning the file
|
||||
my $removesource = $::opt_cleanup;
|
||||
return sshtransferreturn(@_,0,$removesource);
|
||||
}
|
||||
|
||||
sub sshcleanup {
|
||||
# Return the sshcommand needed to remove the file
|
||||
my ($sshlogin,$file) = (@_);
|
||||
my ($sshcmd,$serverlogin) = sshcommand_of_sshlogin($sshlogin);
|
||||
return "$sshcmd $serverlogin rm -f ".shell_quote($file);
|
||||
}
|
||||
|
||||
sub sshtransferreturn {
|
||||
my ($sshlogin,$file,$transfer,$removesource) = (@_);
|
||||
my ($sshcmd,$serverlogin) = sshcommand_of_sshlogin($sshlogin);
|
||||
my $rsync_opt = "-rlDzRE -e".shell_quote($sshcmd);
|
||||
$file =~ s:/\./:/:g; # Rsync treats /./ special. We dont want that
|
||||
$file =~ s:^\./::g; # Remove ./ if any
|
||||
my $relpath = ($file !~ m:^/:); # Is the path relative?
|
||||
# Use different subdirs depending on abs or rel path
|
||||
my $rsync_destdir = ($relpath ? "./" : "/");
|
||||
if($transfer) {
|
||||
# Abs path: rsync -rlDzR /home/tange/dir/subdir/file.gz server:/
|
||||
# Rel path: rsync -rlDzR ./subdir/file.gz server:./
|
||||
if(-r shell_unquote($file)) {
|
||||
return "rsync $rsync_opt $file $serverlogin:$rsync_destdir";
|
||||
} else {
|
||||
print STDERR "Warning: $file is not readable and will not be transferred\n";
|
||||
return "true"; # dummy command to run
|
||||
}
|
||||
} else {
|
||||
my $noext = no_extension($file); # Remove .ext before prepending ./
|
||||
# If relative path: prepend ./ (to avoid problems with ':')
|
||||
$noext = ($relpath ? "./".$noext : $noext);
|
||||
$file = ($relpath ? "./".$file : $file);
|
||||
my @cmd = ();
|
||||
for my $ret_file (@Global::ret_files) {
|
||||
my $remove = $removesource ? "--remove-source-files" : "";
|
||||
my $replaced = context_replace($ret_file,[$file],[$noext]);
|
||||
# --return
|
||||
# Abs path: rsync -rlDzR server:/home/tange/dir/subdir/file.gz /
|
||||
# Rel path: rsync -rlDzR server:./subsir/file.gz ./
|
||||
push(@cmd, "rsync $rsync_opt $remove $serverlogin:"
|
||||
.shell_quote($replaced)." ".$rsync_destdir);
|
||||
}
|
||||
return join(";",@cmd);
|
||||
}
|
||||
}
|
||||
|
||||
sub setup_basefile {
|
||||
# Transfer basefiles to each $sshlogin
|
||||
# This needs to be done before first jobs on $sshlogin is run
|
||||
|
|
@ -2951,8 +2988,8 @@ $Global::control_path = 0;
|
|||
# TODO --max-number-of-jobs print the system limited number of jobs
|
||||
|
||||
# TODO Debian package
|
||||
# TODO transfer a script to be run
|
||||
|
||||
# TODO transfer a script to be run - are permissions kept?
|
||||
# TODO try transfer + return of file beginning with :
|
||||
|
||||
# TODO to kill from a run script parallel should set PARALLEL_PID that can be sig termed
|
||||
# TAGS: parallel | parallel processing | multicore | multiprocessor | Clustering/Distributed Networks
|
||||
|
|
|
|||
|
|
@ -81,9 +81,9 @@ parallel-server1 cat tmp/parallel.file.'
|
|||
'newline2.out;cat tmp/parallel.file.'
|
||||
'newline2 > tmp/parallel.file.'
|
||||
'newline2.out2
|
||||
parallel-server1 rsync --server --sender -lDrRze.iLsf --remove-source-files . tmp/parallel.file.'
|
||||
parallel-server1 rsync --server --sender -lDrRze.iLsf --remove-source-files . ./tmp/parallel.file.'
|
||||
'newline2.out
|
||||
parallel-server1 rsync --server --sender -lDrRze.iLsf --remove-source-files . tmp/parallel.file.'
|
||||
parallel-server1 rsync --server --sender -lDrRze.iLsf --remove-source-files . ./tmp/parallel.file.'
|
||||
'newline2.out2
|
||||
parallel-server1 rm -f tmp/parallel.file.'
|
||||
'newline2
|
||||
|
|
@ -93,9 +93,9 @@ parallel@parallel-server2 cat tmp/parallel.file.'
|
|||
'newline1.out;cat tmp/parallel.file.'
|
||||
'newline1 > tmp/parallel.file.'
|
||||
'newline1.out2
|
||||
-l parallel parallel-server2 rsync --server --sender -lDrRze.iLsf --remove-source-files . tmp/parallel.file.'
|
||||
-l parallel parallel-server2 rsync --server --sender -lDrRze.iLsf --remove-source-files . ./tmp/parallel.file.'
|
||||
'newline1.out
|
||||
-l parallel parallel-server2 rsync --server --sender -lDrRze.iLsf --remove-source-files . tmp/parallel.file.'
|
||||
-l parallel parallel-server2 rsync --server --sender -lDrRze.iLsf --remove-source-files . ./tmp/parallel.file.'
|
||||
'newline1.out2
|
||||
parallel@parallel-server2 rm -f tmp/parallel.file.'
|
||||
'newline1
|
||||
|
|
|
|||
|
|
@ -12,10 +12,10 @@ sleep 2;false
|
|||
sh: non_exist: command not found
|
||||
2
|
||||
parallel: Starting no more jobs. Waiting for 3 jobs to finish. This job failed:
|
||||
sleep 1;false
|
||||
sleep 2;false
|
||||
sh: non_exist: command not found
|
||||
parallel: Starting no more jobs. Waiting for 1 jobs to finish. This job failed:
|
||||
sleep 1; non_exist
|
||||
sleep 4; non_exist
|
||||
127
|
||||
parallel: This job failed:
|
||||
sleep 2;false
|
||||
|
|
|
|||
|
|
@ -19,9 +19,9 @@ echo $?
|
|||
(echo "sleep 1;true"; echo "sleep 2;false";echo "sleep 3;true") | parallel --halt-on-error 2
|
||||
echo $?
|
||||
|
||||
(echo "sleep 1;true"; echo "sleep 1;false";echo "sleep 1;true";echo "sleep 1; non_exist") | parallel -H0
|
||||
(echo "sleep 1;true"; echo "sleep 2;false";echo "sleep 3;true";echo "sleep 4; non_exist") | parallel -H0
|
||||
echo $?
|
||||
(echo "sleep 1;true"; echo "sleep 1;false";echo "sleep 1;true";echo "sleep 1; non_exist") | parallel -H1
|
||||
(echo "sleep 1;true"; echo "sleep 2;false";echo "sleep 3;true";echo "sleep 4; non_exist") | parallel -H1
|
||||
echo $?
|
||||
(echo "sleep 1;true"; echo "sleep 2;false";echo "sleep 3;true";echo "sleep 4; non_exist") | parallel -H2
|
||||
echo $?
|
||||
|
|
|
|||
|
|
@ -81,9 +81,9 @@ parallel-server1 cat tmp/parallel.file.'
|
|||
'newline2.out;cat tmp/parallel.file.'
|
||||
'newline2 > tmp/parallel.file.'
|
||||
'newline2.out2
|
||||
parallel-server1 rsync --server --sender -lDrRze.iLsf --remove-source-files . tmp/parallel.file.'
|
||||
parallel-server1 rsync --server --sender -lDrRze.iLsf --remove-source-files . ./tmp/parallel.file.'
|
||||
'newline2.out
|
||||
parallel-server1 rsync --server --sender -lDrRze.iLsf --remove-source-files . tmp/parallel.file.'
|
||||
parallel-server1 rsync --server --sender -lDrRze.iLsf --remove-source-files . ./tmp/parallel.file.'
|
||||
'newline2.out2
|
||||
parallel-server1 rm -f tmp/parallel.file.'
|
||||
'newline2
|
||||
|
|
@ -93,9 +93,9 @@ parallel@parallel-server2 cat tmp/parallel.file.'
|
|||
'newline1.out;cat tmp/parallel.file.'
|
||||
'newline1 > tmp/parallel.file.'
|
||||
'newline1.out2
|
||||
-l parallel parallel-server2 rsync --server --sender -lDrRze.iLsf --remove-source-files . tmp/parallel.file.'
|
||||
-l parallel parallel-server2 rsync --server --sender -lDrRze.iLsf --remove-source-files . ./tmp/parallel.file.'
|
||||
'newline1.out
|
||||
-l parallel parallel-server2 rsync --server --sender -lDrRze.iLsf --remove-source-files . tmp/parallel.file.'
|
||||
-l parallel parallel-server2 rsync --server --sender -lDrRze.iLsf --remove-source-files . ./tmp/parallel.file.'
|
||||
'newline1.out2
|
||||
parallel@parallel-server2 rm -f tmp/parallel.file.'
|
||||
'newline1
|
||||
|
|
|
|||
|
|
@ -12,10 +12,10 @@ sleep 2;false
|
|||
sh: non_exist: command not found
|
||||
2
|
||||
parallel: Starting no more jobs. Waiting for 3 jobs to finish. This job failed:
|
||||
sleep 1;false
|
||||
sleep 2;false
|
||||
sh: non_exist: command not found
|
||||
parallel: Starting no more jobs. Waiting for 1 jobs to finish. This job failed:
|
||||
sleep 1; non_exist
|
||||
sleep 4; non_exist
|
||||
127
|
||||
parallel: This job failed:
|
||||
sleep 2;false
|
||||
|
|
|
|||
Loading…
Reference in a new issue