src/parallel: --trc works for minimal examples.

src/parallel

@@ -1181,8 +1181,12 @@ sub parse_options {
 	       "max-line-length-allowed" => \$::opt_max_line_length_allowed,
 	       "number-of-cpus" => \$::opt_number_of_cpus,
 	       "number-of-cores" => \$::opt_number_of_cores,
-	       "sshlogin|S=s" => \@Global::sshlogin,
+	       "sshlogin|S=s" => \@::opt_sshlogin,
 	       "sshloginfile=s" => \$::opt_sshloginfile,
+	       "return=s" => \@::opt_return,
+	       "trc=s" => \@::opt_trc,
+	       "transfer" => \$::opt_transfer,
+	       "cleanup" => \$::opt_cleanup,
 	       # xargs-compatibility - implemented, man, unittest
 	       "max-procs|P=s" => \$::opt_P,
 	       "delimiter|d=s" => \$::opt_d,
@@ -1200,7 +1204,6 @@ sub parse_options {
 	       ## xargs-compatibility - implemented, man - unittest missing
 	       "interactive|p" => \$::opt_p,
 	       ## How to unittest? tty skal emuleres
-	       
 	       # xargs-compatibility - unimplemented
 	       "L=i" => \$::opt_L,
 	       "max-lines|l:i" => \$::opt_l,
@@ -1236,7 +1239,14 @@ sub parse_options {
     if(defined $::opt_max_line_length_allowed) { print real_max_length(),"\n"; exit(0); }
     if(defined $::opt_version) { version(); exit(0); }
     if(defined $::opt_show_limits) { show_limits(); }
+    if(defined @::opt_sshlogin) { @Global::sshlogin = @::opt_sshlogin; }
     if(defined $::opt_sshloginfile) { read_sshloginfile($::opt_sshloginfile); }
+    if(defined @::opt_return) { push @Global::ret_files, @::opt_return; }
+    if(defined @::opt_trc) {
+	push @Global::ret_files, @::opt_trc;
+	$::opt_transfer = 1;
+	$::opt_cleanup = 1;
+    }
 
     if(defined $::opt_a) {
 	if(not open(ARGFILE,"<".$::opt_a)) {
@@ -1248,7 +1258,7 @@ sub parse_options {
 
     if(@ARGV) {
 	if($Global::quoting) {
-	    $Global::command = join(" ", shell_quote(@ARGV)); 
+	    $Global::command = shell_quote(@ARGV);
 	} else {
 	    $Global::command = join(" ", @ARGV);
 	}
@@ -1276,6 +1286,12 @@ sub parse_options {
 # Generating the command line
 #
 
+sub no_extension {
+    my $no_ext = shift;
+    $no_ext =~ s/\.[^\.]*$//; # Remove .ext from argument
+    return $no_ext;
+}
+
 sub generate_command_line {
     my $command = shift;
     my ($job_line,$last_good);
@@ -1290,8 +1306,7 @@ sub generate_command_line {
     # max number of lines (-L) =
     # number_of_read_lines = 0
     while (defined($next_arg = get_next_arg())) {
-	my $next_arg_no_ext = $next_arg;
+	my $next_arg_no_ext = no_extension($next_arg);
-	$next_arg_no_ext =~ s/\.[^\.]*$//; # Remove .ext from argument
 	# if defined max_number_of_lines
 	# number_of_read_lines++
 	# if $next_arg =~ /\w$/ then number_of_read_lines--
@@ -1349,7 +1364,7 @@ sub generate_command_line {
 	}
 	debug("Return jobline: $job_line\n");
     }
-    return $job_line;
+    return ($job_line,\@quoted_args);
 }
 
 
@@ -1376,7 +1391,6 @@ sub xargs_computations {
 	if($Global::Xargs) {
 	    $c =~ s/\S*\Q$Global::replacestring\E\S*//go;
 	    $c =~ s/\S*\Q$Global::replace_no_ext\E\S*//go;
-	    
 	    $length_of_command_no_args = length($c) - 1;
 	    $length_of_context = length($command) - $length_of_command_no_args
 		- $no_of_replace * length($Global::replacestring)
@@ -1403,8 +1417,7 @@ sub shell_quote {
 	$arg =~ s/([\002-\011\013-\032])/\\$1/g;
 	$arg =~ s/([\n])/'\n'/g; # filenames with '\n' is quoted using \'
     }
-    #return wantarray ? @strings : "@strings";  
+    return wantarray ? @strings : "@strings";
-    return @strings;
 }
 
 # Replace foo{}bar or foo{.}bar
@@ -1508,7 +1521,7 @@ sub processes_available_by_system_limit {
     my $sshlogin = shift;
     my $system_limit=0;
     my @command_lines=();
-    my $next_command_line;
+    my ($next_command_line, $args_ref);
     my $more_filehandles;
     my $max_system_proc_reached=0;
     my $spawning_too_slow=0;
@@ -1528,9 +1541,9 @@ sub processes_available_by_system_limit {
 
 	# If there are no more command lines, then we have a process
 	# per command line, so no need to go further
-	$next_command_line = next_command_line();
+	($next_command_line, $args_ref) = next_command_line();
 	if(defined $next_command_line) {
-	    push(@command_lines, $next_command_line);
+	    push(@command_lines, $next_command_line, $args_ref);
 	}
 
 	# Every simultaneous process uses 2 filehandles when grouping
@@ -1585,7 +1598,7 @@ sub processes_available_by_system_limit {
 	waitpid($pid,0);
     }
     wait();
-    # Cleanup: Unget the command_lines
+    # Cleanup: Unget the command_lines (and args_refs)
     unget_command_line(@command_lines);
     return $system_limit;
 }
@@ -1727,26 +1740,49 @@ sub init_run_jobs {
     $SIG{TERM} = \&StartNoNewJobs;
 }
 
+sub login_and_host {
+    my $sshlogin = shift;
+    $sshlogin =~ /(\S+$)/ or die;
+    return $1;
+}
+
 sub next_command_line_with_sshlogin {
     my $sshlogin = shift;
-    my $next_command_line = next_command_line();
+    my ($next_command_line, $args_ref) = next_command_line();
+    my ($pre,$post)=("","");
     if($next_command_line and $sshlogin ne ":") {
-	return "ssh $sshlogin ".join("",shell_quote($next_command_line));
+	my $remote = login_and_host($sshlogin);
+	for my $file (@$args_ref) {
+	    $file =~ s:^([^/]):./$1:; # If relative path: prepend ./ (to avoid problems with ':')
+	    if($::opt_transfer) {
+		$pre = "rsync -az $file $remote:".shell_quote($file)." ;";
+	    }
+	    for my $ret_file (@Global::ret_files) {
+		my $remove = $::opt_cleanup ? "--remove-source-files" : "";
+		my $replaced = context_replace($ret_file,[$file],[no_extension($file)]);
+		$post .= "rsync -az $remove $remote:".shell_quote($replaced)." $replaced ;";
+	    }
+	    if($::opt_cleanup) {
+		$post .= "ssh $sshlogin rm ".shell_quote($file).";";
+	    }
+	}
+	return "$pre ssh $sshlogin ".shell_quote($next_command_line)."; $post";
     } else {
 	return $next_command_line;
     }
 }
 
 sub next_command_line {
-    my $cmd_line;
+    my ($cmd_line,$args_ref);
     if(@Global::unget_next_command_line) {
 	$cmd_line = shift @Global::unget_next_command_line;
+	$args_ref = shift @Global::unget_next_command_line;
     } else {
 	do {
-	    $cmd_line = generate_command_line($Global::command);
+	    ($cmd_line,$args_ref) = generate_command_line($Global::command);
 	} while (defined $cmd_line and $cmd_line =~ /^\s*$/); # Skip empty lines
     }
-    return $cmd_line;
+    return ($cmd_line,$args_ref);
 }
 
 sub unget_command_line {
@@ -1774,7 +1810,7 @@ sub get_next_arg {
 	    }
 	}
 	if($Global::input_is_filename) {
-	    ($arg) = shell_quote($arg);
+	    $arg = shell_quote($arg);
 	}
     }
     debug("Next arg: ".$arg."\n");
@@ -2193,3 +2229,9 @@ $main::opt_E = $main::opt_r = $Global::xargs = $Global::keeporder = 0;
 # blokkene samlet med den mest egnede algoritme.
 
 # Moved parse options to parse_options
+
+# TODO max_line_length on remote
+# TODO compute how many can be transferred within max_line_length
+# TODO Unittest with filename that is long and requires a lot of quoting. Will there be to many
+# TODO Unittest --trc without -S should warn
+# TODO Unittest --transfer file called ' : & ) *.jpg'

unittest/actual-results/test17

@@ -1,53 +1,53 @@
 1
 2
 3
-sleep 1; echo 1
-1
 sleep 1; echo 2
 2
-ssh nlv.pi.dk sleep\ 1\;\ echo\ 3
+sleep 1; echo 1
+1
+ ssh nlv.pi.dk sleep\ 1\;\ echo\ 3; 
 3
-ssh nlv.pi.dk sleep\ 1\;\ echo\ \\\>/tmp/fire
+sleep 1; echo \>/tmp/fire
 >/tmp/fire
 sleep 1; echo 5
 5
-sleep 1; echo 6
+ ssh nlv.pi.dk sleep\ 1\;\ echo\ 6; 
 6
-ssh nlv.pi.dk sleep\ 1\;\ echo\ 7
+sleep 1; echo 7
 7
-ssh nlv.pi.dk sleep\ 1\;\ echo\ 8
+sleep 1; echo 8
 8
-sleep 1; echo 9
+ ssh nlv.pi.dk sleep\ 1\;\ echo\ 9; 
 9
 sleep 1; echo 10
 10
 sleep 1; hostname; echo 1
 alpha
 1
-sleep 1; hostname; echo 2
+ ssh nlv.pi.dk sleep\ 1\;\ hostname\;\ echo\ 2; 
-alpha
+nlv.pi.dk
 2
-ssh nlv.pi.dk sleep\ 1\;\ hostname\;\ echo\ 3
+ ssh nlv.pi.dk sleep\ 1\;\ hostname\;\ echo\ 3; 
 nlv.pi.dk
 3
-ssh nlv.pi.dk sleep\ 1\;\ hostname\;\ echo\ \\\>/tmp/fire
+ ssh nlv.pi.dk sleep\ 1\;\ hostname\;\ echo\ \\\>/tmp/fire; 
 nlv.pi.dk
 >/tmp/fire
-sleep 1; hostname; echo 5
+ ssh nlv.pi.dk sleep\ 1\;\ hostname\;\ echo\ 5; 
-alpha
+nlv.pi.dk
 5
-sleep 1; hostname; echo 6
+ ssh nlv.pi.dk sleep\ 1\;\ hostname\;\ echo\ 6; 
-alpha
+nlv.pi.dk
 6
-ssh nlv.pi.dk sleep\ 1\;\ hostname\;\ echo\ 7
+ ssh nlv.pi.dk sleep\ 1\;\ hostname\;\ echo\ 7; 
 nlv.pi.dk
 7
-ssh nlv.pi.dk sleep\ 1\;\ hostname\;\ echo\ 8
+ ssh nlv.pi.dk sleep\ 1\;\ hostname\;\ echo\ 8; 
 nlv.pi.dk
 8
-sleep 1; hostname; echo 9
+ ssh nlv.pi.dk sleep\ 1\;\ hostname\;\ echo\ 9; 
-alpha
+nlv.pi.dk
 9
-sleep 1; hostname; echo 10
+ ssh nlv.pi.dk sleep\ 1\;\ hostname\;\ echo\ 10; 
-alpha
+nlv.pi.dk
 10

unittest/wanted-results/test17

@@ -0,0 +1,53 @@
+1
+2
+3
+sleep 1; echo 2
+2
+sleep 1; echo 1
+1
+ ssh nlv.pi.dk sleep\ 1\;\ echo\ 3; 
+3
+sleep 1; echo \>/tmp/fire
+>/tmp/fire
+sleep 1; echo 5
+5
+ ssh nlv.pi.dk sleep\ 1\;\ echo\ 6; 
+6
+sleep 1; echo 7
+7
+sleep 1; echo 8
+8
+ ssh nlv.pi.dk sleep\ 1\;\ echo\ 9; 
+9
+sleep 1; echo 10
+10
+sleep 1; hostname; echo 1
+alpha
+1
+ ssh nlv.pi.dk sleep\ 1\;\ hostname\;\ echo\ 2; 
+nlv.pi.dk
+2
+ ssh nlv.pi.dk sleep\ 1\;\ hostname\;\ echo\ 3; 
+nlv.pi.dk
+3
+ ssh nlv.pi.dk sleep\ 1\;\ hostname\;\ echo\ \\\>/tmp/fire; 
+nlv.pi.dk
+>/tmp/fire
+ ssh nlv.pi.dk sleep\ 1\;\ hostname\;\ echo\ 5; 
+nlv.pi.dk
+5
+ ssh nlv.pi.dk sleep\ 1\;\ hostname\;\ echo\ 6; 
+nlv.pi.dk
+6
+ ssh nlv.pi.dk sleep\ 1\;\ hostname\;\ echo\ 7; 
+nlv.pi.dk
+7
+ ssh nlv.pi.dk sleep\ 1\;\ hostname\;\ echo\ 8; 
+nlv.pi.dk
+8
+ ssh nlv.pi.dk sleep\ 1\;\ hostname\;\ echo\ 9; 
+nlv.pi.dk
+9
+ ssh nlv.pi.dk sleep\ 1\;\ hostname\;\ echo\ 10; 
+nlv.pi.dk
+10