mirror of
https://git.savannah.gnu.org/git/parallel.git
synced 2024-11-22 14:07:55 +00:00
src/parallel: --trc works for minimal examples.
This commit is contained in:
Ole Tange
parent
fc7fba878b
commit
26868cdc4f
80
src/parallel
80
src/parallel
|
|
@ -1181,8 +1181,12 @@ sub parse_options {
|
|||
"max-line-length-allowed" => \$::opt_max_line_length_allowed,
|
||||
"number-of-cpus" => \$::opt_number_of_cpus,
|
||||
"number-of-cores" => \$::opt_number_of_cores,
|
||||
"sshlogin|S=s" => \@Global::sshlogin,
|
||||
"sshlogin|S=s" => \@::opt_sshlogin,
|
||||
"sshloginfile=s" => \$::opt_sshloginfile,
|
||||
"return=s" => \@::opt_return,
|
||||
"trc=s" => \@::opt_trc,
|
||||
"transfer" => \$::opt_transfer,
|
||||
"cleanup" => \$::opt_cleanup,
|
||||
# xargs-compatibility - implemented, man, unittest
|
||||
"max-procs|P=s" => \$::opt_P,
|
||||
"delimiter|d=s" => \$::opt_d,
|
||||
|
|
@ -1200,7 +1204,6 @@ sub parse_options {
|
|||
## xargs-compatibility - implemented, man - unittest missing
|
||||
"interactive|p" => \$::opt_p,
|
||||
## How to unittest? tty skal emuleres
|
||||
|
||||
# xargs-compatibility - unimplemented
|
||||
"L=i" => \$::opt_L,
|
||||
"max-lines|l:i" => \$::opt_l,
|
||||
|
|
@ -1236,7 +1239,14 @@ sub parse_options {
|
|||
if(defined $::opt_max_line_length_allowed) { print real_max_length(),"\n"; exit(0); }
|
||||
if(defined $::opt_version) { version(); exit(0); }
|
||||
if(defined $::opt_show_limits) { show_limits(); }
|
||||
if(defined @::opt_sshlogin) { @Global::sshlogin = @::opt_sshlogin; }
|
||||
if(defined $::opt_sshloginfile) { read_sshloginfile($::opt_sshloginfile); }
|
||||
if(defined @::opt_return) { push @Global::ret_files, @::opt_return; }
|
||||
if(defined @::opt_trc) {
|
||||
push @Global::ret_files, @::opt_trc;
|
||||
$::opt_transfer = 1;
|
||||
$::opt_cleanup = 1;
|
||||
}
|
||||
|
||||
if(defined $::opt_a) {
|
||||
if(not open(ARGFILE,"<".$::opt_a)) {
|
||||
|
|
@ -1248,7 +1258,7 @@ sub parse_options {
|
|||
|
||||
if(@ARGV) {
|
||||
if($Global::quoting) {
|
||||
$Global::command = join(" ", shell_quote(@ARGV));
|
||||
$Global::command = shell_quote(@ARGV);
|
||||
} else {
|
||||
$Global::command = join(" ", @ARGV);
|
||||
}
|
||||
|
|
@ -1276,6 +1286,12 @@ sub parse_options {
|
|||
# Generating the command line
|
||||
#
|
||||
|
||||
sub no_extension {
|
||||
my $no_ext = shift;
|
||||
$no_ext =~ s/\.[^\.]*$//; # Remove .ext from argument
|
||||
return $no_ext;
|
||||
}
|
||||
|
||||
sub generate_command_line {
|
||||
my $command = shift;
|
||||
my ($job_line,$last_good);
|
||||
|
|
@ -1290,8 +1306,7 @@ sub generate_command_line {
|
|||
# max number of lines (-L) =
|
||||
# number_of_read_lines = 0
|
||||
while (defined($next_arg = get_next_arg())) {
|
||||
my $next_arg_no_ext = $next_arg;
|
||||
$next_arg_no_ext =~ s/\.[^\.]*$//; # Remove .ext from argument
|
||||
my $next_arg_no_ext = no_extension($next_arg);
|
||||
# if defined max_number_of_lines
|
||||
# number_of_read_lines++
|
||||
# if $next_arg =~ /\w$/ then number_of_read_lines--
|
||||
|
|
@ -1349,7 +1364,7 @@ sub generate_command_line {
|
|||
}
|
||||
debug("Return jobline: $job_line\n");
|
||||
}
|
||||
return $job_line;
|
||||
return ($job_line,\@quoted_args);
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -1376,7 +1391,6 @@ sub xargs_computations {
|
|||
if($Global::Xargs) {
|
||||
$c =~ s/\S*\Q$Global::replacestring\E\S*//go;
|
||||
$c =~ s/\S*\Q$Global::replace_no_ext\E\S*//go;
|
||||
|
||||
$length_of_command_no_args = length($c) - 1;
|
||||
$length_of_context = length($command) - $length_of_command_no_args
|
||||
- $no_of_replace * length($Global::replacestring)
|
||||
|
|
@ -1403,8 +1417,7 @@ sub shell_quote {
|
|||
$arg =~ s/([\002-\011\013-\032])/\\$1/g;
|
||||
$arg =~ s/([\n])/'\n'/g; # filenames with '\n' is quoted using \'
|
||||
}
|
||||
#return wantarray ? @strings : "@strings";
|
||||
return @strings;
|
||||
return wantarray ? @strings : "@strings";
|
||||
}
|
||||
|
||||
# Replace foo{}bar or foo{.}bar
|
||||
|
|
@ -1508,7 +1521,7 @@ sub processes_available_by_system_limit {
|
|||
my $sshlogin = shift;
|
||||
my $system_limit=0;
|
||||
my @command_lines=();
|
||||
my $next_command_line;
|
||||
my ($next_command_line, $args_ref);
|
||||
my $more_filehandles;
|
||||
my $max_system_proc_reached=0;
|
||||
my $spawning_too_slow=0;
|
||||
|
|
@ -1528,9 +1541,9 @@ sub processes_available_by_system_limit {
|
|||
|
||||
# If there are no more command lines, then we have a process
|
||||
# per command line, so no need to go further
|
||||
$next_command_line = next_command_line();
|
||||
($next_command_line, $args_ref) = next_command_line();
|
||||
if(defined $next_command_line) {
|
||||
push(@command_lines, $next_command_line);
|
||||
push(@command_lines, $next_command_line, $args_ref);
|
||||
}
|
||||
|
||||
# Every simultaneous process uses 2 filehandles when grouping
|
||||
|
|
@ -1585,7 +1598,7 @@ sub processes_available_by_system_limit {
|
|||
waitpid($pid,0);
|
||||
}
|
||||
wait();
|
||||
# Cleanup: Unget the command_lines
|
||||
# Cleanup: Unget the command_lines (and args_refs)
|
||||
unget_command_line(@command_lines);
|
||||
return $system_limit;
|
||||
}
|
||||
|
|
@ -1727,26 +1740,49 @@ sub init_run_jobs {
|
|||
$SIG{TERM} = \&StartNoNewJobs;
|
||||
}
|
||||
|
||||
sub login_and_host {
|
||||
my $sshlogin = shift;
|
||||
$sshlogin =~ /(\S+$)/ or die;
|
||||
return $1;
|
||||
}
|
||||
|
||||
sub next_command_line_with_sshlogin {
|
||||
my $sshlogin = shift;
|
||||
my $next_command_line = next_command_line();
|
||||
my ($next_command_line, $args_ref) = next_command_line();
|
||||
my ($pre,$post)=("","");
|
||||
if($next_command_line and $sshlogin ne ":") {
|
||||
return "ssh $sshlogin ".join("",shell_quote($next_command_line));
|
||||
my $remote = login_and_host($sshlogin);
|
||||
for my $file (@$args_ref) {
|
||||
$file =~ s:^([^/]):./$1:; # If relative path: prepend ./ (to avoid problems with ':')
|
||||
if($::opt_transfer) {
|
||||
$pre = "rsync -az $file $remote:".shell_quote($file)." ;";
|
||||
}
|
||||
for my $ret_file (@Global::ret_files) {
|
||||
my $remove = $::opt_cleanup ? "--remove-source-files" : "";
|
||||
my $replaced = context_replace($ret_file,[$file],[no_extension($file)]);
|
||||
$post .= "rsync -az $remove $remote:".shell_quote($replaced)." $replaced ;";
|
||||
}
|
||||
if($::opt_cleanup) {
|
||||
$post .= "ssh $sshlogin rm ".shell_quote($file).";";
|
||||
}
|
||||
}
|
||||
return "$pre ssh $sshlogin ".shell_quote($next_command_line)."; $post";
|
||||
} else {
|
||||
return $next_command_line;
|
||||
}
|
||||
}
|
||||
|
||||
sub next_command_line {
|
||||
my $cmd_line;
|
||||
my ($cmd_line,$args_ref);
|
||||
if(@Global::unget_next_command_line) {
|
||||
$cmd_line = shift @Global::unget_next_command_line;
|
||||
$args_ref = shift @Global::unget_next_command_line;
|
||||
} else {
|
||||
do {
|
||||
$cmd_line = generate_command_line($Global::command);
|
||||
($cmd_line,$args_ref) = generate_command_line($Global::command);
|
||||
} while (defined $cmd_line and $cmd_line =~ /^\s*$/); # Skip empty lines
|
||||
}
|
||||
return $cmd_line;
|
||||
return ($cmd_line,$args_ref);
|
||||
}
|
||||
|
||||
sub unget_command_line {
|
||||
|
|
@ -1774,7 +1810,7 @@ sub get_next_arg {
|
|||
}
|
||||
}
|
||||
if($Global::input_is_filename) {
|
||||
($arg) = shell_quote($arg);
|
||||
$arg = shell_quote($arg);
|
||||
}
|
||||
}
|
||||
debug("Next arg: ".$arg."\n");
|
||||
|
|
@ -2193,3 +2229,9 @@ $main::opt_E = $main::opt_r = $Global::xargs = $Global::keeporder = 0;
|
|||
# blokkene samlet med den mest egnede algoritme.
|
||||
|
||||
# Moved parse options to parse_options
|
||||
|
||||
# TODO max_line_length on remote
|
||||
# TODO compute how many can be transferred within max_line_length
|
||||
# TODO Unittest with filename that is long and requires a lot of quoting. Will there be to many
|
||||
# TODO Unittest --trc without -S should warn
|
||||
# TODO Unittest --transfer file called ' : & ) *.jpg'
|
||||
|
|
|
|||
|
|
@ -1,53 +1,53 @@
|
|||
1
|
||||
2
|
||||
3
|
||||
sleep 1; echo 1
|
||||
1
|
||||
sleep 1; echo 2
|
||||
2
|
||||
ssh nlv.pi.dk sleep\ 1\;\ echo\ 3
|
||||
sleep 1; echo 1
|
||||
1
|
||||
ssh nlv.pi.dk sleep\ 1\;\ echo\ 3;
|
||||
3
|
||||
ssh nlv.pi.dk sleep\ 1\;\ echo\ \\\>/tmp/fire
|
||||
sleep 1; echo \>/tmp/fire
|
||||
>/tmp/fire
|
||||
sleep 1; echo 5
|
||||
5
|
||||
sleep 1; echo 6
|
||||
ssh nlv.pi.dk sleep\ 1\;\ echo\ 6;
|
||||
6
|
||||
ssh nlv.pi.dk sleep\ 1\;\ echo\ 7
|
||||
sleep 1; echo 7
|
||||
7
|
||||
ssh nlv.pi.dk sleep\ 1\;\ echo\ 8
|
||||
sleep 1; echo 8
|
||||
8
|
||||
sleep 1; echo 9
|
||||
ssh nlv.pi.dk sleep\ 1\;\ echo\ 9;
|
||||
9
|
||||
sleep 1; echo 10
|
||||
10
|
||||
sleep 1; hostname; echo 1
|
||||
alpha
|
||||
1
|
||||
sleep 1; hostname; echo 2
|
||||
alpha
|
||||
ssh nlv.pi.dk sleep\ 1\;\ hostname\;\ echo\ 2;
|
||||
nlv.pi.dk
|
||||
2
|
||||
ssh nlv.pi.dk sleep\ 1\;\ hostname\;\ echo\ 3
|
||||
ssh nlv.pi.dk sleep\ 1\;\ hostname\;\ echo\ 3;
|
||||
nlv.pi.dk
|
||||
3
|
||||
ssh nlv.pi.dk sleep\ 1\;\ hostname\;\ echo\ \\\>/tmp/fire
|
||||
ssh nlv.pi.dk sleep\ 1\;\ hostname\;\ echo\ \\\>/tmp/fire;
|
||||
nlv.pi.dk
|
||||
>/tmp/fire
|
||||
sleep 1; hostname; echo 5
|
||||
alpha
|
||||
ssh nlv.pi.dk sleep\ 1\;\ hostname\;\ echo\ 5;
|
||||
nlv.pi.dk
|
||||
5
|
||||
sleep 1; hostname; echo 6
|
||||
alpha
|
||||
ssh nlv.pi.dk sleep\ 1\;\ hostname\;\ echo\ 6;
|
||||
nlv.pi.dk
|
||||
6
|
||||
ssh nlv.pi.dk sleep\ 1\;\ hostname\;\ echo\ 7
|
||||
ssh nlv.pi.dk sleep\ 1\;\ hostname\;\ echo\ 7;
|
||||
nlv.pi.dk
|
||||
7
|
||||
ssh nlv.pi.dk sleep\ 1\;\ hostname\;\ echo\ 8
|
||||
ssh nlv.pi.dk sleep\ 1\;\ hostname\;\ echo\ 8;
|
||||
nlv.pi.dk
|
||||
8
|
||||
sleep 1; hostname; echo 9
|
||||
alpha
|
||||
ssh nlv.pi.dk sleep\ 1\;\ hostname\;\ echo\ 9;
|
||||
nlv.pi.dk
|
||||
9
|
||||
sleep 1; hostname; echo 10
|
||||
alpha
|
||||
ssh nlv.pi.dk sleep\ 1\;\ hostname\;\ echo\ 10;
|
||||
nlv.pi.dk
|
||||
10
|
||||
|
|
|
|||
53
unittest/wanted-results/test17
Normal file
53
unittest/wanted-results/test17
Normal file
|
|
@ -0,0 +1,53 @@
|
|||
1
|
||||
2
|
||||
3
|
||||
sleep 1; echo 2
|
||||
2
|
||||
sleep 1; echo 1
|
||||
1
|
||||
ssh nlv.pi.dk sleep\ 1\;\ echo\ 3;
|
||||
3
|
||||
sleep 1; echo \>/tmp/fire
|
||||
>/tmp/fire
|
||||
sleep 1; echo 5
|
||||
5
|
||||
ssh nlv.pi.dk sleep\ 1\;\ echo\ 6;
|
||||
6
|
||||
sleep 1; echo 7
|
||||
7
|
||||
sleep 1; echo 8
|
||||
8
|
||||
ssh nlv.pi.dk sleep\ 1\;\ echo\ 9;
|
||||
9
|
||||
sleep 1; echo 10
|
||||
10
|
||||
sleep 1; hostname; echo 1
|
||||
alpha
|
||||
1
|
||||
ssh nlv.pi.dk sleep\ 1\;\ hostname\;\ echo\ 2;
|
||||
nlv.pi.dk
|
||||
2
|
||||
ssh nlv.pi.dk sleep\ 1\;\ hostname\;\ echo\ 3;
|
||||
nlv.pi.dk
|
||||
3
|
||||
ssh nlv.pi.dk sleep\ 1\;\ hostname\;\ echo\ \\\>/tmp/fire;
|
||||
nlv.pi.dk
|
||||
>/tmp/fire
|
||||
ssh nlv.pi.dk sleep\ 1\;\ hostname\;\ echo\ 5;
|
||||
nlv.pi.dk
|
||||
5
|
||||
ssh nlv.pi.dk sleep\ 1\;\ hostname\;\ echo\ 6;
|
||||
nlv.pi.dk
|
||||
6
|
||||
ssh nlv.pi.dk sleep\ 1\;\ hostname\;\ echo\ 7;
|
||||
nlv.pi.dk
|
||||
7
|
||||
ssh nlv.pi.dk sleep\ 1\;\ hostname\;\ echo\ 8;
|
||||
nlv.pi.dk
|
||||
8
|
||||
ssh nlv.pi.dk sleep\ 1\;\ hostname\;\ echo\ 9;
|
||||
nlv.pi.dk
|
||||
9
|
||||
ssh nlv.pi.dk sleep\ 1\;\ hostname\;\ echo\ 10;
|
||||
nlv.pi.dk
|
||||
10
|
||||
Loading…
Reference in a new issue