tange/parallel
1
0
Fork 0
mirror of https://git.savannah.gnu.org/git/parallel.git synced 2024-11-25 23:47:53 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

src/parallel: Number of remote CPUs now detected.

Browse source 
Potential bug: If max_command_length is different on the machines.
This commit is contained in:
Ole Tange 2010-05-17 08:41:28 +02:00
parent d26f62a363
commit fc7fba878b
1 changed files with 44 additions and 34 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

76
src/parallel
View file

@ -267,12 +267,21 @@ Print the number of CPUs and exit (used by GNU B<parallel> itself to
determine the number of CPUs on remote machines). determine the number of CPUs on remote machines).
=item B<--number-of-cores> =item B<--number-of-cores> (not implemented)
Print the number of cores and exit (used by GNU B<parallel> itself to determine the Print the number of cores and exit (used by GNU B<parallel> itself to determine the
number of cores on remote machines). number of cores on remote machines).
=item B<--interactive>
=item B<-p>
Prompt the user about whether to run each command line and read a line
from the terminal. Only run the command line if the response starts
with 'y' or 'Y'. Implies B<-t>.
=item B<--quote> =item B<--quote>
=item B<-q> =item B<-q>
@ -283,15 +292,6 @@ QUOTING. Most people will never need this. Quoting is disabled by
default. default.
=item B<--interactive>
=item B<-p>
Prompt the user about whether to run each command line and read a line
from the terminal. Only run the command line if the response starts
with 'y' or 'Y'. Implies B<-t>.
=item B<--no-run-if-empty> =item B<--no-run-if-empty>
=item B<-r> =item B<-r>
@ -1261,7 +1261,7 @@ sub parse_options {
if(defined $::opt_P) { if(defined $::opt_P) {
for my $sshlogin (keys %Global::host) { for my $sshlogin (keys %Global::host) {
$Global::host{$sshlogin}{'max_no_of_running'} = $Global::host{$sshlogin}{'max_no_of_running'} =
compute_number_of_processes($::opt_P); compute_number_of_processes($::opt_P,$sshlogin);
} }
} else { } else {
for my $sshlogin (keys %Global::host) { for my $sshlogin (keys %Global::host) {
@ -1491,9 +1491,10 @@ sub is_acceptable_command_line_length {
sub compute_number_of_processes { sub compute_number_of_processes {
# Number of processes wanted and limited by system ressources # Number of processes wanted and limited by system ressources
my $opt_P = shift; my $opt_P = shift;
my $wanted_processes = user_requested_processes($opt_P); my $sshlogin = shift;
my $wanted_processes = user_requested_processes($opt_P,$sshlogin);
debug("Wanted procs: $wanted_processes\n"); debug("Wanted procs: $wanted_processes\n");
my $system_limit = processes_available_by_system_limit($wanted_processes); my $system_limit = processes_available_by_system_limit($wanted_processes,$sshlogin);
debug("Limited to procs: $system_limit\n"); debug("Limited to procs: $system_limit\n");
return $system_limit; return $system_limit;
} }
@ -1504,6 +1505,7 @@ sub processes_available_by_system_limit {
# Limits are: File handles, number of input lines, processes, # Limits are: File handles, number of input lines, processes,
# and taking > 1 second to spawn 10 extra processes # and taking > 1 second to spawn 10 extra processes
my $wanted_processes = shift; my $wanted_processes = shift;
my $sshlogin = shift;
my $system_limit=0; my $system_limit=0;
my @command_lines=(); my @command_lines=();
my $next_command_line; my $next_command_line;
@ -1609,19 +1611,20 @@ sub enough_file_handles {
sub user_requested_processes { sub user_requested_processes {
# Parse the number of processes that the user asked for # Parse the number of processes that the user asked for
my $opt_P = shift; my $opt_P = shift;
my $sshlogin = shift;
my $processes; my $processes;
if(defined $opt_P) { if(defined $opt_P) {
if($opt_P =~ /^\+(\d+)$/) { if($opt_P =~ /^\+(\d+)$/) {
# E.g. -P +2 # E.g. -P +2
my $j = $1; my $j = $1;
$processes = $j + no_of_cpus(); $processes = $j + no_of_cpus_sshlogin($sshlogin);
} elsif ($opt_P =~ /^-(\d+)$/) { } elsif ($opt_P =~ /^-(\d+)$/) {
# E.g. -P -2 # E.g. -P -2
my $j = $1; my $j = $1;
$processes = no_of_cpus() - $j; $processes = no_of_cpus_sshlogin($sshlogin) - $j;
} elsif ($opt_P =~ /^(\d+)\%$/) { } elsif ($opt_P =~ /^(\d+)\%$/) {
my $j = $1; my $j = $1;
$processes = no_of_cpus() * $j / 100; $processes = no_of_cpus_sshlogin($sshlogin) * $j / 100;
} elsif ($opt_P =~ /^(\d+)$/) { } elsif ($opt_P =~ /^(\d+)$/) {
$processes = $1; $processes = $1;
if($processes == 0) { if($processes == 0) {
@ -1643,6 +1646,21 @@ sub no_of_cores {
return no_of_cpus(@_); return no_of_cpus(@_);
} }
sub no_of_cpus_sshlogin {
# Number of CPUs at this sshlogin
my $sshlogin = shift;
if(not $Global::host{$sshlogin}{'ncpus'}) {
if($sshlogin eq ":") {
$Global::host{$sshlogin}{'ncpus'} = no_of_cpus();
} else {
my $ncpu = qx(echo|ssh $sshlogin parallel --number-of-cpus);
chomp($ncpu);
$Global::host{$sshlogin}{'ncpus'} = $ncpu;
}
}
return $Global::host{$sshlogin}{'ncpus'};
}
sub no_of_cpus { sub no_of_cpus {
if(not $Global::no_of_cpus) { if(not $Global::no_of_cpus) {
local $/="\n"; # If delimiter is set, then $/ will be wrong local $/="\n"; # If delimiter is set, then $/ will be wrong
@ -1819,7 +1837,6 @@ sub start_another_job {
} }
} else { } else {
# No more file handles # No more file handles
warn("Not enough file handles to start another job");
return 0; return 0;
} }
} }
@ -1951,26 +1968,19 @@ sub read_sshloginfile {
sub parse_sshlogin { sub parse_sshlogin {
my ($ncpu,@login); my (@login);
if(not @Global::sshlogin) { @Global::sshlogin = (":"); } if(not @Global::sshlogin) { @Global::sshlogin = (":"); }
for my $ssh (@Global::sshlogin) { for my $sshlogin (@Global::sshlogin) {
# Split up -S sshlogin,sshlogin # Split up -S sshlogin,sshlogin
push (@login, (split /,/, $ssh)); push (@login, (split /,/, $sshlogin));
} }
for my $ssh (@login) { for my $sshlogin (@login) {
if($ssh =~ s:^(\d+)/::) { if($sshlogin =~ s:^(\d+)/::) {
$ncpu = $1; # Override default autodetected ncpus
} else { $Global::host{$sshlogin}{'ncpus'} = $1;
if($ssh eq ":") {
$ncpu = no_of_cpus();
} else {
$ncpu = qx(echo|ssh $ssh parallel --number-of-cpus);
chomp($ncpu);
}
} }
$Global::host{$ssh}{'no_of_running'} = 0; $Global::host{$sshlogin}{'no_of_running'} = 0;
$Global::host{$ssh}{'ncpus'} = $ncpu; $Global::host{$sshlogin}{'maxlength'} = max_length_of_command_line();
$Global::host{$ssh}{'maxlength'} = max_length_of_command_line();
} }
debug("sshlogin: ", my_dump(%Global::host)); debug("sshlogin: ", my_dump(%Global::host));
} }