tange/parallel
1
0
Fork 0
mirror of https://git.savannah.gnu.org/git/parallel.git synced 2024-11-25 23:47:53 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

src/parallel:parse_options refactoring

Browse source
This commit is contained in:
Ole Tange 2010-05-15 23:17:39 +02:00
parent 4d4cd03a0d
commit 3ab7103c70
1 changed files with 154 additions and 123 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

175
src/parallel
View file

@ -241,6 +241,7 @@ If the evaluated number is less than 1 then 1 will be used. See also
Keep sequence of output same as the order of input. If jobs 1 2 3 4 Keep sequence of output same as the order of input. If jobs 1 2 3 4
end in the sequence 3 1 4 2 the output will still be 1 2 3 4. end in the sequence 3 1 4 2 the output will still be 1 2 3 4.
=item B<--max-args>=I<max-args> =item B<--max-args>=I<max-args>
=item B<-n> I<max-args> =item B<-n> I<max-args>
@ -252,6 +253,14 @@ GNU B<parallel> will exit.
Only used with B<-m> and B<-X>. Only used with B<-m> and B<-X>.
=item B<--max-line-length-allowed>
Print the maximal number characters allowed on the command line and
exit (used by GNU B<parallel> itself to determine the line length
on remote machines).
=item B<--number-of-cpus> =item B<--number-of-cpus>
Print the number of CPUs and exit (used by GNU B<parallel> itself to Print the number of CPUs and exit (used by GNU B<parallel> itself to
@ -1129,28 +1138,34 @@ use File::Temp qw/ tempfile tempdir /;
use Getopt::Long; use Getopt::Long;
use strict; use strict;
my ($processes,$command); parse_options();
parse_sshlogin();
init_run_jobs();
DoNotReap();
start_more_jobs();
ReapIfNeeded();
drain_job_queue();
# Defaults: sub parse_options {
$Global::version = 20100428; # Defaults:
$Global::progname = 'parallel'; $Global::version = 20100428;
$Global::debug = 0; $Global::progname = 'parallel';
$Global::processes_to_run = 10; $Global::debug = 0;
$command = undef; $Global::processes_to_run = 10;
$Global::verbose = 0; $Global::verbose = 0;
$Global::grouped = 1; $Global::grouped = 1;
$Global::keeporder = 0; $Global::keeporder = 0;
$Global::quoting = 0; $Global::quoting = 0;
$Global::replacestring = '{}'; $Global::replacestring = '{}';
$Global::replace_no_ext = '{.}'; $Global::replace_no_ext = '{.}';
$/="\n"; $/="\n";
$Global::ignore_empty = 0; $Global::ignore_empty = 0;
$Global::argfile = *STDIN; $Global::argfile = *STDIN;
$Global::interactive = 0; $Global::interactive = 0;
$Global::stderr_verbose = 0; $Global::stderr_verbose = 0;
Getopt::Long::Configure ("bundling","require_order"); Getopt::Long::Configure ("bundling","require_order");
GetOptions("debug|D" => \$::opt_D, GetOptions("debug|D" => \$::opt_D,
"xargs|m" => \$::opt_m, "xargs|m" => \$::opt_m,
"X" => \$::opt_X, "X" => \$::opt_X,
"v" => \$::opt_v, "v" => \$::opt_v,
@ -1165,6 +1180,7 @@ GetOptions("debug|D" => \$::opt_D,
"I=s" => \$::opt_I, "I=s" => \$::opt_I,
"extensionreplace|U=s" => \$::opt_U, "extensionreplace|U=s" => \$::opt_U,
"jobs|j=s" => \$::opt_P, "jobs|j=s" => \$::opt_P,
"max-line-length-allowed" => \$::opt_max_line_length_allowed,
"number-of-cpus" => \$::opt_number_of_cpus, "number-of-cpus" => \$::opt_number_of_cpus,
"number-of-cores" => \$::opt_number_of_cores, "number-of-cores" => \$::opt_number_of_cores,
"sshlogin|S=s" => \@Global::sshlogin, "sshlogin|S=s" => \@Global::sshlogin,
@ -1187,71 +1203,64 @@ GetOptions("debug|D" => \$::opt_D,
"interactive|p" => \$::opt_p, "interactive|p" => \$::opt_p,
## How to unittest? tty skal emuleres ## How to unittest? tty skal emuleres
# xargs-compatibility - implemented, unittest - man missing # xargs-compatibility - unimplemented
#none
# xargs-compatability - unimplemented
"L=i" => \$::opt_L, "L=i" => \$::opt_L,
"max-lines|l:i" => \$::opt_l, "max-lines|l:i" => \$::opt_l,
## (echo a b;echo c) | xargs -l1 echo ## (echo a b;echo c) | xargs -l1 echo
## (echo a b' ';echo c) | xargs -l1 echo ## (echo a b' ';echo c) | xargs -l1 echo
"exit|x" => \$::opt_x, "exit|x" => \$::opt_x,
) || die_usage(); ) || die_usage();
$Global::debug = (defined $::opt_D); $Global::debug = (defined $::opt_D);
$Global::input_is_filename = (@ARGV); $Global::input_is_filename = (@ARGV);
if(defined $::opt_m) { $Global::xargs = 1; } if(defined $::opt_m) { $Global::xargs = 1; }
if(defined $::opt_X) { $Global::Xargs = 1; } if(defined $::opt_X) { $Global::Xargs = 1; }
if(defined $::opt_v) { $Global::verbose = 1; } if(defined $::opt_v) { $Global::verbose = 1; }
if(defined $::opt_silent) { $Global::verbose = 0; } if(defined $::opt_silent) { $Global::verbose = 0; }
if(defined $::opt_k) { $Global::keeporder = 1; } if(defined $::opt_k) { $Global::keeporder = 1; }
if(defined $::opt_g) { $Global::grouped = 1; } if(defined $::opt_g) { $Global::grouped = 1; }
if(defined $::opt_u) { $Global::grouped = 0; } if(defined $::opt_u) { $Global::grouped = 0; }
if(defined $::opt_c) { $Global::input_is_filename = 0; } if(defined $::opt_c) { $Global::input_is_filename = 0; }
if(defined $::opt_f) { $Global::input_is_filename = 1; } if(defined $::opt_f) { $Global::input_is_filename = 1; }
if(defined $::opt_0) { $/ = "\0"; } if(defined $::opt_0) { $/ = "\0"; }
if(defined $::opt_d) { my $e="sprintf \"$::opt_d\""; $/ = eval $e; } if(defined $::opt_d) { my $e="sprintf \"$::opt_d\""; $/ = eval $e; }
if(defined $::opt_p) { $Global::interactive = $::opt_p; } if(defined $::opt_p) { $Global::interactive = $::opt_p; }
if(defined $::opt_q) { $Global::quoting = 1; } if(defined $::opt_q) { $Global::quoting = 1; }
if(defined $::opt_r) { $Global::ignore_empty = 1; } if(defined $::opt_r) { $Global::ignore_empty = 1; }
if(defined $::opt_verbose) { $Global::stderr_verbose = 1; } if(defined $::opt_verbose) { $Global::stderr_verbose = 1; }
if(defined $::opt_I) { $Global::replacestring = $::opt_I; } if(defined $::opt_I) { $Global::replacestring = $::opt_I; }
if(defined $::opt_U) { $Global::replace_no_ext = $::opt_U; } if(defined $::opt_U) { $Global::replace_no_ext = $::opt_U; }
if(defined $::opt_i and $::opt_i) { $Global::replacestring = $::opt_i; } if(defined $::opt_i and $::opt_i) { $Global::replacestring = $::opt_i; }
if(defined $::opt_E and $::opt_E) { $Global::end_of_file_string = $::opt_E; } if(defined $::opt_E and $::opt_E) { $Global::end_of_file_string = $::opt_E; }
if(defined $::opt_n and $::opt_n) { $Global::max_number_of_args = $::opt_n; } if(defined $::opt_n and $::opt_n) { $Global::max_number_of_args = $::opt_n; }
if(defined $::opt_help) { die_usage(); } if(defined $::opt_help) { die_usage(); }
if(defined $::opt_number_of_cpus) { print no_of_cpus(),"\n"; exit(0); } if(defined $::opt_number_of_cpus) { print no_of_cpus(),"\n"; exit(0); }
if(defined $::opt_number_of_cores) { print no_of_cores(),"\n"; exit(0); } if(defined $::opt_number_of_cores) { print no_of_cores(),"\n"; exit(0); }
if(defined $::opt_version) { version(); exit(0); } if(defined $::opt_max_line_length_allowed) { print real_max_length(),"\n"; exit(0); }
if(defined $::opt_show_limits) { show_limits(); } if(defined $::opt_version) { version(); exit(0); }
if(defined $::opt_sshloginfile) { read_sshloginfile($::opt_sshloginfile); } if(defined $::opt_show_limits) { show_limits(); }
if(defined $::opt_sshloginfile) { read_sshloginfile($::opt_sshloginfile); }
if(defined $::opt_a) { if(defined $::opt_a) {
if(not open(ARGFILE,"<".$::opt_a)) { if(not open(ARGFILE,"<".$::opt_a)) {
print STDERR "$Global::progname: Cannot open input file `$::opt_a': No such file or directory\n"; print STDERR "$Global::progname: Cannot open input file `$::opt_a': No such file or directory\n";
exit(-1); exit(-1);
} }
$Global::argfile = *ARGFILE; $Global::argfile = *ARGFILE;
} }
if(@ARGV) { if(@ARGV) {
if($Global::quoting) { if($Global::quoting) {
$Global::command = join(" ", shell_quote(@ARGV)); $Global::command = join(" ", shell_quote(@ARGV));
} else { } else {
$Global::command = join(" ", @ARGV); $Global::command = join(" ", @ARGV);
} }
}
# Needs to be done after setting $Global::command and $Global::command_line_max_len
# as '-m' influences the number of commands that needs to be run
if(defined $::opt_P) { $Global::processes_to_run = compute_number_of_processes($::opt_P); }
$Global::job_end_sequence=1;
} }
# Needs to be done after setting $Global::command and $Global::command_line_max_len
# as '-m' influences the number of commands that needs to be run
if(defined $::opt_P) { $Global::processes_to_run = compute_number_of_processes($::opt_P); }
$Global::job_end_sequence=1;
parse_sshlogin();
init_run_jobs();
DoNotReap();
start_more_jobs();
ReapIfNeeded();
drain_job_queue();
# #
# Generating the command line # Generating the command line
@ -1373,9 +1382,6 @@ sub shell_quote {
my (@strings) = (@_); my (@strings) = (@_);
my $arg; my $arg;
for $arg (@strings) { for $arg (@strings) {
# what is the right thing to do about '-' at start of line?
# maybe substitute with './'
# so it is not regarded as -option.
$arg =~ s/\\/\\\\/g; $arg =~ s/\\/\\\\/g;
$arg =~ s/([\#\?\`\(\)\*\>\<\~\|\; \"\!\$\&\'])/\\$1/g; $arg =~ s/([\#\?\`\(\)\*\>\<\~\|\; \"\!\$\&\'])/\\$1/g;
@ -1587,6 +1593,7 @@ sub enough_file_handles {
sub user_requested_processes { sub user_requested_processes {
# Parse the number of processes that the user asked for # Parse the number of processes that the user asked for
my $opt_P = shift; my $opt_P = shift;
my $processes;
if(defined $opt_P) { if(defined $opt_P) {
if($opt_P =~ /^\+(\d+)$/) { if($opt_P =~ /^\+(\d+)$/) {
# E.g. -P +2 # E.g. -P +2
@ -1622,8 +1629,8 @@ sub no_of_cores {
sub no_of_cpus { sub no_of_cpus {
if(not $Global::no_of_cpus) { if(not $Global::no_of_cpus) {
my $no_of_cpus = (no_of_cpus_darwin() my $no_of_cpus = (no_of_cpus_gnu_linux()
|| no_of_cpus_gnu_linux() || no_of_cpus_darwin()
|| no_of_cpus_solaris()); || no_of_cpus_solaris());
if($no_of_cpus) { if($no_of_cpus) {
$Global::no_of_cpus = $no_of_cpus; $Global::no_of_cpus = $no_of_cpus;
@ -1746,7 +1753,7 @@ sub start_more_jobs {
# do { # do {
# $started_jobs_this_round = 0; # $started_jobs_this_round = 0;
# for slave in sshlogins { # for slave in sshlogins {
# if running_jobs{slave} < processed_to_run{$slave} { # if running_jobs{slave} < processes_to_run{$slave} {
# my $started += start_another_job($slave) # my $started += start_another_job($slave)
# $started_jobs_this_round += started # $started_jobs_this_round += started
# $jobs_started{$slave}++ # $jobs_started{$slave}++
@ -2110,3 +2117,27 @@ $main::opt_P = $main::opt_i = $main::opt_p = $main::opt_a =
$main::opt_version = $main::opt_L = $main::opt_l = $main::opt_version = $main::opt_L = $main::opt_l =
$main::opt_show_limits = $main::opt_n = $main::opt_e = $main::opt_verbose = $main::opt_show_limits = $main::opt_n = $main::opt_e = $main::opt_verbose =
$main::opt_E = $main::opt_r = $Global::xargs = $Global::keeporder = 0; $main::opt_E = $main::opt_r = $Global::xargs = $Global::keeporder = 0;
# Per host variables:
# Can depend on OS
#$Global::command_line_max_len =
# Can depend on processes_available_by_system_limit
#$Global::processes_to_run =
#
# $sshlogin, $ncpus
# $Global::running_jobs = 0;
# $Global::running{$pid}{'seq'} = printsequence
# $Global::running{$pid}{sshlogin} = server to run on
# $Global::host{sshlogin}{'no_of_running'} = number of currently running jobs
# $Global::host{sshlogin}{'ncpus'} = number of cpus
# $Global::host{sshlogin}{'maxlength'} = $Global::command_line_max_len
# $Global::host{sshlogin}{'max_no_of_running'} = number of currently running jobs
# $Global::running_jobs = sum $Global::host{sshlogin}{'no_of_running'}
# Hvordan udregnes system limits på remote systems hvis jeg ikke ved, hvormange
# argumenter, der er? Lav system limits lokalt og lad det være max
# Compress ide: identificer blokke af samme type (ascii binary picture
# sound). Flyt dem rundt, så de står ved siden af hinanden. Kompremer
# blokkene samlet med den mest egnede algoritme.
# Moved parse options to parse_options