tange/parallel
1
0
Fork 0
mirror of https://git.savannah.gnu.org/git/parallel.git synced 2024-11-26 07:57:58 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

parallel: Code cleanup. Passes local testsuite.

Browse source
This commit is contained in:
Ole Tange 2013-05-11 12:53:17 +02:00
parent ff813d52ce
commit 5b0be9941c
4 changed files with 49 additions and 63 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

89
src/parallel
View file

@ -33,7 +33,6 @@ use Getopt::Long;
# Used to ensure code quality # Used to ensure code quality
use strict; use strict;
$::oodebug=0;
$SIG{TERM} ||= sub { exit 0; }; # $SIG{TERM} is not set on Mac OS X $SIG{TERM} ||= sub { exit 0; }; # $SIG{TERM} is not set on Mac OS X
if(not $ENV{SHELL}) { if(not $ENV{SHELL}) {
# $ENV{SHELL} is sometimes not set on Mac OS X and Windows # $ENV{SHELL} is sometimes not set on Mac OS X and Windows
@ -646,7 +645,7 @@ sub get_options_from_array {
sub parse_options { sub parse_options {
# Returns: N/A # Returns: N/A
# Defaults: # Defaults:
$Global::version = 20130429; $Global::version = 20130511;
$Global::progname = 'parallel'; $Global::progname = 'parallel';
$Global::infinity = 2**31; $Global::infinity = 2**31;
$Global::debug = 0; $Global::debug = 0;
@ -761,7 +760,12 @@ sub parse_options {
} }
if($opt::tollef and not $opt::gnu and not $opt::plain) { if($opt::tollef and not $opt::gnu and not $opt::plain) {
# Behave like tollef parallel (from moreutils) # Behave like tollef parallel (from moreutils)
::warning("YOU ARE USING --tollef. --tollef is obsolete and will be retired 20140222.\n"); if($Global::version > 20140222) {
::error("--tollef has been retired. See http://lists.gnu.org/archive/html/parallel/2013-02/msg00018.html\n");
::wait_and_exit(255);
}
::warning("YOU ARE USING --tollef. IF THINGS ARE ACTING WEIRD USE --gnu.\n");
::warning("--tollef is obsolete and will be retired 20140222.\n");
::warning("See: http://lists.gnu.org/archive/html/parallel/2013-02/msg00018.html\n"); ::warning("See: http://lists.gnu.org/archive/html/parallel/2013-02/msg00018.html\n");
$opt::u = 1; $opt::u = 1;
$Global::grouped = 0; $Global::grouped = 0;
@ -1304,40 +1308,41 @@ sub start_more_jobs {
} }
do { do {
$jobs_started_this_round = 0; $jobs_started_this_round = 0;
for my $sshlogin (values %Global::host) { # This will start 1 job on each --sshlogin (if possible)
debug("Running jobs before on ".$sshlogin->string().": ".$sshlogin->jobs_running()."\n"); # thus distribute the jobs on the --sshlogins round robin
if ($sshlogin->jobs_running() < $sshlogin->max_jobs_running()) { for my $sshlogin (values %Global::host) {
if($opt::load and $sshlogin->loadavg_too_high()) {
# The load is too high or unknown
next;
}
if($opt::noswap and $sshlogin->swapping()) {
# The server is swapping
next;
}
if($sshlogin->too_fast_remote_login()) {
next;
}
if($Global::JobQueue->empty() and not $opt::pipe) { if($Global::JobQueue->empty() and not $opt::pipe) {
last; last;
} }
debug($sshlogin->string()." has ".$sshlogin->jobs_running() debug("Running jobs before on ".$sshlogin->string().": ".$sshlogin->jobs_running()."\n");
. " out of " . $sshlogin->max_jobs_running() if ($sshlogin->jobs_running() < $sshlogin->max_jobs_running()) {
. " jobs running. Start another.\n"); if($opt::load and $sshlogin->loadavg_too_high()) {
if(start_another_job($sshlogin) == 0) { # The load is too high or unknown
# No more jobs to start on this $sshlogin next;
debug("No jobs started on ".$sshlogin->string()."\n"); }
next; if($opt::noswap and $sshlogin->swapping()) {
# The server is swapping
next;
}
if($sshlogin->too_fast_remote_login()) {
next;
}
debug($sshlogin->string()." has ".$sshlogin->jobs_running()
. " out of " . $sshlogin->max_jobs_running()
. " jobs running. Start another.\n");
if(start_another_job($sshlogin) == 0) {
# No more jobs to start on this $sshlogin
debug("No jobs started on ".$sshlogin->string()."\n");
next;
}
$sshlogin->inc_jobs_running();
$sshlogin->set_last_login_at(::now());
$jobs_started++;
$jobs_started_this_round++;
} }
debug("Job started on ".$sshlogin->string()."\n"); debug("Running jobs after on ".$sshlogin->string().": ".$sshlogin->jobs_running()
$sshlogin->inc_jobs_running(); ." of ".$sshlogin->max_jobs_running() ."\n");
$sshlogin->set_last_login_at(::now());
$jobs_started++;
$jobs_started_this_round++;
} }
debug("Running jobs after on ".$sshlogin->string().": ".$sshlogin->jobs_running()
." of ".$sshlogin->max_jobs_running() ."\n");
}
} while($jobs_started_this_round); } while($jobs_started_this_round);
return $jobs_started; return $jobs_started;
@ -1645,10 +1650,6 @@ sub get_job_with_sshlogin {
# next job object for $sshlogin if any available # next job object for $sshlogin if any available
my $sshlogin = shift; my $sshlogin = shift;
if($::oodebug and $Global::JobQueue->empty()) {
Carp::confess("get_job_with_sshlogin should never be called if empty");
}
my $job = $Global::JobQueue->get(); my $job = $Global::JobQueue->get();
if(not defined $job) { if(not defined $job) {
# No more jobs # No more jobs
@ -1656,9 +1657,6 @@ sub get_job_with_sshlogin {
return undef; return undef;
} }
if($::oodebug and not defined $job->{'commandline'}) {
Carp::confess("get_job_with_sshlogin job->commandline should never be empty");
}
my $clean_command = $job->replaced(); my $clean_command = $job->replaced();
if($clean_command =~ /^\s*$/) { if($clean_command =~ /^\s*$/) {
# Do not run empty lines # Do not run empty lines
@ -1673,7 +1671,6 @@ sub get_job_with_sshlogin {
$job->failed_here()) { $job->failed_here()) {
# This command with these args failed for this sshlogin # This command with these args failed for this sshlogin
my ($no_of_failed_sshlogins,$min_failures) = $job->min_failed(); my ($no_of_failed_sshlogins,$min_failures) = $job->min_failed();
#::my_dump(($no_of_failed_sshlogins,$min_failures));
if($no_of_failed_sshlogins == keys %Global::host and if($no_of_failed_sshlogins == keys %Global::host and
$job->failed_here() == $min_failures) { $job->failed_here() == $min_failures) {
# It failed the same or more times on another host: # It failed the same or more times on another host:
@ -4761,11 +4758,6 @@ sub replaced {
::shell_quote_scalar(::shell_quote_scalar($self->{'replaced'})); ::shell_quote_scalar(::shell_quote_scalar($self->{'replaced'}));
} }
} }
if($::oodebug and length($self->{'replaced'}) != ($self->len())) {
::my_dump($self);
Carp::cluck("replaced len=" . length($self->{'replaced'})
. " computed=" . ($self->len()));
}
return $self->{'replaced'}; return $self->{'replaced'};
} }
@ -5255,10 +5247,6 @@ sub new {
my $fhs = shift; my $fhs = shift;
for my $fh (@$fhs) { for my $fh (@$fhs) {
if(-t $fh) { if(-t $fh) {
if($opt::tollef and not $opt::gnu) {
::warning("YOU ARE USING --tollef. "
. "IF THINGS ARE ACTING WEIRD USE --gnu.");
}
::warning("Input is read from the terminal. ". ::warning("Input is read from the terminal. ".
"Only experts do this on purpose. ". "Only experts do this on purpose. ".
"Press CTRL-D to exit.\n"); "Press CTRL-D to exit.\n");
@ -5482,9 +5470,6 @@ package Arg;
sub new { sub new {
my $class = shift; my $class = shift;
my $orig = shift; my $orig = shift;
if($::oodebug and not defined $orig) {
Carp::cluck($orig);
}
return bless { return bless {
'orig' => $orig, 'orig' => $orig,
}, ref($class) || $class; }, ref($class) || $class;

4
testsuite/tests-to-run/parallel-local18.sh
View file

@ -36,10 +36,10 @@ parallel -S csh@localhost --env SPC echo 'a"$SPC"b' ::: 5
parallel -S tcsh@localhost --env SPC echo 'a"$SPC"b' ::: 5 parallel -S tcsh@localhost --env SPC echo 'a"$SPC"b' ::: 5
echo '### Test --env for \n and \\ - single and double - no output is good' echo '### Test --env for \n and \\ - single and double - no output is good'
perl -e 'for(10,92) { printf "%c%c %c%d\0",$_,$_,$_,$_ }' | stdout parallel --nice 19 -j4 -k -I // --arg-sep _ -0 V=// V2=V2=// parallel -k -j1 -S :,1/lo,1/tcsh@lo,1/csh@lo --env V,V2 echo \''"{}$V$V2"'\' ::: {#} {#} {#} {#} | uniq -c | grep -v ' 4 '|grep -v xauth |grep -v X11 perl -e 'for(10,92) { printf "%c%c %c%d\0",$_,$_,$_,$_ }' | stdout parallel --nice 19 -j4 -k -I // --arg-sep _ -0 V=// V2=V2=// parallel -k -j1 -S :,1/lo,1/tcsh@lo,1/csh@lo --env V,V2 echo \''"{}$V$V2"'\' ::: {#} {#} {#} {#} | sort | uniq -c | grep -v ' 4 '|grep -v xauth |grep -v X11
echo '### Test --env for \n and \\ - single and double --onall - no output is good' echo '### Test --env for \n and \\ - single and double --onall - no output is good'
perl -e 'for(10,92) { printf "%c%c %c%d\0",$_,$_,$_,$_ }' | stdout parallel --nice 19 -j4 -k -I // --arg-sep _ -0 V=// V2=V2=// parallel -k -j1 -S :,1/lo,1/tcsh@lo,1/csh@lo --onall --env V,V2 echo \''"{}$V$V2"'\' ::: {#} | uniq -c | grep -v ' 4 '|grep -v xauth |grep -v X11 perl -e 'for(10,92) { printf "%c%c %c%d\0",$_,$_,$_,$_ }' | stdout parallel --nice 19 -j4 -k -I // --arg-sep _ -0 V=// V2=V2=// parallel -k -j1 -S :,1/lo,1/tcsh@lo,1/csh@lo --onall --env V,V2 echo \''"{}$V$V2"'\' ::: {#} | sort | uniq -c | grep -v ' 4 '|grep -v xauth |grep -v X11
echo '### Test --env for \160 - which kills csh - single and double - no output is good' echo '### Test --env for \160 - which kills csh - single and double - no output is good'
perl -e 'for(160) { printf "%c%c %c%d\0",$_,$_,$_,$_ }' | stdout parallel --nice 19 -j4 -k -I // --arg-sep _ -0 V=// V2=V2=// parallel -k -j1 -S :,1/lo,1/tcsh@lo --env V,V2 echo \''"{}$V$V2"'\' ::: {#} {#} {#} | uniq -c | grep -v ' 3 '|grep -v xauth |grep -v X11 perl -e 'for(160) { printf "%c%c %c%d\0",$_,$_,$_,$_ }' | stdout parallel --nice 19 -j4 -k -I // --arg-sep _ -0 V=// V2=V2=// parallel -k -j1 -S :,1/lo,1/tcsh@lo --env V,V2 echo \''"{}$V$V2"'\' ::: {#} {#} {#} | uniq -c | grep -v ' 3 '|grep -v xauth |grep -v X11

3
testsuite/wanted-results/parallel-local164
View file

@ -91,8 +91,9 @@ one 1
a a
b b
c c
parallel: Warning: --tollef is obsolete and will be retired 20140222.
parallel: Warning: See: http://lists.gnu.org/archive/html/parallel/2013-02/msg00018.html parallel: Warning: See: http://lists.gnu.org/archive/html/parallel/2013-02/msg00018.html
parallel: Warning: YOU ARE USING --tollef. --tollef is obsolete and will be retired 20140222. parallel: Warning: YOU ARE USING --tollef. IF THINGS ARE ACTING WEIRD USE --gnu.
### Test --tollef --gnu ### Test --tollef --gnu
1 1
2 2

16
testsuite/wanted-results/parallel-local18
View file

@ -25,18 +25,18 @@ a' * ? >o <i*? ][\!#¤%=( ) | }b 5
a' * ? >o <i*? ][\!#¤%=( ) | }b 5 a' * ? >o <i*? ][\!#¤%=( ) | }b 5
a' * ? >o <i*? ][\!#¤%=( ) | }b 5 a' * ? >o <i*? ][\!#¤%=( ) | }b 5
### Test --env for \n and \\ - single and double - no output is good ### Test --env for \n and \\ - single and double - no output is good
2 setenv: Too many arguments.
1 export: Command not found.
1 V: Undefined variable.
2 1 10V2= 10 2 1 10V2= 10
3 2\\ \92V2=\\ \92
1 2\ \92V2=\ \92 1 2\ \92V2=\ \92
3 2\\ \92V2=\\ \92
1 V: Undefined variable.
1 export: Command not found.
2 setenv: Too many arguments.
### Test --env for \n and \\ - single and double --onall - no output is good ### Test --env for \n and \\ - single and double --onall - no output is good
2 setenv: Too many arguments.
1 export: Command not found.
1 V: Undefined variable.
2 1 10V2= 10 2 1 10V2= 10
3 2\\ \92V2=\\ \92
1 2\ \92V2=\ \92 1 2\ \92V2=\ \92
3 2\\ \92V2=\\ \92
1 V: Undefined variable.
1 export: Command not found.
2 setenv: Too many arguments.
### Test --env for \160 - which kills csh - single and double - no output is good ### Test --env for \160 - which kills csh - single and double - no output is good
### Test --env for \160 - which kills csh - single and double --onall - no output is good ### Test --env for \160 - which kills csh - single and double --onall - no output is good