From ab899c36a6b1c4182fa64a282f9f054a2466a947 Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Thu, 28 Jul 2011 21:09:27 +0200 Subject: [PATCH] parallel: fixed with test case bug #33623: --load doesn't parse percentage or filename arguments. parallel: Empty job caused not being run in parallel. --- src/parallel | 58 +++++++++++++++++++--------- testsuite/tests-to-run/niceload01.sh | 8 ++-- testsuite/tests-to-run/test45.sh | 19 +++++++-- testsuite/tests-to-run/test64.sh | 22 +++++++++++ testsuite/wanted-results/niceload01 | 5 ++- testsuite/wanted-results/test45 | 14 +++++-- testsuite/wanted-results/test64 | 5 +++ 7 files changed, 99 insertions(+), 32 deletions(-) create mode 100644 testsuite/tests-to-run/test64.sh create mode 100644 testsuite/wanted-results/test64 diff --git a/src/parallel b/src/parallel index 98d285e3..59094f7d 100755 --- a/src/parallel +++ b/src/parallel @@ -356,7 +356,7 @@ sub options_hash { "basenameextensionreplace|bner=s" => \$::opt_basenameextensionreplace, "seqreplace=s" => \$::opt_seqreplace, "jobs|j=s" => \$::opt_P, - "load=f" => \$::opt_load, + "load=s" => \$::opt_load, "noswap" => \$::opt_noswap, "max-line-length-allowed" => \$::opt_max_line_length_allowed, "number-of-cpus" => \$::opt_number_of_cpus, @@ -1016,7 +1016,9 @@ sub drain_job_queue { my $sleep = 0.2; do { while($Global::total_running > 0) { - debug("jobs running: ",$Global::total_running," ",scalar keys %Global::running, " Memory usage:".my_memory_usage()."\n"); + debug("jobs running: ", $Global::total_running, "==", scalar + keys %Global::running," slots: ", $Global::max_jobs_running, + " Memory usage:".my_memory_usage()." "); $sleep = ($sleep < 1000) ? ($sleep * 1.1) : ($sleep); usleep($sleep); do_not_reap(); @@ -1262,6 +1264,16 @@ sub start_more_jobs { } } } + if($Global::max_load_file) { + my $mtime = (stat($Global::max_load_file))[9]; + if($mtime > $Global::max_load_file_last_mod) { + $Global::max_load_file_last_mod = $mtime; + for my $sshlogin (values %Global::host) { + $sshlogin->set_max_loadavg(undef); + } + } + } + for my $sshlogin (values %Global::host) { debug("Running jobs before on ".$sshlogin->string().": ".$sshlogin->jobs_running()."\n"); if($::opt_load and $sshlogin->loadavg_too_high()) { @@ -1276,10 +1288,12 @@ sub start_more_jobs { if($Global::JobQueue->empty() and not $::opt_pipe) { last; } - debug("Try starting a job on ".$sshlogin->string()."\n"); + debug($sshlogin->string()." has ".$sshlogin->jobs_running() + . " out of " . $sshlogin->max_jobs_running() + . " jobs running. Start another.\n"); if(start_another_job($sshlogin) == 0) { # No more jobs to start on this $sshlogin - debug("Empty after retry: ",$Global::JobQueue->empty(),"\n"); + debug("No jobs started on ".$sshlogin->string()."\n"); last; } debug("Job started on ".$sshlogin->string()."\n"); @@ -1304,11 +1318,13 @@ sub start_another_job { if(enough_file_handles()) { if($Global::JobQueue->empty() and not $::opt_pipe) { # No more commands to run - return 0; + debug("Not starting: JobQueue empty\n"); + return 0; } else { my $job = get_job_with_sshlogin($sshlogin); if(not defined $job) { # No command available for that sshlogin + debug("Not starting: no jobs available for ".$sshlogin->string()."\n"); return 0; } debug("Command to run on '".$job->sshlogin()."': '".$job->replaced()."'\n"); @@ -1323,6 +1339,7 @@ sub start_another_job { } } else { # No more file handles + debug("Not starting: no more file handles\n"); return 0; } } @@ -1342,6 +1359,7 @@ sub get_job_with_sshlogin { my $job = $Global::JobQueue->get(); if(not defined $job) { # No more jobs + ::debug("No more jobs: JobQueue empty\n"); return undef; } @@ -1557,7 +1575,7 @@ sub reaper { $Private::reaperlevel++; my $stiff; my $children_reaped = 0; - debug("Reaper called $Private::reaperlevel\n"); + debug("Reaper called $Private::reaperlevel "); while (($stiff = waitpid(-1, &WNOHANG)) > 0) { $children_reaped++; if($Global::sshmaster{$stiff}) { @@ -2019,6 +2037,11 @@ sub max_loadavg { return $self->{'max_loadavg'}; } +sub set_max_loadavg { + my $self = shift; + $self->{'max_loadavg'} = shift; +} + sub compute_max_loadavg { # Parse the max loadaverage that the user asked for using --load # Returns: @@ -2043,19 +2066,13 @@ sub compute_max_loadavg { $self->ncpus() * $j / 100; } elsif ($loadspec =~ /^(\d+(\.\d+)?)$/) { $load = $1; - if($load == 0) { - # --load 0 = infinity (or at least close) - $load = 2**31; - } } elsif (-f $loadspec) { - # TODO this needs to be done for $loadspec - ::die_bug("loadspec-unimplemented"); - $Global::max_procs_file = $loadspec; - $Global::max_procs_file_last_mod = (stat($Global::max_procs_file))[9]; - if(open(IN, $Global::max_procs_file)) { - my $opt_P_file = join("",); + $Global::max_load_file = $loadspec; + $Global::max_load_file_last_mod = (stat($Global::max_load_file))[9]; + if(open(IN, $Global::max_load_file)) { + my $opt_load_file = join("",); close IN; - $load = $self->compute_max_loadavg($opt_P_file); + $load = $self->compute_max_loadavg($opt_load_file); } else { print $Global::original_stderr "Cannot open $loadspec\n"; exit(255); @@ -3920,12 +3937,15 @@ sub get { ); $cmd_line->populate(); ::debug("cmd_line->number_of_args ".$cmd_line->number_of_args()."\n"); - if(not $::opt_pipe and ($cmd_line->number_of_args() == 0 or $cmd_line->replaced() eq "")) { + if(not $::opt_pipe and $cmd_line->number_of_args() == 0) { # We did not get more args - maybe at EOF string? return undef; + } elsif($cmd_line->replaced() eq "") { + # Empty command - get the next instead + return $self->get(); } else { $self->set_seq($self->seq()+1); - return ($cmd_line); + return $cmd_line; } } } diff --git a/testsuite/tests-to-run/niceload01.sh b/testsuite/tests-to-run/niceload01.sh index 744cc14d..96c45359 100644 --- a/testsuite/tests-to-run/niceload01.sh +++ b/testsuite/tests-to-run/niceload01.sh @@ -1,7 +1,7 @@ #!/bin/bash -echo '### Test niceload' -niceload -q -s 1 perl -e '$|=1;do{$l==$r or print "."; $l=$r}until(($r=time-$^T)>10)' +echo '### Test niceload -q' +niceload -q perl -e '$a = "works";$b="This $a\n"; print($b);' echo # Force less than 1 GB buffer+cache @@ -45,8 +45,8 @@ wait while uptime | grep -v age:.[1-9][0-9].[0-9][0-9] >/dev/null ; do (timeout 5 nice burnP6 2>/dev/null &) done echo '### -H and --hard' -niceload -H -l 9.9 uptime | grep '[1-9][0-9]\.[0-9][0-9],' || echo OK -niceload --hard -l 9 uptime | grep '[1-9][0-9]\.[0-9][0-9],' || echo OK +niceload -H -l 9.9 uptime | grep ':.[1-9][0-9].[0-9][0-9],' || echo OK +niceload --hard -l 9 uptime | grep ':.[1-9][0-9].[0-9][0-9],' || echo OK echo '### -f and --factor' niceload -H -f 0.1 -l6 echo f 0.1 first & diff --git a/testsuite/tests-to-run/test45.sh b/testsuite/tests-to-run/test45.sh index e32993cd..8e424e0b 100644 --- a/testsuite/tests-to-run/test45.sh +++ b/testsuite/tests-to-run/test45.sh @@ -5,9 +5,6 @@ SERVER2=parallel-server2 # -L1 will join lines ending in ' ' cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | parallel -j0 -k -L1 -echo '### Bug in --load'; - parallel -k --load 30 sleep 0.1\;echo ::: 1 2 3 - echo '### Test --load locally'; echo '# This will force the loadavg > 10'; while uptime | grep -v age:.[1-9][0-9].[0-9][0-9] >/dev/null ; do (timeout 5 nice burnP6 2>/dev/null &) done; @@ -16,4 +13,20 @@ echo '### Test --load locally'; echo '### Test --load remote'; ssh parallel@$SERVER2 "while uptime | grep -v age:.[1-9][0-9].[0-9][0-9] >/dev/null ; do (timeout 5 nice burnP6 2>/dev/null &) done"; stdout /usr/bin/time -f %e parallel -S parallel@$SERVER2 --load 10 sleep ::: 1 | perl -ne '$_ > 10 and print "OK\n"' + +echo '### Test --load read from a file - more than 3s'; + echo '# This will force the loadavg > 10'; + while uptime | grep -v age:.[1-9][0-9].[0-9][0-9] >/dev/null ; do (timeout 5 nice burnP6 2>/dev/null &) done; + ( echo 8 > /tmp/parallel_load_file; sleep 3; echo 100 > /tmp/parallel_load_file ) & + stdout /usr/bin/time -f %e parallel --load /tmp/parallel_load_file sleep ::: 1 | perl -ne '$_ > 3 and print "OK\n"' + +echo '### Test --load read from a file - less than 10s'; + echo '# This will force the loadavg > 10'; + while uptime | grep -v age:.[1-9][0-9].[0-9][0-9] >/dev/null ; do (timeout 5 nice burnP6 2>/dev/null &) done; + ( echo 8 > /tmp/parallel_load_file2; sleep 3; echo 100 > /tmp/parallel_load_file2 ) & + stdout /usr/bin/time -f %e parallel --load /tmp/parallel_load_file2 sleep ::: 1 | perl -ne '$_ < 10 and print "OK\n"' + +echo '### Bug in --load'; + parallel -k --load 30 sleep 0.1\;echo ::: 1 2 3 + EOF diff --git a/testsuite/tests-to-run/test64.sh b/testsuite/tests-to-run/test64.sh new file mode 100644 index 00000000..cd7401fd --- /dev/null +++ b/testsuite/tests-to-run/test64.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +SERVER1=parallel-server3 +SERVER2=parallel-server2 + +# -L1 will join lines ending in ' ' +cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | parallel -j10 -k -L1 +echo '### Test -k 5'; + sleep 5 + +echo '### Test -k 3'; + sleep 3 + +echo '### Test -k 4'; + sleep 4 + +echo '### Test -k 2'; + sleep 2 + +echo '### Test -k 1'; + sleep 1 +EOF diff --git a/testsuite/wanted-results/niceload01 b/testsuite/wanted-results/niceload01 index 692e89bb..48e374b2 100644 --- a/testsuite/wanted-results/niceload01 +++ b/testsuite/wanted-results/niceload01 @@ -1,5 +1,6 @@ -### Test niceload -.......... +### Test niceload -q +This works + ### --rm and --runmem OK OK diff --git a/testsuite/wanted-results/test45 b/testsuite/wanted-results/test45 index c834f875..1ee77480 100644 --- a/testsuite/wanted-results/test45 +++ b/testsuite/wanted-results/test45 @@ -1,9 +1,15 @@ -### Bug in --load -1 -2 -3 ### Test --load locally # This will force the loadavg > 10 OK ### Test --load remote OK +### Test --load read from a file - more than 3s +# This will force the loadavg > 10 +OK +### Test --load read from a file - less than 10s +# This will force the loadavg > 10 +OK +### Bug in --load +1 +2 +3 diff --git a/testsuite/wanted-results/test64 b/testsuite/wanted-results/test64 new file mode 100644 index 00000000..885fcaca --- /dev/null +++ b/testsuite/wanted-results/test64 @@ -0,0 +1,5 @@ +### Test -k 5 +### Test -k 3 +### Test -k 4 +### Test -k 2 +### Test -k 1