tange/parallel
1
0
Fork 0
mirror of https://git.savannah.gnu.org/git/parallel.git synced 2024-11-25 23:47:53 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

parallel: eta time smoothing depending on pct complete

Browse source
This commit is contained in:
Ole Tange 2011-04-09 00:09:00 +02:00
parent 7b4f747877
commit 1478df33f7
3 changed files with 19 additions and 13 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
doc/FUTURE_IDEAS
View file

@ -1,5 +1,3 @@
Bug: parallel --colsep ')' echo {1}
job->start(): job->start():
$jobslot = Global::jobslot->$sshlogin $jobslot = Global::jobslot->$sshlogin

28
src/parallel
View file

@ -49,6 +49,10 @@ if(@ARGV) {
$Global::JobQueue = JobQueue->new( $Global::JobQueue = JobQueue->new(
$command,\@fhlist,$Global::Xargs,$number_of_args,\@Global::ret_files); $command,\@fhlist,$Global::Xargs,$number_of_args,\@Global::ret_files);
if($::opt_eta) {
# Count the number of jobs before starting any
$Global::JobQueue->total_jobs();
}
for my $sshlogin (values %Global::host) { for my $sshlogin (values %Global::host) {
$sshlogin->max_jobs_running(); $sshlogin->max_jobs_running();
} }
@ -1000,17 +1004,17 @@ sub progress {
my $completed = 0; my $completed = 0;
for(@workers) { $completed += $Global::host{$_}->jobs_completed() } for(@workers) { $completed += $Global::host{$_}->jobs_completed() }
if($completed) { if($completed) {
my $jobs_left = ($Global::JobQueue->total_jobs() - $completed); my $total = $Global::JobQueue->total_jobs();
my $avgtime = (time-$Private::first_completed)/$completed; my $left = $total - $completed;
my $pctcomplete = $completed / $total;
my $timepassed = (time - $Private::first_completed);
my $avgtime = $timepassed / $completed;
$Private::smoothed_avg_time ||= $avgtime; $Private::smoothed_avg_time ||= $avgtime;
$Private::smoothed_avg_time = 0.90 * $Private::smoothed_avg_time + 0.10 * $avgtime;
my $this_eta = $jobs_left * $Private::smoothed_avg_time;
#my $this_eta = ($Global::JobQueue->total_jobs() - $completed) * $avgtime;
#$Private::eta ||= $this_eta;
# Smooth the eta so it does not jump wildly # Smooth the eta so it does not jump wildly
#$Private::eta = 0.95 * $Private::eta + 0.05 * $this_eta; $Private::smoothed_avg_time = (1 - $pctcomplete) *
#$eta = sprintf("ETA: %ds $avgtime", $Private::eta); $Private::smoothed_avg_time + $pctcomplete * $avgtime;
$eta = sprintf("ETA: %ds %dleft %.2favg ", $this_eta, $jobs_left, $avgtime); my $this_eta = $left * $Private::smoothed_avg_time;
$eta = sprintf("ETA: %ds %dleft %.2favg ", $this_eta, $left, $avgtime);
} }
} }
@ -1282,7 +1286,10 @@ sub read_sshloginfile {
if($file eq "..") { if($file eq "..") {
$file = $ENV{'HOME'}."/.parallel/sshloginfile"; $file = $ENV{'HOME'}."/.parallel/sshloginfile";
} }
open(IN, $file) || ::die_bug("Cannot open $file"); if(not open(IN, $file)) {
print $Global::original_stderr "Cannot open $file\n";
exit(255);
}
while(<IN>) { while(<IN>) {
chomp; chomp;
/^\s*#/ and next; /^\s*#/ and next;
@ -2380,7 +2387,6 @@ sub new {
'unget' => \@unget, 'unget' => \@unget,
'commandlinequeue' => $commandlinequeue, 'commandlinequeue' => $commandlinequeue,
'total_jobs' => undef, 'total_jobs' => undef,
# 'next_seq' => 1,
}, ref($class) || $class; }, ref($class) || $class;
} }

2
testsuite/tests-to-run/test37.sh
View file

@ -31,6 +31,7 @@ PARALLEL='-k --jobs 1 -S '$SERVER1' perl -pe "\\$a=1; print\\$a"' parallel -v ::
echo '### Test ugly quoting from profile file' echo '### Test ugly quoting from profile file'
cat <<EOF >~/.parallel/test_profile cat <<EOF >~/.parallel/test_profile
# testprofile
-k --jobs 1 perl -pe '\$a=1; print \$a' -k --jobs 1 perl -pe '\$a=1; print \$a'
EOF EOF
parallel -v -J test_profile ::: <(echo a) <(echo b) parallel -v -J test_profile ::: <(echo a) <(echo b)
@ -58,6 +59,7 @@ PARALLEL='-k --jobs 1 -S ssh\ '$SERVER1'\ ssh\ parallel@'$SERVER2' perl -pe "\\$
echo '### Test quoting of space in long arguments (--sshlogin) from profile file' echo '### Test quoting of space in long arguments (--sshlogin) from profile file'
cat <<EOF >~/.parallel/test_profile cat <<EOF >~/.parallel/test_profile
# testprofile
-k --jobs 1 --sshlogin ssh\ $SERVER1\ ssh\ parallel@$SERVER2 perl -pe '\$a=1; print \$a' -k --jobs 1 --sshlogin ssh\ $SERVER1\ ssh\ parallel@$SERVER2 perl -pe '\$a=1; print \$a'
EOF EOF
parallel -v -J test_profile ::: /bin/gunzip parallel -v -J test_profile ::: /bin/gunzip