tange/parallel
1
0
Fork 0
mirror of https://git.savannah.gnu.org/git/parallel.git synced 2024-10-18 04:49:40 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

parallel: --progress bug: Short jobs had first status before the header.

Browse source
This commit is contained in:
Ole Tange 2023-12-13 01:02:19 +01:00
parent 0553dbd55c
commit 2e352aa51f
6 changed files with 192 additions and 182 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

330
src/parallel
View file

@ -4272,9 +4272,11 @@ sub init_progress() {
if($opt::bar) { if($opt::bar) {
return("",""); return("","");
} }
my %progress = progress(); my $progress = progress();
return ("\nComputers / CPU cores / Max jobs to run\n", my $cpu_units = $opt::use_sockets_instead_of_threads ? "CPU sockets" :
$progress{'workerlist'}); ($opt::use_cores_instead_of_threads ? "CPU cores" : "CPU threads");
return ("\nComputers / $cpu_units / Max jobs to run\n",
$progress->{'workerlist'},"\n",$progress->{'header'});
} }
sub drain_job_queue(@) { sub drain_job_queue(@) {
@ -4288,10 +4290,6 @@ sub drain_job_queue(@) {
# $Global::start_no_new_jobs # $Global::start_no_new_jobs
# Returns: N/A # Returns: N/A
my @command = @_; my @command = @_;
if($opt::progress) {
::status_no_nl(init_progress());
}
my $last_header = "";
my $sleep = 0.2; my $sleep = 0.2;
my $sleepsum = 0; my $sleepsum = 0;
do { do {
@ -4306,12 +4304,8 @@ sub drain_job_queue(@) {
} }
} }
if($opt::progress) { if($opt::progress) {
my %progress = progress(); my $progress = progress();
if($last_header ne $progress{'header'}) { ::status_no_nl("\r",$progress->{'status'});
::status("", $progress{'header'});
$last_header = $progress{'header'};
}
::status_no_nl("\r",$progress{'status'});
} }
if($Global::total_running < $Global::max_jobs_running if($Global::total_running < $Global::max_jobs_running
and not $Global::JobQueue->empty()) { and not $Global::JobQueue->empty()) {
@ -4373,8 +4367,8 @@ sub drain_job_queue(@) {
$opt::sqlmaster and not $Global::sql->finished()); $opt::sqlmaster and not $Global::sql->finished());
$Global::all_jobs_done = 1; $Global::all_jobs_done = 1;
if($opt::progress) { if($opt::progress) {
my %progress = progress(); my $progress = progress();
::status("\r".$progress{'status'}); ::status("\r".$progress->{'status'});
} }
} }
@ -4389,165 +4383,148 @@ sub toggle_progress() {
} }
} }
sub progress() { {
# Uses: my $last_header;
# $opt::bar my $eol;
# $opt::eta
# %Global::host sub progress() {
# $Global::total_started # Uses:
# Returns: # $opt::bar
# $workerlist = list of workers # $opt::eta
# $header = that will fit on the screen # %Global::host
# $status = message that will fit on the screen # $Global::total_started
if($opt::bar) { # Returns:
return ("workerlist" => "", "header" => "", "status" => bar()); # $workerlist = list of workers
# $header = that will fit on the screen
# $status = message that will fit on the screen
if($opt::bar) {
return {"workerlist" => "", "header" => "", "status" => bar()};
}
my $eta = "";
my ($status,$header)=("","");
if($opt::eta) {
my($total, $completed, $left, $pctcomplete, $avgtime, $this_eta) =
compute_eta();
$eta = sprintf("ETA: %ds Left: %d AVG: %.2fs ",
$this_eta, $left, $avgtime);
}
my $termcols = terminal_columns();
my @workers = sort keys %Global::host;
my $workerno = 1;
my %wrk;
for my $w (@workers) {
my %i;
$i{'sshlogin'} = $w eq ":" ? "local" : $w;
$i{'no'} = $workerno++;
$i{'ncpu'} = ($Global::host{$w}->ncpus() || "-");
$i{'jobslots'} = $Global::host{$w}->max_jobs_running();
$i{'completed'} = ($Global::host{$w}->jobs_completed() || 0);
$i{'running'} = $Global::host{$w}->jobs_running();
$i{'pct'} = $Global::total_started ?
(($i{'running'}+$i{'completed'})*100 /
$Global::total_started) : 0;
$i{'time'} = $i{'completed'} ? (time-$^T)/($i{'completed'}) : 0;
$wrk{$w} = \%i;
}
my $workerlist = "";
for my $w (@workers) {
$workerlist .=
$wrk{$w}{'no'}.":".$wrk{$w}{'sshlogin'} ." / ".
$wrk{$w}{'ncpu'}." / ".
$wrk{$w}{'jobslots'}."\n";
}
# Force $status to select one of the below formats
$status = "c"x($termcols+1);
# Select an output format that will fit on a single line
if(length $status > $termcols) {
# sshlogin1:XX/XX/XX%/XX.Xs s2:XX/XX/XX%/XX.Xs s3:XX/XX/XX%/XX.Xs
$header = "Computer:jobs running/jobs completed/".
"%of started jobs/Average seconds to complete";
$status = $eta . join(" ",map {
sprintf("%s:%d/%d/%d%%/%.1fs ",
@{$wrk{$_}}
{'sshlogin','running','completed','pct','time'}
); } @workers);
}
if(length $status > $termcols) {
# 1:XX/XX/XX%/X.Xs 2:XX/XX/XX%/X.Xs 3:XX/XX/XX%/X.Xs
$header = "Computer:jobs running/jobs completed/%of started jobs";
$status = $eta . join(" ",map {
sprintf("%s:%d/%d/%d%%/%.1fs ",
@{$wrk{$_}}
{'no','running','completed','pct','time'}
); } @workers);
}
if(length $status > $termcols) {
# sshlogin1:XX/XX/XX% sshlogin2:XX/XX/XX% sshlogin3:XX/XX/XX%
$header = "Computer:jobs running/jobs completed/%of started jobs";
$status = $eta . join(" ",map {
sprintf("%s:%d/%d/%d%%",
@{$wrk{$_}}
{'sshlogin','running','completed','pct'}
); } @workers);
}
if(length $status > $termcols) {
# 1:XX/XX/XX% 2:XX/XX/XX% 3:XX/XX/XX% 4:XX/XX/XX% 5:XX/XX/XX%
$header = "Computer:jobs running/jobs completed/%of started jobs";
$status = $eta . join(" ",map {
sprintf("%s:%d/%d/%d%%",
@{$wrk{$_}}
{'no','running','completed','pct'}
); } @workers);
}
if(length $status > $termcols) {
# sshlogin1:XX/XX sshlogin2:XX/XX sshlogin3:XX/XX
$header = "Computer:jobs running/jobs completed";
$status = $eta . join(" ", map {
sprintf("%s:%d/%d",
@{$wrk{$_}}
{'sshlogin','running','completed'}
); } @workers);
}
if(length $status > $termcols) {
# 1:XX/XX 2:XX/XX 3:XX/XX 4:XX/XX 5:XX/XX 6:XX/XX
$header = "Computer:jobs running/jobs completed";
$status = $eta . join(" ", map {
sprintf("%s:%d/%d",
@{$wrk{$_}}
{'no','running','completed'}
); } @workers);
}
if(length $status > $termcols) {
# sshlogin1:XX sshlogin2:XX sshlogin3:XX sshlogin4:XX sshlogin5:XX
$header = "Computer:jobs completed";
$status = $eta . join(" ", map {
sprintf("%s:%d",
@{$wrk{$_}}
{'sshlogin','completed'}
); } @workers);
}
if(length $status > $termcols) {
# 1:XX 2:XX 3:XX 4:XX 5:XX 6:XX
$header = "Computer:jobs completed";
$status = $eta . join(" ", map {
sprintf("%s:%d",
@{$wrk{$_}}
{'no','completed'}
); } @workers);
}
if($last_header ne $header) {
$header .= "\n";
$last_header = $header;
} else {
$header = "";
}
if(not $eol) {
$eol = `sh -c "tput el </dev/tty" 2>/dev/null`;
chomp($eol);
if($eol eq "") { $eol = "\033[K"; }
}
return {"workerlist" => $workerlist, "header" => $header,
"status" => $status.$eol};
} }
my $eta = "";
my ($status,$header)=("","");
if($opt::eta) {
my($total, $completed, $left, $pctcomplete, $avgtime, $this_eta) =
compute_eta();
$eta = sprintf("ETA: %ds Left: %d AVG: %.2fs ",
$this_eta, $left, $avgtime);
}
my $termcols = terminal_columns();
my @workers = sort keys %Global::host;
my %sshlogin = map { $_ eq ":" ? ($_ => "local") : ($_ => $_) } @workers;
my $workerno = 1;
my %workerno = map { ($_=>$workerno++) } @workers;
my $workerlist = "";
for my $w (@workers) {
$workerlist .=
$workerno{$w}.":".$sshlogin{$w} ." / ".
($Global::host{$w}->ncpus() || "-")." / ".
$Global::host{$w}->max_jobs_running()."\n";
}
$status = "c"x($termcols+1);
# Select an output format that will fit on a single line
if(length $status > $termcols) {
# sshlogin1:XX/XX/XX%/XX.Xs s2:XX/XX/XX%/XX.Xs s3:XX/XX/XX%/XX.Xs
$header = "Computer:jobs running/jobs completed/".
"%of started jobs/Average seconds to complete";
$status = $eta .
join(" ",map
{
if($Global::total_started) {
my $completed =
($Global::host{$_}->jobs_completed()||0);
my $running = $Global::host{$_}->jobs_running();
my $time = $completed ? (time-$^T)/($completed) : "0";
sprintf("%s:%d/%d/%d%%/%.1fs ",
$sshlogin{$_}, $running, $completed,
($running+$completed)*100
/ $Global::total_started, $time);
}
} @workers);
}
if(length $status > $termcols) {
# 1:XX/XX/XX%/X.Xs 2:XX/XX/XX%/X.Xs 3:XX/XX/XX%/X.Xs 4:XX/XX/XX%/X.Xs
$header = "Computer:jobs running/jobs completed/%of started jobs";
$status = $eta .
join(" ",map
{
if($Global::total_started) {
my $completed =
($Global::host{$_}->jobs_completed()||0);
my $running = $Global::host{$_}->jobs_running();
my $time = $completed ? (time-$^T)/($completed) : "0";
sprintf("%s:%d/%d/%d%%/%.1fs ",
$workerno{$_}, $running, $completed,
($running+$completed)*100
/ $Global::total_started, $time);
}
} @workers);
}
if(length $status > $termcols) {
# sshlogin1:XX/XX/XX% sshlogin2:XX/XX/XX% sshlogin3:XX/XX/XX%
$header = "Computer:jobs running/jobs completed/%of started jobs";
$status = $eta .
join(" ",map
{
if($Global::total_started) {
sprintf("%s:%d/%d/%d%%",
$sshlogin{$_},
$Global::host{$_}->jobs_running(),
($Global::host{$_}->jobs_completed()||0),
($Global::host{$_}->jobs_running()+
($Global::host{$_}->jobs_completed()||0))*100
/ $Global::total_started)
}
}
@workers);
}
if(length $status > $termcols) {
# 1:XX/XX/XX% 2:XX/XX/XX% 3:XX/XX/XX% 4:XX/XX/XX% 5:XX/XX/XX%
$header = "Computer:jobs running/jobs completed/%of started jobs";
$status = $eta .
join(" ",map
{
if($Global::total_started) {
sprintf("%s:%d/%d/%d%%",
$workerno{$_},
$Global::host{$_}->jobs_running(),
($Global::host{$_}->jobs_completed()||0),
($Global::host{$_}->jobs_running()+
($Global::host{$_}->jobs_completed()||0))*100
/ $Global::total_started)
}
}
@workers);
}
if(length $status > $termcols) {
# sshlogin1:XX/XX/XX% sshlogin2:XX/XX/XX% sshlogin3:XX/XX
$header = "Computer:jobs running/jobs completed";
$status = $eta .
join(" ",
map { sprintf("%s:%d/%d",
$sshlogin{$_}, $Global::host{$_}->jobs_running(),
($Global::host{$_}->jobs_completed()||0)) }
@workers);
}
if(length $status > $termcols) {
# sshlogin1:XX/XX sshlogin2:XX/XX sshlogin3:XX/XX sshlogin4:XX/XX
$header = "Computer:jobs running/jobs completed";
$status = $eta .
join(" ",
map { sprintf("%s:%d/%d",
$sshlogin{$_}, $Global::host{$_}->jobs_running(),
($Global::host{$_}->jobs_completed()||0)) }
@workers);
}
if(length $status > $termcols) {
# 1:XX/XX 2:XX/XX 3:XX/XX 4:XX/XX 5:XX/XX 6:XX/XX
$header = "Computer:jobs running/jobs completed";
$status = $eta .
join(" ",
map { sprintf("%s:%d/%d", $workerno{$_},
$Global::host{$_}->jobs_running(),
($Global::host{$_}->jobs_completed()||0)) }
@workers);
}
if(length $status > $termcols) {
# sshlogin1:XX sshlogin2:XX sshlogin3:XX sshlogin4:XX sshlogin5:XX
$header = "Computer:jobs completed";
$status = $eta .
join(" ",
map { sprintf("%s:%d", $sshlogin{$_},
($Global::host{$_}->jobs_completed()||0)) }
@workers);
}
if(length $status > $termcols) {
# 1:XX 2:XX 3:XX 4:XX 5:XX 6:XX
$header = "Computer:jobs completed";
$status = $eta .
join(" ",
map { sprintf("%s:%d",
$workerno{$_},
($Global::host{$_}->jobs_completed()||0)) }
@workers);
}
return ("workerlist" => $workerlist, "header" => $header,
"status" => $status);
} }
{ {
@ -5691,8 +5668,8 @@ sub reaper() {
$job->cleanup(); $job->cleanup();
if($opt::progress) { if($opt::progress) {
my %progress = progress(); my $progress = progress();
::status_no_nl("\r",$progress{'status'}); ::status_no_nl("\r",$progress->{'status'});
} }
debug("run", "jobdone \n"); debug("run", "jobdone \n");
@ -15426,6 +15403,9 @@ sub main() {
$SIG{TERM} = $Global::original_sig{TERM}; $SIG{TERM} = $Global::original_sig{TERM};
$SIG{HUP} = \&start_no_new_jobs; $SIG{HUP} = \&start_no_new_jobs;
if($opt::progress) {
::status_no_nl(init_progress());
}
if($opt::tee or $opt::shard or $opt::bin) { if($opt::tee or $opt::shard or $opt::bin) {
# All jobs must be running in parallel for --tee/--shard/--bin # All jobs must be running in parallel for --tee/--shard/--bin
while(start_more_jobs()) {} while(start_more_jobs()) {}

8
testsuite/tests-to-run/parallel-local-1s.sh
View file

@ -8,6 +8,14 @@
# Each should be taking 1-3s and be possible to run in parallel # Each should be taking 1-3s and be possible to run in parallel
# I.e.: No race conditions, no logins # I.e.: No race conditions, no logins
par_progress() {
(
parallel --progress --use-sockets-instead-of-threads true ::: a b c
parallel --progress --use-cores-instead-of-threads true ::: a b c
parallel --progress --use-cpus-instead-of-cores true ::: a b c
) 2>&1 | perl -pe 's/.*\r//; s/\d.\ds/9.9s/'
}
par_citation_no_config_dir() { par_citation_no_config_dir() {
echo '### bug #64329: parallel --citation will loop forever unless the config dir exists' echo '### bug #64329: parallel --citation will loop forever unless the config dir exists'
t=$(mktemp -d) t=$(mktemp -d)

2
testsuite/wanted-results/parallel-local-0.3s
View file

@ -18,7 +18,7 @@ par_PARALLEL_HOME_with_+ bug #59453: PARALLEL_HOME with plus sign causes error:
par_PARALLEL_HOME_with_+ parallel: Warning: $PARALLEL_HOME can only contain [-a-z0-9_+,.%:/= ]. par_PARALLEL_HOME_with_+ parallel: Warning: $PARALLEL_HOME can only contain [-a-z0-9_+,.%:/= ].
par_PARALLEL_HOME_with_+ Parallel_home_with+ par_PARALLEL_HOME_with_+ Parallel_home_with+
par_X_eta_div_zero ### bug #34422: parallel -X --eta crashes with div by zero par_X_eta_div_zero ### bug #34422: parallel -X --eta crashes with div by zero
par_X_eta_div_zero Computers / CPU cores / Max jobs to run par_X_eta_div_zero Computers / CPU threads / Max jobs to run
par_X_eta_div_zero 0:local / 0 / 0 par_X_eta_div_zero 0:local / 0 / 0
par_append_joblog ### can you append to a joblog using + par_append_joblog ### can you append to a joblog using +
par_append_joblog 1 par_append_joblog 1

18
testsuite/wanted-results/parallel-local-1s
View file

@ -699,6 +699,24 @@ par_profiles_with_space /bin/bash=/bin/bash
par_profiles_with_space echo '/bin/bash=/bin/bash' par_profiles_with_space echo '/bin/bash=/bin/bash'
par_profiles_with_space /bin/bash=/bin/bash par_profiles_with_space /bin/bash=/bin/bash
par_profiles_with_space With script in $PARALLEL /bin/bash=/TMP par_profiles_with_space With script in $PARALLEL /bin/bash=/TMP
par_progress
par_progress Computers / CPU sockets / Max jobs to run
par_progress 1:local / 1 / 1
par_progress
par_progress Computer:jobs running/jobs completed/%of started jobs/Average seconds to complete
par_progress local:0/3/100%/9.9s 
par_progress
par_progress Computers / CPU cores / Max jobs to run
par_progress 1:local / 4 / 4
par_progress
par_progress Computer:jobs running/jobs completed/%of started jobs/Average seconds to complete
par_progress local:0/3/100%/9.9s 
par_progress
par_progress Computers / CPU threads / Max jobs to run
par_progress 1:local / 8 / 8
par_progress
par_progress Computer:jobs running/jobs completed/%of started jobs/Average seconds to complete
par_progress local:0/3/100%/9.9s 
par_pxz_complains bug #44250: pxz complains File format not recognized but decompresses anyway par_pxz_complains bug #44250: pxz complains File format not recognized but decompresses anyway
par_pxz_complains ls: cannot access '/OK-if-missing-file': No such file or directory par_pxz_complains ls: cannot access '/OK-if-missing-file': No such file or directory
par_pxz_complains can not seek in input: Illegal seek par_pxz_complains can not seek in input: Illegal seek

12
testsuite/wanted-results/parallel-local-3s
View file

@ -412,9 +412,11 @@ par_eta ### Test of --eta
par_eta 16 par_eta 16
par_eta ### Test of --eta with no jobs par_eta ### Test of --eta with no jobs
par_eta par_eta
par_eta Computers / CPU cores / Max jobs to run par_eta Computers / CPU threads / Max jobs to run
par_eta 1:local / 9 / 9 par_eta 1:local / 9 / 9
par_eta par_eta ETA: 0s Left: 0 AVG: 0.00s 0 par_eta
par_eta Computer:jobs running/jobs completed/%of started jobs/Average seconds to complete
par_eta par_eta ETA: 0s Left: 0 AVG: 0.00s local:0/0/0%/0.0s 
par_exitval_signal ### Test --joblog with exitval and Test --joblog with signal -- timing dependent par_exitval_signal ### Test --joblog with exitval and Test --joblog with signal -- timing dependent
par_exitval_signal exitval=128+6 OK par_exitval_signal exitval=128+6 OK
par_exitval_signal signal OK par_exitval_signal signal OK
@ -847,9 +849,11 @@ par_progress ### Test of --progress
par_progress 16 par_progress 16
par_progress ### Test of --progress with no jobs par_progress ### Test of --progress with no jobs
par_progress par_progress
par_progress Computers / CPU cores / Max jobs to run par_progress Computers / CPU threads / Max jobs to run
par_progress 1:local / 9 / 9 par_progress 1:local / 9 / 9
par_progress par_progress 0 par_progress
par_progress Computer:jobs running/jobs completed/%of started jobs/Average seconds to complete
par_progress par_progress local:0/0/0%/0.0s 
par_replacement_slashslash ### Test {//} par_replacement_slashslash ### Test {//}
par_replacement_slashslash . a par_replacement_slashslash . a
par_replacement_slashslash a a/b par_replacement_slashslash a a/b

4
testsuite/wanted-results/parallel-tutorial
View file

@ -489,11 +489,11 @@ parallel: Warning: sleep 8; echo 8
parallel: Warning: This job was killed because it timed out: parallel: Warning: This job was killed because it timed out:
parallel: Warning: sleep 7; echo 7 parallel: Warning: sleep 7; echo 7
parallel --eta sleep ::: 1 3 2 2 1 3 3 2 1 parallel --eta sleep ::: 1 3 2 2 1 3 3 2 1
Computers / CPU cores / Max jobs to run Computers / CPU threads / Max jobs to run
1:local / 9 / 9 1:local / 9 / 9
Computer:jobs running/jobs completed/%of started jobs/Average seconds to complete Computer:jobs running/jobs completed/%of started jobs/Average seconds to complete
parallel --progress sleep ::: 1 3 2 2 1 3 3 2 1 parallel --progress sleep ::: 1 3 2 2 1 3 3 2 1
Computers / CPU cores / Max jobs to run Computers / CPU threads / Max jobs to run
1:local / 9 / 9 1:local / 9 / 9
Computer:jobs running/jobs completed/%of started jobs/Average seconds to complete Computer:jobs running/jobs completed/%of started jobs/Average seconds to complete
seq 1000 | parallel -j10 --bar '(echo -n {};sleep 0.1)' \ seq 1000 | parallel -j10 --bar '(echo -n {};sleep 0.1)' \