tange/parallel
1
0
Fork 0
mirror of https://git.savannah.gnu.org/git/parallel.git synced 2024-10-18 04:49:40 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

parallel: --progress bug: Short jobs had first status before the header.

Browse source
This commit is contained in:
Ole Tange 2023-12-13 01:02:19 +01:00
parent 0553dbd55c
commit 2e352aa51f
6 changed files with 192 additions and 182 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

206
src/parallel
View file

@ -4272,9 +4272,11 @@ sub init_progress() {
if($opt::bar) { if($opt::bar) {
return("",""); return("","");
} }
my %progress = progress(); my $progress = progress();
return ("\nComputers / CPU cores / Max jobs to run\n", my $cpu_units = $opt::use_sockets_instead_of_threads ? "CPU sockets" :
$progress{'workerlist'}); ($opt::use_cores_instead_of_threads ? "CPU cores" : "CPU threads");
return ("\nComputers / $cpu_units / Max jobs to run\n",
$progress->{'workerlist'},"\n",$progress->{'header'});
} }
sub drain_job_queue(@) { sub drain_job_queue(@) {
@ -4288,10 +4290,6 @@ sub drain_job_queue(@) {
# $Global::start_no_new_jobs # $Global::start_no_new_jobs
# Returns: N/A # Returns: N/A
my @command = @_; my @command = @_;
if($opt::progress) {
::status_no_nl(init_progress());
}
my $last_header = "";
my $sleep = 0.2; my $sleep = 0.2;
my $sleepsum = 0; my $sleepsum = 0;
do { do {
@ -4306,12 +4304,8 @@ sub drain_job_queue(@) {
} }
} }
if($opt::progress) { if($opt::progress) {
my %progress = progress(); my $progress = progress();
if($last_header ne $progress{'header'}) { ::status_no_nl("\r",$progress->{'status'});
::status("", $progress{'header'});
$last_header = $progress{'header'};
}
::status_no_nl("\r",$progress{'status'});
} }
if($Global::total_running < $Global::max_jobs_running if($Global::total_running < $Global::max_jobs_running
and not $Global::JobQueue->empty()) { and not $Global::JobQueue->empty()) {
@ -4373,8 +4367,8 @@ sub drain_job_queue(@) {
$opt::sqlmaster and not $Global::sql->finished()); $opt::sqlmaster and not $Global::sql->finished());
$Global::all_jobs_done = 1; $Global::all_jobs_done = 1;
if($opt::progress) { if($opt::progress) {
my %progress = progress(); my $progress = progress();
::status("\r".$progress{'status'}); ::status("\r".$progress->{'status'});
} }
} }
@ -4389,7 +4383,11 @@ sub toggle_progress() {
} }
} }
sub progress() { {
my $last_header;
my $eol;
sub progress() {
# Uses: # Uses:
# $opt::bar # $opt::bar
# $opt::eta # $opt::eta
@ -4400,7 +4398,7 @@ sub progress() {
# $header = that will fit on the screen # $header = that will fit on the screen
# $status = message that will fit on the screen # $status = message that will fit on the screen
if($opt::bar) { if($opt::bar) {
return ("workerlist" => "", "header" => "", "status" => bar()); return {"workerlist" => "", "header" => "", "status" => bar()};
} }
my $eta = ""; my $eta = "";
my ($status,$header)=("",""); my ($status,$header)=("","");
@ -4412,142 +4410,121 @@ sub progress() {
} }
my $termcols = terminal_columns(); my $termcols = terminal_columns();
my @workers = sort keys %Global::host; my @workers = sort keys %Global::host;
my %sshlogin = map { $_ eq ":" ? ($_ => "local") : ($_ => $_) } @workers;
my $workerno = 1; my $workerno = 1;
my %workerno = map { ($_=>$workerno++) } @workers; my %wrk;
for my $w (@workers) {
my %i;
$i{'sshlogin'} = $w eq ":" ? "local" : $w;
$i{'no'} = $workerno++;
$i{'ncpu'} = ($Global::host{$w}->ncpus() || "-");
$i{'jobslots'} = $Global::host{$w}->max_jobs_running();
$i{'completed'} = ($Global::host{$w}->jobs_completed() || 0);
$i{'running'} = $Global::host{$w}->jobs_running();
$i{'pct'} = $Global::total_started ?
(($i{'running'}+$i{'completed'})*100 /
$Global::total_started) : 0;
$i{'time'} = $i{'completed'} ? (time-$^T)/($i{'completed'}) : 0;
$wrk{$w} = \%i;
}
my $workerlist = ""; my $workerlist = "";
for my $w (@workers) { for my $w (@workers) {
$workerlist .= $workerlist .=
$workerno{$w}.":".$sshlogin{$w} ." / ". $wrk{$w}{'no'}.":".$wrk{$w}{'sshlogin'} ." / ".
($Global::host{$w}->ncpus() || "-")." / ". $wrk{$w}{'ncpu'}." / ".
$Global::host{$w}->max_jobs_running()."\n"; $wrk{$w}{'jobslots'}."\n";
} }
# Force $status to select one of the below formats
$status = "c"x($termcols+1); $status = "c"x($termcols+1);
# Select an output format that will fit on a single line # Select an output format that will fit on a single line
if(length $status > $termcols) { if(length $status > $termcols) {
# sshlogin1:XX/XX/XX%/XX.Xs s2:XX/XX/XX%/XX.Xs s3:XX/XX/XX%/XX.Xs # sshlogin1:XX/XX/XX%/XX.Xs s2:XX/XX/XX%/XX.Xs s3:XX/XX/XX%/XX.Xs
$header = "Computer:jobs running/jobs completed/". $header = "Computer:jobs running/jobs completed/".
"%of started jobs/Average seconds to complete"; "%of started jobs/Average seconds to complete";
$status = $eta . $status = $eta . join(" ",map {
join(" ",map
{
if($Global::total_started) {
my $completed =
($Global::host{$_}->jobs_completed()||0);
my $running = $Global::host{$_}->jobs_running();
my $time = $completed ? (time-$^T)/($completed) : "0";
sprintf("%s:%d/%d/%d%%/%.1fs ", sprintf("%s:%d/%d/%d%%/%.1fs ",
$sshlogin{$_}, $running, $completed, @{$wrk{$_}}
($running+$completed)*100 {'sshlogin','running','completed','pct','time'}
/ $Global::total_started, $time); ); } @workers);
}
} @workers);
} }
if(length $status > $termcols) { if(length $status > $termcols) {
# 1:XX/XX/XX%/X.Xs 2:XX/XX/XX%/X.Xs 3:XX/XX/XX%/X.Xs 4:XX/XX/XX%/X.Xs # 1:XX/XX/XX%/X.Xs 2:XX/XX/XX%/X.Xs 3:XX/XX/XX%/X.Xs
$header = "Computer:jobs running/jobs completed/%of started jobs"; $header = "Computer:jobs running/jobs completed/%of started jobs";
$status = $eta . $status = $eta . join(" ",map {
join(" ",map
{
if($Global::total_started) {
my $completed =
($Global::host{$_}->jobs_completed()||0);
my $running = $Global::host{$_}->jobs_running();
my $time = $completed ? (time-$^T)/($completed) : "0";
sprintf("%s:%d/%d/%d%%/%.1fs ", sprintf("%s:%d/%d/%d%%/%.1fs ",
$workerno{$_}, $running, $completed, @{$wrk{$_}}
($running+$completed)*100 {'no','running','completed','pct','time'}
/ $Global::total_started, $time); ); } @workers);
}
} @workers);
} }
if(length $status > $termcols) { if(length $status > $termcols) {
# sshlogin1:XX/XX/XX% sshlogin2:XX/XX/XX% sshlogin3:XX/XX/XX% # sshlogin1:XX/XX/XX% sshlogin2:XX/XX/XX% sshlogin3:XX/XX/XX%
$header = "Computer:jobs running/jobs completed/%of started jobs"; $header = "Computer:jobs running/jobs completed/%of started jobs";
$status = $eta . $status = $eta . join(" ",map {
join(" ",map
{
if($Global::total_started) {
sprintf("%s:%d/%d/%d%%", sprintf("%s:%d/%d/%d%%",
$sshlogin{$_}, @{$wrk{$_}}
$Global::host{$_}->jobs_running(), {'sshlogin','running','completed','pct'}
($Global::host{$_}->jobs_completed()||0), ); } @workers);
($Global::host{$_}->jobs_running()+
($Global::host{$_}->jobs_completed()||0))*100
/ $Global::total_started)
}
}
@workers);
} }
if(length $status > $termcols) { if(length $status > $termcols) {
# 1:XX/XX/XX% 2:XX/XX/XX% 3:XX/XX/XX% 4:XX/XX/XX% 5:XX/XX/XX% # 1:XX/XX/XX% 2:XX/XX/XX% 3:XX/XX/XX% 4:XX/XX/XX% 5:XX/XX/XX%
$header = "Computer:jobs running/jobs completed/%of started jobs"; $header = "Computer:jobs running/jobs completed/%of started jobs";
$status = $eta . $status = $eta . join(" ",map {
join(" ",map
{
if($Global::total_started) {
sprintf("%s:%d/%d/%d%%", sprintf("%s:%d/%d/%d%%",
$workerno{$_}, @{$wrk{$_}}
$Global::host{$_}->jobs_running(), {'no','running','completed','pct'}
($Global::host{$_}->jobs_completed()||0), ); } @workers);
($Global::host{$_}->jobs_running()+
($Global::host{$_}->jobs_completed()||0))*100
/ $Global::total_started)
}
}
@workers);
} }
if(length $status > $termcols) { if(length $status > $termcols) {
# sshlogin1:XX/XX/XX% sshlogin2:XX/XX/XX% sshlogin3:XX/XX # sshlogin1:XX/XX sshlogin2:XX/XX sshlogin3:XX/XX
$header = "Computer:jobs running/jobs completed"; $header = "Computer:jobs running/jobs completed";
$status = $eta . $status = $eta . join(" ", map {
join(" ", sprintf("%s:%d/%d",
map { sprintf("%s:%d/%d", @{$wrk{$_}}
$sshlogin{$_}, $Global::host{$_}->jobs_running(), {'sshlogin','running','completed'}
($Global::host{$_}->jobs_completed()||0)) } ); } @workers);
@workers);
}
if(length $status > $termcols) {
# sshlogin1:XX/XX sshlogin2:XX/XX sshlogin3:XX/XX sshlogin4:XX/XX
$header = "Computer:jobs running/jobs completed";
$status = $eta .
join(" ",
map { sprintf("%s:%d/%d",
$sshlogin{$_}, $Global::host{$_}->jobs_running(),
($Global::host{$_}->jobs_completed()||0)) }
@workers);
} }
if(length $status > $termcols) { if(length $status > $termcols) {
# 1:XX/XX 2:XX/XX 3:XX/XX 4:XX/XX 5:XX/XX 6:XX/XX # 1:XX/XX 2:XX/XX 3:XX/XX 4:XX/XX 5:XX/XX 6:XX/XX
$header = "Computer:jobs running/jobs completed"; $header = "Computer:jobs running/jobs completed";
$status = $eta . $status = $eta . join(" ", map {
join(" ", sprintf("%s:%d/%d",
map { sprintf("%s:%d/%d", $workerno{$_}, @{$wrk{$_}}
$Global::host{$_}->jobs_running(), {'no','running','completed'}
($Global::host{$_}->jobs_completed()||0)) } ); } @workers);
@workers);
} }
if(length $status > $termcols) { if(length $status > $termcols) {
# sshlogin1:XX sshlogin2:XX sshlogin3:XX sshlogin4:XX sshlogin5:XX # sshlogin1:XX sshlogin2:XX sshlogin3:XX sshlogin4:XX sshlogin5:XX
$header = "Computer:jobs completed"; $header = "Computer:jobs completed";
$status = $eta . $status = $eta . join(" ", map {
join(" ", sprintf("%s:%d",
map { sprintf("%s:%d", $sshlogin{$_}, @{$wrk{$_}}
($Global::host{$_}->jobs_completed()||0)) } {'sshlogin','completed'}
@workers); ); } @workers);
} }
if(length $status > $termcols) { if(length $status > $termcols) {
# 1:XX 2:XX 3:XX 4:XX 5:XX 6:XX # 1:XX 2:XX 3:XX 4:XX 5:XX 6:XX
$header = "Computer:jobs completed"; $header = "Computer:jobs completed";
$status = $eta . $status = $eta . join(" ", map {
join(" ", sprintf("%s:%d",
map { sprintf("%s:%d", @{$wrk{$_}}
$workerno{$_}, {'no','completed'}
($Global::host{$_}->jobs_completed()||0)) } ); } @workers);
@workers); }
if($last_header ne $header) {
$header .= "\n";
$last_header = $header;
} else {
$header = "";
}
if(not $eol) {
$eol = `sh -c "tput el </dev/tty" 2>/dev/null`;
chomp($eol);
if($eol eq "") { $eol = "\033[K"; }
}
return {"workerlist" => $workerlist, "header" => $header,
"status" => $status.$eol};
} }
return ("workerlist" => $workerlist, "header" => $header,
"status" => $status);
} }
{ {
@ -5691,8 +5668,8 @@ sub reaper() {
$job->cleanup(); $job->cleanup();
if($opt::progress) { if($opt::progress) {
my %progress = progress(); my $progress = progress();
::status_no_nl("\r",$progress{'status'}); ::status_no_nl("\r",$progress->{'status'});
} }
debug("run", "jobdone \n"); debug("run", "jobdone \n");
@ -15426,6 +15403,9 @@ sub main() {
$SIG{TERM} = $Global::original_sig{TERM}; $SIG{TERM} = $Global::original_sig{TERM};
$SIG{HUP} = \&start_no_new_jobs; $SIG{HUP} = \&start_no_new_jobs;
if($opt::progress) {
::status_no_nl(init_progress());
}
if($opt::tee or $opt::shard or $opt::bin) { if($opt::tee or $opt::shard or $opt::bin) {
# All jobs must be running in parallel for --tee/--shard/--bin # All jobs must be running in parallel for --tee/--shard/--bin
while(start_more_jobs()) {} while(start_more_jobs()) {}

8
testsuite/tests-to-run/parallel-local-1s.sh
View file

@ -8,6 +8,14 @@
# Each should be taking 1-3s and be possible to run in parallel # Each should be taking 1-3s and be possible to run in parallel
# I.e.: No race conditions, no logins # I.e.: No race conditions, no logins
par_progress() {
(
parallel --progress --use-sockets-instead-of-threads true ::: a b c
parallel --progress --use-cores-instead-of-threads true ::: a b c
parallel --progress --use-cpus-instead-of-cores true ::: a b c
) 2>&1 | perl -pe 's/.*\r//; s/\d.\ds/9.9s/'
}
par_citation_no_config_dir() { par_citation_no_config_dir() {
echo '### bug #64329: parallel --citation will loop forever unless the config dir exists' echo '### bug #64329: parallel --citation will loop forever unless the config dir exists'
t=$(mktemp -d) t=$(mktemp -d)

2
testsuite/wanted-results/parallel-local-0.3s
View file

@ -18,7 +18,7 @@ par_PARALLEL_HOME_with_+ bug #59453: PARALLEL_HOME with plus sign causes error:
par_PARALLEL_HOME_with_+ parallel: Warning: $PARALLEL_HOME can only contain [-a-z0-9_+,.%:/= ]. par_PARALLEL_HOME_with_+ parallel: Warning: $PARALLEL_HOME can only contain [-a-z0-9_+,.%:/= ].
par_PARALLEL_HOME_with_+ Parallel_home_with+ par_PARALLEL_HOME_with_+ Parallel_home_with+
par_X_eta_div_zero ### bug #34422: parallel -X --eta crashes with div by zero par_X_eta_div_zero ### bug #34422: parallel -X --eta crashes with div by zero
par_X_eta_div_zero Computers / CPU cores / Max jobs to run par_X_eta_div_zero Computers / CPU threads / Max jobs to run
par_X_eta_div_zero 0:local / 0 / 0 par_X_eta_div_zero 0:local / 0 / 0
par_append_joblog ### can you append to a joblog using + par_append_joblog ### can you append to a joblog using +
par_append_joblog 1 par_append_joblog 1

18
testsuite/wanted-results/parallel-local-1s
View file

@ -699,6 +699,24 @@ par_profiles_with_space /bin/bash=/bin/bash
par_profiles_with_space echo '/bin/bash=/bin/bash' par_profiles_with_space echo '/bin/bash=/bin/bash'
par_profiles_with_space /bin/bash=/bin/bash par_profiles_with_space /bin/bash=/bin/bash
par_profiles_with_space With script in $PARALLEL /bin/bash=/TMP par_profiles_with_space With script in $PARALLEL /bin/bash=/TMP
par_progress
par_progress Computers / CPU sockets / Max jobs to run
par_progress 1:local / 1 / 1
par_progress
par_progress Computer:jobs running/jobs completed/%of started jobs/Average seconds to complete
par_progress local:0/3/100%/9.9s 
par_progress
par_progress Computers / CPU cores / Max jobs to run
par_progress 1:local / 4 / 4
par_progress
par_progress Computer:jobs running/jobs completed/%of started jobs/Average seconds to complete
par_progress local:0/3/100%/9.9s 
par_progress
par_progress Computers / CPU threads / Max jobs to run
par_progress 1:local / 8 / 8
par_progress
par_progress Computer:jobs running/jobs completed/%of started jobs/Average seconds to complete
par_progress local:0/3/100%/9.9s 
par_pxz_complains bug #44250: pxz complains File format not recognized but decompresses anyway par_pxz_complains bug #44250: pxz complains File format not recognized but decompresses anyway
par_pxz_complains ls: cannot access '/OK-if-missing-file': No such file or directory par_pxz_complains ls: cannot access '/OK-if-missing-file': No such file or directory
par_pxz_complains can not seek in input: Illegal seek par_pxz_complains can not seek in input: Illegal seek

12
testsuite/wanted-results/parallel-local-3s
View file

@ -412,9 +412,11 @@ par_eta ### Test of --eta
par_eta 16 par_eta 16
par_eta ### Test of --eta with no jobs par_eta ### Test of --eta with no jobs
par_eta par_eta
par_eta Computers / CPU cores / Max jobs to run par_eta Computers / CPU threads / Max jobs to run
par_eta 1:local / 9 / 9 par_eta 1:local / 9 / 9
par_eta par_eta ETA: 0s Left: 0 AVG: 0.00s 0 par_eta
par_eta Computer:jobs running/jobs completed/%of started jobs/Average seconds to complete
par_eta par_eta ETA: 0s Left: 0 AVG: 0.00s local:0/0/0%/0.0s 
par_exitval_signal ### Test --joblog with exitval and Test --joblog with signal -- timing dependent par_exitval_signal ### Test --joblog with exitval and Test --joblog with signal -- timing dependent
par_exitval_signal exitval=128+6 OK par_exitval_signal exitval=128+6 OK
par_exitval_signal signal OK par_exitval_signal signal OK
@ -847,9 +849,11 @@ par_progress ### Test of --progress
par_progress 16 par_progress 16
par_progress ### Test of --progress with no jobs par_progress ### Test of --progress with no jobs
par_progress par_progress
par_progress Computers / CPU cores / Max jobs to run par_progress Computers / CPU threads / Max jobs to run
par_progress 1:local / 9 / 9 par_progress 1:local / 9 / 9
par_progress par_progress 0 par_progress
par_progress Computer:jobs running/jobs completed/%of started jobs/Average seconds to complete
par_progress par_progress local:0/0/0%/0.0s 
par_replacement_slashslash ### Test {//} par_replacement_slashslash ### Test {//}
par_replacement_slashslash . a par_replacement_slashslash . a
par_replacement_slashslash a a/b par_replacement_slashslash a a/b

4
testsuite/wanted-results/parallel-tutorial
View file

@ -489,11 +489,11 @@ parallel: Warning: sleep 8; echo 8
parallel: Warning: This job was killed because it timed out: parallel: Warning: This job was killed because it timed out:
parallel: Warning: sleep 7; echo 7 parallel: Warning: sleep 7; echo 7
parallel --eta sleep ::: 1 3 2 2 1 3 3 2 1 parallel --eta sleep ::: 1 3 2 2 1 3 3 2 1
Computers / CPU cores / Max jobs to run Computers / CPU threads / Max jobs to run
1:local / 9 / 9 1:local / 9 / 9
Computer:jobs running/jobs completed/%of started jobs/Average seconds to complete Computer:jobs running/jobs completed/%of started jobs/Average seconds to complete
parallel --progress sleep ::: 1 3 2 2 1 3 3 2 1 parallel --progress sleep ::: 1 3 2 2 1 3 3 2 1
Computers / CPU cores / Max jobs to run Computers / CPU threads / Max jobs to run
1:local / 9 / 9 1:local / 9 / 9
Computer:jobs running/jobs completed/%of started jobs/Average seconds to complete Computer:jobs running/jobs completed/%of started jobs/Average seconds to complete
seq 1000 | parallel -j10 --bar '(echo -n {};sleep 0.1)' \ seq 1000 | parallel -j10 --bar '(echo -n {};sleep 0.1)' \