mirror of
https://git.savannah.gnu.org/git/parallel.git
synced 2024-11-25 23:47:53 +00:00
Fixed bug #45841: Replacement string for total no of jobs.
This commit is contained in:
Ole Tange
parent
554b723687
commit
03a35016a9
53
src/parallel
53
src/parallel
|
|
@ -1131,6 +1131,8 @@ sub init_globals {
|
|||
'{...}' => 's:\.[^/.]+$::; s:\.[^/.]+$::; s:\.[^/.]+$::',
|
||||
'{/..}' => 's:.*/::; s:\.[^/.]+$::; s:\.[^/.]+$::',
|
||||
'{/...}' => 's:.*/::; s:\.[^/.]+$::; s:\.[^/.]+$::; s:\.[^/.]+$::',
|
||||
# {##} = number of jobs
|
||||
'{##}' => '$_=$Global::JobQueue->total_jobs()',
|
||||
);
|
||||
# Modifiable copy of %Global::replace
|
||||
%Global::rpl = %Global::replace;
|
||||
|
|
@ -5746,6 +5748,7 @@ sub new {
|
|||
return bless {
|
||||
'unget' => \@unget,
|
||||
'commandlinequeue' => $commandlinequeue,
|
||||
'this_job_no' => 1,
|
||||
'total_jobs' => undef,
|
||||
}, ref($class) || $class;
|
||||
}
|
||||
|
|
@ -5754,11 +5757,13 @@ sub get {
|
|||
my $self = shift;
|
||||
|
||||
if(@{$self->{'unget'}}) {
|
||||
$self->{'this_job_no'}++;
|
||||
my $job = shift @{$self->{'unget'}};
|
||||
return ($job);
|
||||
} else {
|
||||
my $commandline = $self->{'commandlinequeue'}->get();
|
||||
if(defined $commandline) {
|
||||
$self->{'this_job_no'}++;
|
||||
my $job = Job->new($commandline);
|
||||
return $job;
|
||||
} else {
|
||||
|
|
@ -5770,6 +5775,7 @@ sub get {
|
|||
sub unget {
|
||||
my $self = shift;
|
||||
unshift @{$self->{'unget'}}, @_;
|
||||
$self->{'this_job_no'} -= @_;
|
||||
}
|
||||
|
||||
sub empty {
|
||||
|
|
@ -5793,7 +5799,8 @@ sub total_jobs {
|
|||
my $start = time;
|
||||
while($record = $record_queue->get()) {
|
||||
if(time - $start > 10) {
|
||||
::warning("Reading ".scalar(@arg_records)." arguments took longer than 10 seconds.");
|
||||
::warning("Reading ".scalar(@arg_records).
|
||||
" arguments took longer than 10 seconds.");
|
||||
$opt::eta && ::warning("Consider removing --eta.");
|
||||
$opt::bar && ::warning("Consider removing --bar.");
|
||||
$opt::shuf && ::warning("Consider removing --shuf.");
|
||||
|
|
@ -5812,7 +5819,7 @@ sub total_jobs {
|
|||
}
|
||||
}
|
||||
$record_queue->unget(@arg_records);
|
||||
$self->{'total_jobs'} = $#arg_records+1;
|
||||
$self->{'total_jobs'} = $#arg_records+1+$self->{'this_job_no'};
|
||||
::debug("init","Total jobs: ".$self->{'total_jobs'}."\n");
|
||||
}
|
||||
return $self->{'total_jobs'};
|
||||
|
|
@ -6094,16 +6101,20 @@ sub filter_through_compress {
|
|||
my ($fh, $comfile) = ::tmpfile(SUFFIX => ".pac");
|
||||
close $fh;
|
||||
# Compressor: (echo > $comfile; compress pipe) > output
|
||||
# When the echo is written to $comfile, it is known that output file is opened,
|
||||
# When the echo is written to $comfile,
|
||||
# it is known that output file is opened,
|
||||
# thus output file can then be removed by the decompressor.
|
||||
my $wpid = open(my $fdw,"|-", "(echo > $comfile; ".empty_input_wrapper($opt::compress_program).") >".
|
||||
my $wpid = open(my $fdw,"|-", "(echo > $comfile; ".
|
||||
empty_input_wrapper($opt::compress_program).") >".
|
||||
$self->fh($fdno,'name')) || die $?;
|
||||
$self->set_fh($fdno,'w',$fdw);
|
||||
$self->set_fh($fdno,'wpid',$wpid);
|
||||
# Decompressor: open output; -s $comfile > 0: rm $comfile output; decompress output > stdout
|
||||
# Decompressor: open output; -s $comfile > 0: rm $comfile output;
|
||||
# decompress output > stdout
|
||||
my $rpid = open(my $fdr, "-|", "perl", "-e", $cattail, $comfile,
|
||||
$opt::decompress_program, $wpid,
|
||||
$self->fh($fdno,'name'),$self->fh($fdno,'unlink')) || die $?;
|
||||
$self->fh($fdno,'name'),$self->fh($fdno,'unlink'))
|
||||
|| die $?;
|
||||
$self->set_fh($fdno,'r',$fdr);
|
||||
$self->set_fh($fdno,'rpid',$rpid);
|
||||
}
|
||||
|
|
@ -6125,9 +6136,14 @@ sub max_file_name_length {
|
|||
# Figure out the max length of a subdir
|
||||
# TODO and the max total length
|
||||
# Ext4 = 255,130816
|
||||
# Uses:
|
||||
# $Global::max_file_length is set
|
||||
# Returns:
|
||||
# $Global::max_file_length
|
||||
my $testdir = shift;
|
||||
|
||||
my $upper = 8_000_000;
|
||||
# Dir length of 8 chars is supported everywhere
|
||||
my $len = 8;
|
||||
my $dir = "x"x$len;
|
||||
do {
|
||||
|
|
@ -7107,8 +7123,10 @@ sub start {
|
|||
open ERR, '>&', $stderr_fh or ::die_bug("Can't dup STDERR: $!");
|
||||
|
||||
my $pid;
|
||||
my @setpgrp_wrap = ('perl','-e',
|
||||
"setpgrp\;eval\{setpriority\(0,0,$opt::nice\)\}\;exec '$Global::shell', '-c', \@ARGV");
|
||||
my @setpgrp_wrap =
|
||||
('perl','-e',
|
||||
"setpgrp\;eval\{setpriority\(0,0,$opt::nice\)\}\;".
|
||||
"exec '$Global::shell', '-c', \@ARGV");
|
||||
# The eval is needed to catch exception from open3
|
||||
eval {
|
||||
$pid = ::open3($stdin_fh, ">&OUT", ">&ERR", @setpgrp_wrap, $command)
|
||||
|
|
@ -7191,7 +7209,8 @@ sub start {
|
|||
*IN = *STDIN;
|
||||
# The eval is needed to catch exception from open3
|
||||
my @setpgrp_wrap = ('perl','-e',
|
||||
"setpgrp\;eval\{setpriority\(0,0,$opt::nice\)\}\;exec '$Global::shell', '-c', \@ARGV");
|
||||
"setpgrp\;eval\{setpriority\(0,0,$opt::nice\)\}\;".
|
||||
"exec '$Global::shell', '-c', \@ARGV");
|
||||
eval {
|
||||
$pid = ::open3("<&IN", ">&OUT", ">&ERR", @setpgrp_wrap, $command)
|
||||
|| ::die_bug("open3-<IN");
|
||||
|
|
@ -7394,7 +7413,7 @@ sub should_be_retried {
|
|||
my (%print_later,$job_end_sequence);
|
||||
|
||||
sub print_earlier_jobs {
|
||||
# Print jobs completed earlier
|
||||
# Print jobs whose output is postponed due to --keep-order
|
||||
# Returns: N/A
|
||||
my $job = shift;
|
||||
$print_later{$job->seq()} = $job;
|
||||
|
|
@ -7903,12 +7922,9 @@ sub populate {
|
|||
my $next_arg;
|
||||
my $max_len = $Global::minimal_command_line_length || Limits::Command::max_length();
|
||||
|
||||
if($opt::cat) {
|
||||
# $PARALLEL_TMP will point to a tempfile that will be used as {}
|
||||
$Global::JobQueue->{'commandlinequeue'}->{'arg_queue'}->
|
||||
unget([Arg->new('$PARALLEL_TMP')]);
|
||||
}
|
||||
if($opt::fifo) {
|
||||
if($opt::cat or $opt::fifo) {
|
||||
# Get the empty arg added by --pipepart (if any)
|
||||
$Global::JobQueue->{'commandlinequeue'}->{'arg_queue'}->get();
|
||||
# $PARALLEL_TMP will point to a tempfile that will be used as {}
|
||||
$Global::JobQueue->{'commandlinequeue'}->{'arg_queue'}->
|
||||
unget([Arg->new('$PARALLEL_TMP')]);
|
||||
|
|
@ -8162,11 +8178,6 @@ sub len {
|
|||
$self->{'replacecount'}{$replstring};
|
||||
}
|
||||
}
|
||||
if($opt::nice) {
|
||||
# Pessimistic length if --nice is set
|
||||
# Worse than worst case: every char needs to be quoted with \
|
||||
$len *= 2;
|
||||
}
|
||||
if($Global::quoting) {
|
||||
# Pessimistic length if -q is set
|
||||
# Worse than worst case: every char needs to be quoted with \
|
||||
|
|
|
|||
|
|
@ -55,7 +55,7 @@ bash -c 'echo bug \#43358: shellshock breaks exporting functions using --env _;
|
|||
parallel --env funky -S parallel@192.168.1.72 funky ::: shellshock-hardened'
|
||||
|
||||
echo '### Test --load (must give 1=true)'
|
||||
parallel -j0 -N0 --timeout 5 --nice 10 'bzip2 < /dev/zero >/dev/null' ::: 1 2 3 &
|
||||
parallel -j0 -N0 --timeout 5 --nice 10 'bzip2 < /dev/zero >/dev/null' ::: 1 2 3 4 5 6 &
|
||||
parallel --argsep ,, --joblog - -N0 parallel --load 100% echo ::: 1 ,, 1 |
|
||||
parallel --colsep '\t' --header : echo '{=4 $_=$_>5=}'
|
||||
|
||||
|
|
|
|||
|
|
@ -247,6 +247,14 @@ echo 'bug #46231: {%} with --pipepart broken. Should give 1+2'
|
|||
|
||||
echo '**'
|
||||
|
||||
echo '{##} bug #45841: Replacement string for total no of jobs'
|
||||
|
||||
parallel --plus echo {##} ::: {a..j};
|
||||
parallel -k 'echo {= $::G++ > 3 and ($_=$Global::JobQueue->total_jobs());=}' ::: {1..10}
|
||||
|
||||
echo '**'
|
||||
|
||||
|
||||
EOF
|
||||
echo '### 1 .par file from --files expected'
|
||||
find /tmp{/*,}/*.{par,tms,tmx} 2>/dev/null -mmin -10 | wc -l
|
||||
|
|
|
|||
|
|
@ -60,6 +60,8 @@ perl -ne '$/="\n\n"; /^Output/../^[^O]\S/ and next; /^ / and print;' ../../src/
|
|||
s/tried 2\n//;
|
||||
# Due to order is often mixed up
|
||||
s/echo \d; exit \d\n/echo X; exit X\n/;
|
||||
# Race condition causes outdir to sometime exist
|
||||
s/(std(out|err):) Permission denied/$1 No such file or directory/;
|
||||
'
|
||||
# 3+3 .par files (from --files), 1 .tms-file from tmux attach
|
||||
find {$TMPDIR,/var/tmp,/tmp}/{fif,tms,par[^a]}* -mmin -10 2>/dev/null | wc -l
|
||||
|
|
|
|||
|
|
@ -519,5 +519,30 @@ bug #46231: {%} with --pipepart broken. Should give 1+2
|
|||
1
|
||||
echo '**'
|
||||
**
|
||||
echo '{##} bug #45841: Replacement string for total no of jobs'
|
||||
{##} bug #45841: Replacement string for total no of jobs
|
||||
parallel --plus echo {##} ::: {a..j}; parallel -k 'echo {= $::G++ > 3 and ($_=$Global::JobQueue->total_jobs());=}' ::: {1..10}
|
||||
10
|
||||
10
|
||||
10
|
||||
10
|
||||
10
|
||||
10
|
||||
10
|
||||
10
|
||||
10
|
||||
10
|
||||
1
|
||||
2
|
||||
3
|
||||
4
|
||||
10
|
||||
10
|
||||
10
|
||||
10
|
||||
10
|
||||
10
|
||||
echo '**'
|
||||
**
|
||||
### 1 .par file from --files expected
|
||||
1
|
||||
|
|
|
|||
|
|
@ -1,11 +1,13 @@
|
|||
bug #46120: Suspend should suspend (at least local) children
|
||||
2048 0a:c0:70:5b:ec:f6:c2:de:67:c3:53:7f:29:81:65:54 tange@hk (RSA1)
|
||||
1024 93:d1:21:c5:ff:e3:c3:be:6d:73:ea:aa:1e:a2:dc:06 /home/tange/.ssh/id_dsa (DSA)
|
||||
8192 e1:95:e3:ff:99:a6:3a:b5:53:5a:54:59:d0:72:94:7f /home/tange/.ssh/id_rsa (RSA)
|
||||
4096 94:2a:e3:cb:6b:66:63:21:13:51:8d:e8:4e:09:49:b2 /home/tange/.ssh/id_rsa_openindiana (RSA)
|
||||
|
||||
stdout /usr/bin/time -f CPUTIME=%U parallel --timeout 5 burnP6 ::: 1 | grep --colour=auto -q CPUTIME=1
|
||||
Zero=OK 0
|
||||
2048 0a:c0:70:5b:ec:f6:c2:de:67:c3:53:7f:29:81:65:54 tange@hk (RSA1)
|
||||
1024 93:d1:21:c5:ff:e3:c3:be:6d:73:ea:aa:1e:a2:dc:06 /home/tange/.ssh/id_dsa (DSA)
|
||||
8192 e1:95:e3:ff:99:a6:3a:b5:53:5a:54:59:d0:72:94:7f /home/tange/.ssh/id_rsa (RSA)
|
||||
4096 94:2a:e3:cb:6b:66:63:21:13:51:8d:e8:4e:09:49:b2 /home/tange/.ssh/id_rsa_openindiana (RSA)
|
||||
|
||||
|
|
|
|||
|
|
@ -30,7 +30,7 @@ aix.polarhome.com Error in tempfile() using /XXXXXXXX.arg: Could not create temp
|
|||
copy_and_test redhat.polarhome.com
|
||||
redhat.polarhome.com ### Run the test on redhat.polarhome.com
|
||||
redhat.polarhome.com Works on redhat.polarhome.com
|
||||
redhat.polarhome.com Error in tempfile() using /XXXXXXXX.arg: Could not create temp file /XXXXXXXX.arg: Permission denied at bin/parallel line 0000
|
||||
redhat.polarhome.com Error in tempfile() using template /XXXXXXXX.arg: Could not create temp file /XXXXXXXX.arg: Permission denied at bin/parallel line 0000.
|
||||
copy_and_test hpux.polarhome.com
|
||||
hpux.polarhome.com ### Run the test on hpux.polarhome.com
|
||||
hpux.polarhome.com Works on hpux.polarhome.com
|
||||
|
|
@ -118,6 +118,7 @@ centos.polarhome.com
|
|||
debian
|
||||
dragonfly.polarhome.com
|
||||
freebsd.polarhome.com
|
||||
hpux-ia64
|
||||
hpux64
|
||||
hurd
|
||||
macosx
|
||||
|
|
@ -129,6 +130,7 @@ openindiana
|
|||
pidora
|
||||
qnx
|
||||
raspbian
|
||||
redhat.polarhome.com
|
||||
scosysv.polarhome.com
|
||||
solaris-x86
|
||||
solaris
|
||||
|
|
|
|||
Loading…
Reference in a new issue