tange/parallel
1
0
Fork 0
mirror of https://git.savannah.gnu.org/git/parallel.git synced 2024-11-22 05:57:54 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fixed bug #45841: Replacement string for total no of jobs.

Browse source
This commit is contained in:
Ole Tange 2015-10-18 11:44:02 +02:00
parent 554b723687
commit 03a35016a9
7 changed files with 74 additions and 24 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

55
src/parallel
View file

@ -1131,6 +1131,8 @@ sub init_globals {
'{...}' => 's:\.[^/.]+$::; s:\.[^/.]+$::; s:\.[^/.]+$::',
'{/..}' => 's:.*/::; s:\.[^/.]+$::; s:\.[^/.]+$::',
'{/...}' => 's:.*/::; s:\.[^/.]+$::; s:\.[^/.]+$::; s:\.[^/.]+$::',
# {##} = number of jobs
'{##}' => '$_=$Global::JobQueue->total_jobs()',
);
# Modifiable copy of %Global::replace
%Global::rpl = %Global::replace;
@ -5746,6 +5748,7 @@ sub new {
return bless {
'unget' => \@unget,
'commandlinequeue' => $commandlinequeue,
'this_job_no' => 1,
'total_jobs' => undef,
}, ref($class) || $class;
}
@ -5754,11 +5757,13 @@ sub get {
my $self = shift;
if(@{$self->{'unget'}}) {
$self->{'this_job_no'}++;
my $job = shift @{$self->{'unget'}};
return ($job);
} else {
my $commandline = $self->{'commandlinequeue'}->get();
if(defined $commandline) {
$self->{'this_job_no'}++;
my $job = Job->new($commandline);
return $job;
} else {
@ -5770,6 +5775,7 @@ sub get {
sub unget {
my $self = shift;
unshift @{$self->{'unget'}}, @_;
$self->{'this_job_no'} -= @_;
}
sub empty {
@ -5793,7 +5799,8 @@ sub total_jobs {
my $start = time;
while($record = $record_queue->get()) {
if(time - $start > 10) {
::warning("Reading ".scalar(@arg_records)." arguments took longer than 10 seconds.");
::warning("Reading ".scalar(@arg_records).
" arguments took longer than 10 seconds.");
$opt::eta && ::warning("Consider removing --eta.");
$opt::bar && ::warning("Consider removing --bar.");
$opt::shuf && ::warning("Consider removing --shuf.");
@ -5812,7 +5819,7 @@ sub total_jobs {
}
}
$record_queue->unget(@arg_records);
$self->{'total_jobs'} = $#arg_records+1;
$self->{'total_jobs'} = $#arg_records+1+$self->{'this_job_no'};
::debug("init","Total jobs: ".$self->{'total_jobs'}."\n");
}
return $self->{'total_jobs'};
@ -6094,16 +6101,20 @@ sub filter_through_compress {
my ($fh, $comfile) = ::tmpfile(SUFFIX => ".pac");
close $fh;
# Compressor: (echo > $comfile; compress pipe) > output
# When the echo is written to $comfile, it is known that output file is opened,
# When the echo is written to $comfile,
# it is known that output file is opened,
# thus output file can then be removed by the decompressor.
my $wpid = open(my $fdw,"|-", "(echo > $comfile; ".empty_input_wrapper($opt::compress_program).") >".
my $wpid = open(my $fdw,"|-", "(echo > $comfile; ".
empty_input_wrapper($opt::compress_program).") >".
$self->fh($fdno,'name')) || die $?;
$self->set_fh($fdno,'w',$fdw);
$self->set_fh($fdno,'wpid',$wpid);
# Decompressor: open output; -s $comfile > 0: rm $comfile output; decompress output > stdout
# Decompressor: open output; -s $comfile > 0: rm $comfile output;
# decompress output > stdout
my $rpid = open(my $fdr, "-|", "perl", "-e", $cattail, $comfile,
$opt::decompress_program, $wpid,
$self->fh($fdno,'name'),$self->fh($fdno,'unlink')) || die $?;
$self->fh($fdno,'name'),$self->fh($fdno,'unlink'))
|| die $?;
$self->set_fh($fdno,'r',$fdr);
$self->set_fh($fdno,'rpid',$rpid);
}
@ -6125,9 +6136,14 @@ sub max_file_name_length {
# Figure out the max length of a subdir
# TODO and the max total length
# Ext4 = 255,130816
# Uses:
# $Global::max_file_length is set
# Returns:
# $Global::max_file_length
my $testdir = shift;
my $upper = 8_000_000;
# Dir length of 8 chars is supported everywhere
my $len = 8;
my $dir = "x"x$len;
do {
@ -7107,8 +7123,10 @@ sub start {
open ERR, '>&', $stderr_fh or ::die_bug("Can't dup STDERR: $!");
my $pid;
my @setpgrp_wrap = ('perl','-e',
"setpgrp\;eval\{setpriority\(0,0,$opt::nice\)\}\;exec '$Global::shell', '-c', \@ARGV");
my @setpgrp_wrap =
('perl','-e',
"setpgrp\;eval\{setpriority\(0,0,$opt::nice\)\}\;".
"exec '$Global::shell', '-c', \@ARGV");
# The eval is needed to catch exception from open3
eval {
$pid = ::open3($stdin_fh, ">&OUT", ">&ERR", @setpgrp_wrap, $command)
@ -7169,7 +7187,7 @@ sub start {
# The eval is needed to catch exception from open3
eval {
no warnings;
local(*IN) = $devtty_fh;
local (*IN) = $devtty_fh;
local (*OUT,*ERR);
open OUT, '>&', $stdout_fh or ::die_bug("Can't dup STDOUT: $!");
open ERR, '>&', $stderr_fh or ::die_bug("Can't dup STDERR: $!");
@ -7191,7 +7209,8 @@ sub start {
*IN = *STDIN;
# The eval is needed to catch exception from open3
my @setpgrp_wrap = ('perl','-e',
"setpgrp\;eval\{setpriority\(0,0,$opt::nice\)\}\;exec '$Global::shell', '-c', \@ARGV");
"setpgrp\;eval\{setpriority\(0,0,$opt::nice\)\}\;".
"exec '$Global::shell', '-c', \@ARGV");
eval {
$pid = ::open3("<&IN", ">&OUT", ">&ERR", @setpgrp_wrap, $command)
|| ::die_bug("open3-<IN");
@ -7394,7 +7413,7 @@ sub should_be_retried {
my (%print_later,$job_end_sequence);
sub print_earlier_jobs {
# Print jobs completed earlier
# Print jobs whose output is postponed due to --keep-order
# Returns: N/A
my $job = shift;
$print_later{$job->seq()} = $job;
@ -7903,12 +7922,9 @@ sub populate {
my $next_arg;
my $max_len = $Global::minimal_command_line_length || Limits::Command::max_length();
if($opt::cat) {
# $PARALLEL_TMP will point to a tempfile that will be used as {}
$Global::JobQueue->{'commandlinequeue'}->{'arg_queue'}->
unget([Arg->new('$PARALLEL_TMP')]);
}
if($opt::fifo) {
if($opt::cat or $opt::fifo) {
# Get the empty arg added by --pipepart (if any)
$Global::JobQueue->{'commandlinequeue'}->{'arg_queue'}->get();
# $PARALLEL_TMP will point to a tempfile that will be used as {}
$Global::JobQueue->{'commandlinequeue'}->{'arg_queue'}->
unget([Arg->new('$PARALLEL_TMP')]);
@ -8162,11 +8178,6 @@ sub len {
$self->{'replacecount'}{$replstring};
}
}
if($opt::nice) {
# Pessimistic length if --nice is set
# Worse than worst case: every char needs to be quoted with \
$len *= 2;
}
if($Global::quoting) {
# Pessimistic length if -q is set
# Worse than worst case: every char needs to be quoted with \

2
testsuite/tests-to-run/parallel-freebsd.sh
View file

@ -55,7 +55,7 @@ bash -c 'echo bug \#43358: shellshock breaks exporting functions using --env _;
parallel --env funky -S parallel@192.168.1.72 funky ::: shellshock-hardened'
echo '### Test --load (must give 1=true)'
parallel -j0 -N0 --timeout 5 --nice 10 'bzip2 < /dev/zero >/dev/null' ::: 1 2 3 &
parallel -j0 -N0 --timeout 5 --nice 10 'bzip2 < /dev/zero >/dev/null' ::: 1 2 3 4 5 6 &
parallel --argsep ,, --joblog - -N0 parallel --load 100% echo ::: 1 ,, 1 |
parallel --colsep '\t' --header : echo '{=4 $_=$_>5=}'

8
testsuite/tests-to-run/parallel-local-0.3s.sh
View file

@ -247,6 +247,14 @@ echo 'bug #46231: {%} with --pipepart broken. Should give 1+2'
echo '**'
echo '{##} bug #45841: Replacement string for total no of jobs'
parallel --plus echo {##} ::: {a..j};
parallel -k 'echo {= $::G++ > 3 and ($_=$Global::JobQueue->total_jobs());=}' ::: {1..10}
echo '**'
EOF
echo '### 1 .par file from --files expected'
find /tmp{/*,}/*.{par,tms,tmx} 2>/dev/null -mmin -10 | wc -l

2
testsuite/tests-to-run/parallel-tutorial.sh
View file

@ -60,6 +60,8 @@ perl -ne '$/="\n\n"; /^Output/../^[^O]\S/ and next; /^ / and print;' ../../src/
s/tried 2\n//;
# Due to order is often mixed up
s/echo \d; exit \d\n/echo X; exit X\n/;
# Race condition causes outdir to sometime exist
s/(std(out|err):) Permission denied/$1 No such file or directory/;
'
# 3+3 .par files (from --files), 1 .tms-file from tmux attach
find {$TMPDIR,/var/tmp,/tmp}/{fif,tms,par[^a]}* -mmin -10 2>/dev/null | wc -l

25
testsuite/wanted-results/parallel-local-0.3s
View file

@ -519,5 +519,30 @@ bug #46231: {%} with --pipepart broken. Should give 1+2
1
echo '**'
**
echo '{##} bug #45841: Replacement string for total no of jobs'
{##} bug #45841: Replacement string for total no of jobs
parallel --plus echo {##} ::: {a..j}; parallel -k 'echo {= $::G++ > 3 and ($_=$Global::JobQueue->total_jobs());=}' ::: {1..10}
10
10
10
10
10
10
10
10
10
10
1
2
3
4
10
10
10
10
10
10
echo '**'
**
### 1 .par file from --files expected
1

2
testsuite/wanted-results/parallel-local4
View file

@ -1,11 +1,13 @@
bug #46120: Suspend should suspend (at least local) children
2048 0a:c0:70:5b:ec:f6:c2:de:67:c3:53:7f:29:81:65:54 tange@hk (RSA1)
1024 93:d1:21:c5:ff:e3:c3:be:6d:73:ea:aa:1e:a2:dc:06 /home/tange/.ssh/id_dsa (DSA)
8192 e1:95:e3:ff:99:a6:3a:b5:53:5a:54:59:d0:72:94:7f /home/tange/.ssh/id_rsa (RSA)
4096 94:2a:e3:cb:6b:66:63:21:13:51:8d:e8:4e:09:49:b2 /home/tange/.ssh/id_rsa_openindiana (RSA)
stdout /usr/bin/time -f CPUTIME=%U parallel --timeout 5 burnP6 ::: 1 | grep --colour=auto -q CPUTIME=1
Zero=OK 0
2048 0a:c0:70:5b:ec:f6:c2:de:67:c3:53:7f:29:81:65:54 tange@hk (RSA1)
1024 93:d1:21:c5:ff:e3:c3:be:6d:73:ea:aa:1e:a2:dc:06 /home/tange/.ssh/id_dsa (DSA)
8192 e1:95:e3:ff:99:a6:3a:b5:53:5a:54:59:d0:72:94:7f /home/tange/.ssh/id_rsa (RSA)
4096 94:2a:e3:cb:6b:66:63:21:13:51:8d:e8:4e:09:49:b2 /home/tange/.ssh/id_rsa_openindiana (RSA)

4
testsuite/wanted-results/parallel-polarhome
View file

@ -30,7 +30,7 @@ aix.polarhome.com Error in tempfile() using /XXXXXXXX.arg: Could not create temp
copy_and_test redhat.polarhome.com
redhat.polarhome.com ### Run the test on redhat.polarhome.com
redhat.polarhome.com Works on redhat.polarhome.com
redhat.polarhome.com Error in tempfile() using /XXXXXXXX.arg: Could not create temp file /XXXXXXXX.arg: Permission denied at bin/parallel line 0000
redhat.polarhome.com Error in tempfile() using template /XXXXXXXX.arg: Could not create temp file /XXXXXXXX.arg: Permission denied at bin/parallel line 0000.
copy_and_test hpux.polarhome.com
hpux.polarhome.com ### Run the test on hpux.polarhome.com
hpux.polarhome.com Works on hpux.polarhome.com
@ -118,6 +118,7 @@ centos.polarhome.com
debian
dragonfly.polarhome.com
freebsd.polarhome.com
hpux-ia64
hpux64
hurd
macosx
@ -129,6 +130,7 @@ openindiana
pidora
qnx
raspbian
redhat.polarhome.com
scosysv.polarhome.com
solaris-x86
solaris