Small adjustments. Passes testsuite.

This commit is contained in:
Ole Tange 2013-02-14 23:08:17 +01:00
parent cf38b9078c
commit 21c62a2545
7 changed files with 34 additions and 12 deletions

View file

@ -308,14 +308,16 @@ sub spreadstdin {
}
my $recendrecstart = $recend.$recstart;
my $chunk_number = 1;
my $one_time_through;
my $blocksize = $opt::blocksize;
for my $in (@fhlist) {
piperead: while(1) {
my $anything_written = 0;
if(not read($in,substr($buf,length $buf,0),$blocksize)
and $chunk_number != 1) {
if(not read($in,substr($buf,length $buf,0),$blocksize)) {
# End-of-file
$chunk_number != 1 and last;
# Force the while-loop once if everything was read by header reading
last;
$one_time_through++ and last;
}
if($opt::r) {
@ -344,15 +346,19 @@ sub spreadstdin {
# -L -N => (start..*?end){n*l}
my $read_n_lines = $Global::max_number_of_args * ($Global::max_lines || 1);
while($buf =~ s/((?:$recstart.*?$recend){$read_n_lines})($recstart.*)$/$2/os) {
# Copy to modifiable variable
my $b = $1;
$anything_written +=
write_record_to_pipe($chunk_number++,\$header,\$1,
write_record_to_pipe($chunk_number++,\$header,\$b,
$recstart,$recend,length $1);
}
} else {
# Find the last recend-recstart in $buf
if($buf =~ s/(.*$recend)($recstart.*?)$/$2/os) {
# Copy to modifiable variable
my $b = $1;
$anything_written +=
write_record_to_pipe($chunk_number++,\$header,\$1,
write_record_to_pipe($chunk_number++,\$header,\$b,
$recstart,$recend,length $1);
}
}
@ -374,13 +380,13 @@ sub spreadstdin {
if($i != -1) {
$i += length $recend; # find the actual splitting location
$anything_written +=
write_record_to_pipe($chunk_number++,\$header,
\$buf,$recstart,$recend,$i);
write_record_to_pipe($chunk_number++,\$header,\$buf,
$recstart,$recend,$i);
substr($buf,0,$i) = "";
}
}
}
if(not $anything_written) {
if(not $anything_written and not eof($in)) {
# Nothing was written - maybe the block size < record size?
# Increase blocksize exponentially
$blocksize = ceil($blocksize * 1.3);

View file

@ -1,4 +1,4 @@
testsuite: ../src/parallel tests-to-run/* wanted-results/* startdb prereqlocal prereqremote
testsuite: ../src/parallel tests-to-run/* wanted-results/* prereqlocal startdb prereqremote
time sh Start.sh
date

View file

@ -44,7 +44,7 @@ echo '### Test output is the same for different block size'
echo -n 01a02a0a0a12a34a45a6a |
parallel -k -j1 --blocksize 100 --pipe --recend a -N 3 'echo -n "$PARALLEL_SEQ>"; cat; echo; sleep 0.1'
echo -n 01a02a0a0a12a34a45a6a |
parallel -k -j1 --blocksize 1 --pipe --recend a -N 3 'echo -n "$PARALLEL_SEQ>"; cat; echo; sleep 0.1'
stdout parallel -k -j1 --blocksize 1 --pipe --recend a -N 3 'echo -n "$PARALLEL_SEQ>"; cat; echo; sleep 0.1'
echo '### Test 10M records with too big block';
(

View file

@ -1,7 +1,7 @@
### Test if we can deal with output > 4 GB
46a318993dfc8e2afd71ff2bc6f605f1 -
### Test Force outside the file handle limit, 2009-02-17 Gave fork error
parallel: Warning: Only enough filehandles to run 507 jobs in parallel. Raising ulimit -n may help.
parallel: Warning: Only enough filehandles to run 506 jobs in parallel. Raising ulimit -n may help.
Start
end
### Test of --retries on unreachable host

View file

@ -222,3 +222,14 @@ job3 val 2
job4 val 3
job6 val 1
try 2 again. Gives empty
### --resume -k
job0id
job1id
job2id
job3id
job0id
job5id
try 2 = nothing
two extra
job6id
job7id

View file

@ -1,5 +1,6 @@
### Test --pipe
### Test 200M records with too small block
parallel: Warning: A full record was not matched in a block. Increasing to --blocksize 260000000
303111434
303111434
303111434
@ -126,7 +127,11 @@ c
1>01a02a0a
2>0a12a34a
3>45a6a
parallel: Warning: A full record was not matched in a block. Increasing to --blocksize 2
parallel: Warning: A full record was not matched in a block. Increasing to --blocksize 3
parallel: Warning: A full record was not matched in a block. Increasing to --blocksize 4
1>01a02a0a
parallel: Warning: A full record was not matched in a block. Increasing to --blocksize 6
2>0a12a34a
3>45a6a
### Test 10M records with too big block

View file

@ -1,5 +1,5 @@
### Test -k
parallel: Warning: Only enough filehandles to run 20 jobs in parallel. Raising ulimit -n may help.
parallel: Warning: Only enough filehandles to run 19 jobs in parallel. Raising ulimit -n may help.
begin
1
2