Small adjustments. Passes testsuite.

This commit is contained in:
Ole Tange 2013-02-14 23:08:17 +01:00
parent cf38b9078c
commit 21c62a2545
7 changed files with 34 additions and 12 deletions

View file

@ -308,14 +308,16 @@ sub spreadstdin {
} }
my $recendrecstart = $recend.$recstart; my $recendrecstart = $recend.$recstart;
my $chunk_number = 1; my $chunk_number = 1;
my $one_time_through;
my $blocksize = $opt::blocksize; my $blocksize = $opt::blocksize;
for my $in (@fhlist) { for my $in (@fhlist) {
piperead: while(1) { piperead: while(1) {
my $anything_written = 0; my $anything_written = 0;
if(not read($in,substr($buf,length $buf,0),$blocksize) if(not read($in,substr($buf,length $buf,0),$blocksize)) {
and $chunk_number != 1) { # End-of-file
$chunk_number != 1 and last;
# Force the while-loop once if everything was read by header reading # Force the while-loop once if everything was read by header reading
last; $one_time_through++ and last;
} }
if($opt::r) { if($opt::r) {
@ -344,15 +346,19 @@ sub spreadstdin {
# -L -N => (start..*?end){n*l} # -L -N => (start..*?end){n*l}
my $read_n_lines = $Global::max_number_of_args * ($Global::max_lines || 1); my $read_n_lines = $Global::max_number_of_args * ($Global::max_lines || 1);
while($buf =~ s/((?:$recstart.*?$recend){$read_n_lines})($recstart.*)$/$2/os) { while($buf =~ s/((?:$recstart.*?$recend){$read_n_lines})($recstart.*)$/$2/os) {
# Copy to modifiable variable
my $b = $1;
$anything_written += $anything_written +=
write_record_to_pipe($chunk_number++,\$header,\$1, write_record_to_pipe($chunk_number++,\$header,\$b,
$recstart,$recend,length $1); $recstart,$recend,length $1);
} }
} else { } else {
# Find the last recend-recstart in $buf # Find the last recend-recstart in $buf
if($buf =~ s/(.*$recend)($recstart.*?)$/$2/os) { if($buf =~ s/(.*$recend)($recstart.*?)$/$2/os) {
# Copy to modifiable variable
my $b = $1;
$anything_written += $anything_written +=
write_record_to_pipe($chunk_number++,\$header,\$1, write_record_to_pipe($chunk_number++,\$header,\$b,
$recstart,$recend,length $1); $recstart,$recend,length $1);
} }
} }
@ -374,13 +380,13 @@ sub spreadstdin {
if($i != -1) { if($i != -1) {
$i += length $recend; # find the actual splitting location $i += length $recend; # find the actual splitting location
$anything_written += $anything_written +=
write_record_to_pipe($chunk_number++,\$header, write_record_to_pipe($chunk_number++,\$header,\$buf,
\$buf,$recstart,$recend,$i); $recstart,$recend,$i);
substr($buf,0,$i) = ""; substr($buf,0,$i) = "";
} }
} }
} }
if(not $anything_written) { if(not $anything_written and not eof($in)) {
# Nothing was written - maybe the block size < record size? # Nothing was written - maybe the block size < record size?
# Increase blocksize exponentially # Increase blocksize exponentially
$blocksize = ceil($blocksize * 1.3); $blocksize = ceil($blocksize * 1.3);

View file

@ -1,4 +1,4 @@
testsuite: ../src/parallel tests-to-run/* wanted-results/* startdb prereqlocal prereqremote testsuite: ../src/parallel tests-to-run/* wanted-results/* prereqlocal startdb prereqremote
time sh Start.sh time sh Start.sh
date date

View file

@ -44,7 +44,7 @@ echo '### Test output is the same for different block size'
echo -n 01a02a0a0a12a34a45a6a | echo -n 01a02a0a0a12a34a45a6a |
parallel -k -j1 --blocksize 100 --pipe --recend a -N 3 'echo -n "$PARALLEL_SEQ>"; cat; echo; sleep 0.1' parallel -k -j1 --blocksize 100 --pipe --recend a -N 3 'echo -n "$PARALLEL_SEQ>"; cat; echo; sleep 0.1'
echo -n 01a02a0a0a12a34a45a6a | echo -n 01a02a0a0a12a34a45a6a |
parallel -k -j1 --blocksize 1 --pipe --recend a -N 3 'echo -n "$PARALLEL_SEQ>"; cat; echo; sleep 0.1' stdout parallel -k -j1 --blocksize 1 --pipe --recend a -N 3 'echo -n "$PARALLEL_SEQ>"; cat; echo; sleep 0.1'
echo '### Test 10M records with too big block'; echo '### Test 10M records with too big block';
( (

View file

@ -1,7 +1,7 @@
### Test if we can deal with output > 4 GB ### Test if we can deal with output > 4 GB
46a318993dfc8e2afd71ff2bc6f605f1 - 46a318993dfc8e2afd71ff2bc6f605f1 -
### Test Force outside the file handle limit, 2009-02-17 Gave fork error ### Test Force outside the file handle limit, 2009-02-17 Gave fork error
parallel: Warning: Only enough filehandles to run 507 jobs in parallel. Raising ulimit -n may help. parallel: Warning: Only enough filehandles to run 506 jobs in parallel. Raising ulimit -n may help.
Start Start
end end
### Test of --retries on unreachable host ### Test of --retries on unreachable host

View file

@ -222,3 +222,14 @@ job3 val 2
job4 val 3 job4 val 3
job6 val 1 job6 val 1
try 2 again. Gives empty try 2 again. Gives empty
### --resume -k
job0id
job1id
job2id
job3id
job0id
job5id
try 2 = nothing
two extra
job6id
job7id

View file

@ -1,5 +1,6 @@
### Test --pipe ### Test --pipe
### Test 200M records with too small block ### Test 200M records with too small block
parallel: Warning: A full record was not matched in a block. Increasing to --blocksize 260000000
303111434 303111434
303111434 303111434
303111434 303111434
@ -126,7 +127,11 @@ c
1>01a02a0a 1>01a02a0a
2>0a12a34a 2>0a12a34a
3>45a6a 3>45a6a
parallel: Warning: A full record was not matched in a block. Increasing to --blocksize 2
parallel: Warning: A full record was not matched in a block. Increasing to --blocksize 3
parallel: Warning: A full record was not matched in a block. Increasing to --blocksize 4
1>01a02a0a 1>01a02a0a
parallel: Warning: A full record was not matched in a block. Increasing to --blocksize 6
2>0a12a34a 2>0a12a34a
3>45a6a 3>45a6a
### Test 10M records with too big block ### Test 10M records with too big block

View file

@ -1,5 +1,5 @@
### Test -k ### Test -k
parallel: Warning: Only enough filehandles to run 20 jobs in parallel. Raising ulimit -n may help. parallel: Warning: Only enough filehandles to run 19 jobs in parallel. Raising ulimit -n may help.
begin begin
1 1
2 2