diff --git a/src/parallel b/src/parallel index 6a087782..1a260e8c 100755 --- a/src/parallel +++ b/src/parallel @@ -308,14 +308,16 @@ sub spreadstdin { } my $recendrecstart = $recend.$recstart; my $chunk_number = 1; + my $one_time_through; my $blocksize = $opt::blocksize; for my $in (@fhlist) { piperead: while(1) { my $anything_written = 0; - if(not read($in,substr($buf,length $buf,0),$blocksize) - and $chunk_number != 1) { + if(not read($in,substr($buf,length $buf,0),$blocksize)) { + # End-of-file + $chunk_number != 1 and last; # Force the while-loop once if everything was read by header reading - last; + $one_time_through++ and last; } if($opt::r) { @@ -344,15 +346,19 @@ sub spreadstdin { # -L -N => (start..*?end){n*l} my $read_n_lines = $Global::max_number_of_args * ($Global::max_lines || 1); while($buf =~ s/((?:$recstart.*?$recend){$read_n_lines})($recstart.*)$/$2/os) { + # Copy to modifiable variable + my $b = $1; $anything_written += - write_record_to_pipe($chunk_number++,\$header,\$1, + write_record_to_pipe($chunk_number++,\$header,\$b, $recstart,$recend,length $1); } } else { # Find the last recend-recstart in $buf if($buf =~ s/(.*$recend)($recstart.*?)$/$2/os) { + # Copy to modifiable variable + my $b = $1; $anything_written += - write_record_to_pipe($chunk_number++,\$header,\$1, + write_record_to_pipe($chunk_number++,\$header,\$b, $recstart,$recend,length $1); } } @@ -374,13 +380,13 @@ sub spreadstdin { if($i != -1) { $i += length $recend; # find the actual splitting location $anything_written += - write_record_to_pipe($chunk_number++,\$header, - \$buf,$recstart,$recend,$i); + write_record_to_pipe($chunk_number++,\$header,\$buf, + $recstart,$recend,$i); substr($buf,0,$i) = ""; } } } - if(not $anything_written) { + if(not $anything_written and not eof($in)) { # Nothing was written - maybe the block size < record size? # Increase blocksize exponentially $blocksize = ceil($blocksize * 1.3); diff --git a/testsuite/Makefile b/testsuite/Makefile index 37c12afe..944ae357 100644 --- a/testsuite/Makefile +++ b/testsuite/Makefile @@ -1,4 +1,4 @@ -testsuite: ../src/parallel tests-to-run/* wanted-results/* startdb prereqlocal prereqremote +testsuite: ../src/parallel tests-to-run/* wanted-results/* prereqlocal startdb prereqremote time sh Start.sh date diff --git a/testsuite/tests-to-run/parallel-local5.sh b/testsuite/tests-to-run/parallel-local5.sh index b634961b..b65b7b9b 100644 --- a/testsuite/tests-to-run/parallel-local5.sh +++ b/testsuite/tests-to-run/parallel-local5.sh @@ -44,7 +44,7 @@ echo '### Test output is the same for different block size' echo -n 01a02a0a0a12a34a45a6a | parallel -k -j1 --blocksize 100 --pipe --recend a -N 3 'echo -n "$PARALLEL_SEQ>"; cat; echo; sleep 0.1' echo -n 01a02a0a0a12a34a45a6a | - parallel -k -j1 --blocksize 1 --pipe --recend a -N 3 'echo -n "$PARALLEL_SEQ>"; cat; echo; sleep 0.1' + stdout parallel -k -j1 --blocksize 1 --pipe --recend a -N 3 'echo -n "$PARALLEL_SEQ>"; cat; echo; sleep 0.1' echo '### Test 10M records with too big block'; ( diff --git a/testsuite/wanted-results/parallel-local11 b/testsuite/wanted-results/parallel-local11 index fbacaa44..f91003ba 100644 --- a/testsuite/wanted-results/parallel-local11 +++ b/testsuite/wanted-results/parallel-local11 @@ -1,7 +1,7 @@ ### Test if we can deal with output > 4 GB 46a318993dfc8e2afd71ff2bc6f605f1 - ### Test Force outside the file handle limit, 2009-02-17 Gave fork error -parallel: Warning: Only enough filehandles to run 507 jobs in parallel. Raising ulimit -n may help. +parallel: Warning: Only enough filehandles to run 506 jobs in parallel. Raising ulimit -n may help. Start end ### Test of --retries on unreachable host diff --git a/testsuite/wanted-results/parallel-local164 b/testsuite/wanted-results/parallel-local164 index 74d622d8..b2ac82f6 100644 --- a/testsuite/wanted-results/parallel-local164 +++ b/testsuite/wanted-results/parallel-local164 @@ -222,3 +222,14 @@ job3 val 2 job4 val 3 job6 val 1 try 2 again. Gives empty +### --resume -k +job0id +job1id +job2id +job3id +job0id +job5id +try 2 = nothing +two extra +job6id +job7id diff --git a/testsuite/wanted-results/parallel-local5 b/testsuite/wanted-results/parallel-local5 index b4f75260..4b812247 100644 --- a/testsuite/wanted-results/parallel-local5 +++ b/testsuite/wanted-results/parallel-local5 @@ -1,5 +1,6 @@ ### Test --pipe ### Test 200M records with too small block +parallel: Warning: A full record was not matched in a block. Increasing to --blocksize 260000000 303111434 303111434 303111434 @@ -126,7 +127,11 @@ c 1>01a02a0a 2>0a12a34a 3>45a6a +parallel: Warning: A full record was not matched in a block. Increasing to --blocksize 2 +parallel: Warning: A full record was not matched in a block. Increasing to --blocksize 3 +parallel: Warning: A full record was not matched in a block. Increasing to --blocksize 4 1>01a02a0a +parallel: Warning: A full record was not matched in a block. Increasing to --blocksize 6 2>0a12a34a 3>45a6a ### Test 10M records with too big block diff --git a/testsuite/wanted-results/test13 b/testsuite/wanted-results/test13 index c18ee2e5..2ae8b3db 100644 --- a/testsuite/wanted-results/test13 +++ b/testsuite/wanted-results/test13 @@ -1,5 +1,5 @@ ### Test -k -parallel: Warning: Only enough filehandles to run 20 jobs in parallel. Raising ulimit -n may help. +parallel: Warning: Only enough filehandles to run 19 jobs in parallel. Raising ulimit -n may help. begin 1 2