mirror of
https://git.savannah.gnu.org/git/parallel.git
synced 2024-11-25 23:47:53 +00:00
Small adjustments. Passes testsuite.
This commit is contained in:
parent
cf38b9078c
commit
21c62a2545
22
src/parallel
22
src/parallel
|
@ -308,14 +308,16 @@ sub spreadstdin {
|
||||||
}
|
}
|
||||||
my $recendrecstart = $recend.$recstart;
|
my $recendrecstart = $recend.$recstart;
|
||||||
my $chunk_number = 1;
|
my $chunk_number = 1;
|
||||||
|
my $one_time_through;
|
||||||
my $blocksize = $opt::blocksize;
|
my $blocksize = $opt::blocksize;
|
||||||
for my $in (@fhlist) {
|
for my $in (@fhlist) {
|
||||||
piperead: while(1) {
|
piperead: while(1) {
|
||||||
my $anything_written = 0;
|
my $anything_written = 0;
|
||||||
if(not read($in,substr($buf,length $buf,0),$blocksize)
|
if(not read($in,substr($buf,length $buf,0),$blocksize)) {
|
||||||
and $chunk_number != 1) {
|
# End-of-file
|
||||||
|
$chunk_number != 1 and last;
|
||||||
# Force the while-loop once if everything was read by header reading
|
# Force the while-loop once if everything was read by header reading
|
||||||
last;
|
$one_time_through++ and last;
|
||||||
}
|
}
|
||||||
|
|
||||||
if($opt::r) {
|
if($opt::r) {
|
||||||
|
@ -344,15 +346,19 @@ sub spreadstdin {
|
||||||
# -L -N => (start..*?end){n*l}
|
# -L -N => (start..*?end){n*l}
|
||||||
my $read_n_lines = $Global::max_number_of_args * ($Global::max_lines || 1);
|
my $read_n_lines = $Global::max_number_of_args * ($Global::max_lines || 1);
|
||||||
while($buf =~ s/((?:$recstart.*?$recend){$read_n_lines})($recstart.*)$/$2/os) {
|
while($buf =~ s/((?:$recstart.*?$recend){$read_n_lines})($recstart.*)$/$2/os) {
|
||||||
|
# Copy to modifiable variable
|
||||||
|
my $b = $1;
|
||||||
$anything_written +=
|
$anything_written +=
|
||||||
write_record_to_pipe($chunk_number++,\$header,\$1,
|
write_record_to_pipe($chunk_number++,\$header,\$b,
|
||||||
$recstart,$recend,length $1);
|
$recstart,$recend,length $1);
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
# Find the last recend-recstart in $buf
|
# Find the last recend-recstart in $buf
|
||||||
if($buf =~ s/(.*$recend)($recstart.*?)$/$2/os) {
|
if($buf =~ s/(.*$recend)($recstart.*?)$/$2/os) {
|
||||||
|
# Copy to modifiable variable
|
||||||
|
my $b = $1;
|
||||||
$anything_written +=
|
$anything_written +=
|
||||||
write_record_to_pipe($chunk_number++,\$header,\$1,
|
write_record_to_pipe($chunk_number++,\$header,\$b,
|
||||||
$recstart,$recend,length $1);
|
$recstart,$recend,length $1);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -374,13 +380,13 @@ sub spreadstdin {
|
||||||
if($i != -1) {
|
if($i != -1) {
|
||||||
$i += length $recend; # find the actual splitting location
|
$i += length $recend; # find the actual splitting location
|
||||||
$anything_written +=
|
$anything_written +=
|
||||||
write_record_to_pipe($chunk_number++,\$header,
|
write_record_to_pipe($chunk_number++,\$header,\$buf,
|
||||||
\$buf,$recstart,$recend,$i);
|
$recstart,$recend,$i);
|
||||||
substr($buf,0,$i) = "";
|
substr($buf,0,$i) = "";
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if(not $anything_written) {
|
if(not $anything_written and not eof($in)) {
|
||||||
# Nothing was written - maybe the block size < record size?
|
# Nothing was written - maybe the block size < record size?
|
||||||
# Increase blocksize exponentially
|
# Increase blocksize exponentially
|
||||||
$blocksize = ceil($blocksize * 1.3);
|
$blocksize = ceil($blocksize * 1.3);
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
testsuite: ../src/parallel tests-to-run/* wanted-results/* startdb prereqlocal prereqremote
|
testsuite: ../src/parallel tests-to-run/* wanted-results/* prereqlocal startdb prereqremote
|
||||||
time sh Start.sh
|
time sh Start.sh
|
||||||
date
|
date
|
||||||
|
|
||||||
|
|
|
@ -44,7 +44,7 @@ echo '### Test output is the same for different block size'
|
||||||
echo -n 01a02a0a0a12a34a45a6a |
|
echo -n 01a02a0a0a12a34a45a6a |
|
||||||
parallel -k -j1 --blocksize 100 --pipe --recend a -N 3 'echo -n "$PARALLEL_SEQ>"; cat; echo; sleep 0.1'
|
parallel -k -j1 --blocksize 100 --pipe --recend a -N 3 'echo -n "$PARALLEL_SEQ>"; cat; echo; sleep 0.1'
|
||||||
echo -n 01a02a0a0a12a34a45a6a |
|
echo -n 01a02a0a0a12a34a45a6a |
|
||||||
parallel -k -j1 --blocksize 1 --pipe --recend a -N 3 'echo -n "$PARALLEL_SEQ>"; cat; echo; sleep 0.1'
|
stdout parallel -k -j1 --blocksize 1 --pipe --recend a -N 3 'echo -n "$PARALLEL_SEQ>"; cat; echo; sleep 0.1'
|
||||||
|
|
||||||
echo '### Test 10M records with too big block';
|
echo '### Test 10M records with too big block';
|
||||||
(
|
(
|
||||||
|
|
|
@ -1,7 +1,7 @@
|
||||||
### Test if we can deal with output > 4 GB
|
### Test if we can deal with output > 4 GB
|
||||||
46a318993dfc8e2afd71ff2bc6f605f1 -
|
46a318993dfc8e2afd71ff2bc6f605f1 -
|
||||||
### Test Force outside the file handle limit, 2009-02-17 Gave fork error
|
### Test Force outside the file handle limit, 2009-02-17 Gave fork error
|
||||||
parallel: Warning: Only enough filehandles to run 507 jobs in parallel. Raising ulimit -n may help.
|
parallel: Warning: Only enough filehandles to run 506 jobs in parallel. Raising ulimit -n may help.
|
||||||
Start
|
Start
|
||||||
end
|
end
|
||||||
### Test of --retries on unreachable host
|
### Test of --retries on unreachable host
|
||||||
|
|
|
@ -222,3 +222,14 @@ job3 val 2
|
||||||
job4 val 3
|
job4 val 3
|
||||||
job6 val 1
|
job6 val 1
|
||||||
try 2 again. Gives empty
|
try 2 again. Gives empty
|
||||||
|
### --resume -k
|
||||||
|
job0id
|
||||||
|
job1id
|
||||||
|
job2id
|
||||||
|
job3id
|
||||||
|
job0id
|
||||||
|
job5id
|
||||||
|
try 2 = nothing
|
||||||
|
two extra
|
||||||
|
job6id
|
||||||
|
job7id
|
||||||
|
|
|
@ -1,5 +1,6 @@
|
||||||
### Test --pipe
|
### Test --pipe
|
||||||
### Test 200M records with too small block
|
### Test 200M records with too small block
|
||||||
|
parallel: Warning: A full record was not matched in a block. Increasing to --blocksize 260000000
|
||||||
303111434
|
303111434
|
||||||
303111434
|
303111434
|
||||||
303111434
|
303111434
|
||||||
|
@ -126,7 +127,11 @@ c
|
||||||
1>01a02a0a
|
1>01a02a0a
|
||||||
2>0a12a34a
|
2>0a12a34a
|
||||||
3>45a6a
|
3>45a6a
|
||||||
|
parallel: Warning: A full record was not matched in a block. Increasing to --blocksize 2
|
||||||
|
parallel: Warning: A full record was not matched in a block. Increasing to --blocksize 3
|
||||||
|
parallel: Warning: A full record was not matched in a block. Increasing to --blocksize 4
|
||||||
1>01a02a0a
|
1>01a02a0a
|
||||||
|
parallel: Warning: A full record was not matched in a block. Increasing to --blocksize 6
|
||||||
2>0a12a34a
|
2>0a12a34a
|
||||||
3>45a6a
|
3>45a6a
|
||||||
### Test 10M records with too big block
|
### Test 10M records with too big block
|
||||||
|
|
|
@ -1,5 +1,5 @@
|
||||||
### Test -k
|
### Test -k
|
||||||
parallel: Warning: Only enough filehandles to run 20 jobs in parallel. Raising ulimit -n may help.
|
parallel: Warning: Only enough filehandles to run 19 jobs in parallel. Raising ulimit -n may help.
|
||||||
begin
|
begin
|
||||||
1
|
1
|
||||||
2
|
2
|
||||||
|
|
Loading…
Reference in a new issue