tange/parallel
1
0
Fork 0
mirror of https://git.savannah.gnu.org/git/parallel.git synced 2024-11-26 07:57:58 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fixed bug #44357: Some large file support is broken.

Browse source
This commit is contained in:
Ole Tange 2015-02-24 02:28:49 +01:00
parent e22467f4dd
commit c445232b23
1 changed files with 33 additions and 11 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

36
src/parallel
View file

@ -301,7 +301,8 @@ sub spreadstdin {
my $recendrecstart = $recend.$recstart; my $recendrecstart = $recend.$recstart;
my $chunk_number = 1; my $chunk_number = 1;
my $one_time_through; my $one_time_through;
my $blocksize = $opt::blocksize; my $two_gb = (2<<30)-1;
my $blocksize = ::min($opt::blocksize,$two_gb);
my $in = *STDIN; my $in = *STDIN;
my $header = find_header(\$buf,$in); my $header = find_header(\$buf,$in);
while(1) { while(1) {
@ -367,8 +368,19 @@ sub spreadstdin {
substr($buf,0,$i) = ""; substr($buf,0,$i) = "";
} }
} else { } else {
# Find the last recend-recstart in $buf my $i;
my $i = rindex($buf,$recendrecstart); if(length $buf < $two_gb) {
# Find the last recend+recstart in $buf
$i = rindex($buf,$recendrecstart);
} else {
# Find the last recend+recstart in the last 2 GB of $buf
# rindex does not work on > 2GB
my $over2gb = (length $buf)-$two_gb;
$i = rindex(substr($buf,$over2gb,$two_gb),$recendrecstart);
if($i != -1) {
$i += $over2gb;
}
}
if($i != -1) { if($i != -1) {
$i += length $recend; # find the actual splitting location $i += length $recend; # find the actual splitting location
$anything_written += $anything_written +=
@ -380,13 +392,15 @@ sub spreadstdin {
} }
if(not $anything_written and not eof($in)) { if(not $anything_written and not eof($in)) {
# Nothing was written - maybe the block size < record size? # Nothing was written - maybe the block size < record size?
# Increase blocksize exponentially # Increase blocksize exponentially up to 2GB-1 (2GB causes problems)
if($blocksize < $two_gb) {
my $old_blocksize = $blocksize; my $old_blocksize = $blocksize;
$blocksize = ceil($blocksize * 1.3 + 1); $blocksize = ::min(ceil($blocksize * 1.3 + 1), $two_gb);
::warning("A record was longer than $old_blocksize. " . ::warning("A record was longer than $old_blocksize. " .
"Increasing to --blocksize $blocksize\n"); "Increasing to --blocksize $blocksize\n");
} }
} }
}
::debug("init", "Done reading input\n"); ::debug("init", "Done reading input\n");
# If there is anything left in the buffer write it # If there is anything left in the buffer write it
@ -953,7 +967,7 @@ sub parse_options {
sub init_globals { sub init_globals {
# Defaults: # Defaults:
$Global::version = 20150223; $Global::version = 20150224;
$Global::progname = 'parallel'; $Global::progname = 'parallel';
$Global::infinity = 2**31; $Global::infinity = 2**31;
$Global::debug = 0; $Global::debug = 0;
@ -5485,7 +5499,15 @@ sub write {
my $self = shift; my $self = shift;
my $remaining_ref = shift; my $remaining_ref = shift;
my $stdin_fh = $self->fh(0,"w"); my $stdin_fh = $self->fh(0,"w");
syswrite($stdin_fh,$$remaining_ref);
my $len = length $$remaining_ref;
# syswrite may not write all in one go,
# so make sure everything is written.
while($len) {
my $written = syswrite($stdin_fh,$$remaining_ref);
substr($$remaining_ref,0,$written) = "";
$len -= $written;
}
} }
sub set_stdin_buffer { sub set_stdin_buffer {