tange/parallel
1
0
Fork 0
mirror of https://git.savannah.gnu.org/git/parallel.git synced 2024-11-22 14:07:55 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

parallel: Some Perl versions do not support substr($buf,0,$i) with $i > 2GB, so do this in 2GB chunks.

Browse source
This commit is contained in:
Ole Tange 2015-03-05 23:15:11 +01:00
parent 3c05c6815b
commit d4ab51beb6
1 changed files with 19 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

24
src/parallel
View file

@ -332,7 +332,7 @@ sub spreadstdin {
$anything_written += $anything_written +=
write_record_to_pipe($chunk_number++,\$header,\$buf, write_record_to_pipe($chunk_number++,\$header,\$buf,
$recstart,$recend,$last_newline_pos+1); $recstart,$recend,$last_newline_pos+1);
substr($buf,0,$last_newline_pos+1) = ""; shorten(\$buf,$last_newline_pos+1);
} elsif($opt::regexp) { } elsif($opt::regexp) {
if($Global::max_number_of_args) { if($Global::max_number_of_args) {
# -N => (start..*?end){n} # -N => (start..*?end){n}
@ -365,7 +365,7 @@ sub spreadstdin {
$anything_written += $anything_written +=
write_record_to_pipe($chunk_number++,\$header,\$buf, write_record_to_pipe($chunk_number++,\$header,\$buf,
$recstart,$recend,$i); $recstart,$recend,$i);
substr($buf,0,$i) = ""; shorten(\$buf,$i);
} }
} else { } else {
# Find the last recend+recstart in $buf # Find the last recend+recstart in $buf
@ -375,7 +375,7 @@ sub spreadstdin {
$anything_written += $anything_written +=
write_record_to_pipe($chunk_number++,\$header,\$buf, write_record_to_pipe($chunk_number++,\$header,\$buf,
$recstart,$recend,$i); $recstart,$recend,$i);
substr($buf,0,$i) = ""; shorten(\$buf,$i);
} }
} }
} }
@ -393,7 +393,6 @@ sub spreadstdin {
::debug("init", "Done reading input\n"); ::debug("init", "Done reading input\n");
# If there is anything left in the buffer write it # If there is anything left in the buffer write it
substr($buf,0,0) = "";
write_record_to_pipe($chunk_number++,\$header,\$buf,$recstart,$recend,length $buf); write_record_to_pipe($chunk_number++,\$header,\$buf,$recstart,$recend,length $buf);
$Global::start_no_new_jobs ||= 1; $Global::start_no_new_jobs ||= 1;
@ -583,6 +582,22 @@ sub rindex64 {
return -1; return -1;
} }
sub shorten {
# Do: substr($buf,0,$i) = "";
# Some Perl versions do not support $i > 2GB, so do this in 2GB chunks
# Input:
# $buf_ref = \$buf
# $i = position to shorten to
# Returns: N/A
my ($buf_ref, $i) = @_;
my $two_gb = 2**31-1;
while($i > $two_gb) {
substr($$buf_ref,0,$two_gb) = "";
$i -= $two_gb;
}
substr($$buf_ref,0,$i) = "";
}
sub write_record_to_pipe { sub write_record_to_pipe {
# Fork then # Fork then
# Write record from pos 0 .. $endpos to pipe # Write record from pos 0 .. $endpos to pipe
@ -6055,7 +6070,6 @@ sub base64_zip_eval {
return $script; return $script;
} }
sub sshlogin_wrap { sub sshlogin_wrap {
# Wrap the command with the commands needed to run remotely # Wrap the command with the commands needed to run remotely
# Input: # Input: