tange/parallel
1
0
Fork 0
mirror of https://git.savannah.gnu.org/git/parallel.git synced 2024-11-22 14:07:55 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

parallel: Less memory leak (now around 40 bytes/job).

Browse source
This commit is contained in:
Ole Tange 2016-08-13 19:17:44 +02:00
parent a7b50a054e
commit e94774df14
1 changed files with 16 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

30
src/parallel
View file

@ -1550,10 +1550,10 @@ sub find_compression_program {
# $compress_program = compress program with options # $compress_program = compress program with options
# $decompress_program = decompress program with options # $decompress_program = decompress program with options
# Search for these. Sorted by speed on 16 core # Search for these. Sorted by speed on 20 core
# parallel -j1 --joblog jl --arg-sep , parallel --compress-program \'{3}" "-{2}\' cat ::: gz '>'/dev/null , 1 2 3 , {1..3} , lz4 lzop pigz pxz gzip plzip pbzip2 lzma xz lzip bzip2 # parallel -j1 --joblog jl --arg-sep , parallel --compress-program \'{3}" "-{2}\' cat ::: gz '>'/dev/null , 1 2 3 , {1..3} , zstd clzip lz4 lzop pigz pxz gzip plzip pbzip2 lzma xz lzip bzip2
# sort -nk4 jl # sort -nk4 jl
my @prg = qw(lz4 pigz lzop plzip pbzip2 pxz gzip lzma xz bzip2 lzip); my @prg = qw(lz4 lzop pbzip2 pigz plzip lzip gzip pxz lzma xz bzip2);
for my $p (@prg) { for my $p (@prg) {
if(which($p)) { if(which($p)) {
return ("$p -c -1","$p -dc"); return ("$p -c -1","$p -dc");
@ -4311,6 +4311,13 @@ sub reap_usleep {
# $ms*1.1 if no children reaped # $ms*1.1 if no children reaped
my $ms = shift; my $ms = shift;
if(reaper()) { if(reaper()) {
if(not $Global::total_completed % 100) {
if($opt::timeout) {
# Force cleaning the timeout queue for every 1000 jobs
# Fixes potential memleak
$Global::timeoutq->process_timeouts();
}
}
# Sleep exponentially shorter (1/2^n) if a job finished # Sleep exponentially shorter (1/2^n) if a job finished
return $ms/2+0.001; return $ms/2+0.001;
} else { } else {
@ -6278,7 +6285,7 @@ sub slot {
sub free_slot { sub free_slot {
my $self = shift; my $self = shift;
return $self->{'commandline'}->free_slot(); push @Global::slots, $self->slot();
} }
{ {
@ -8479,28 +8486,23 @@ sub set_seq {
{ {
my $max_slot_number; my $max_slot_number;
my @slots;
sub slot { sub slot {
# Find the number of a free job slot and return it # Find the number of a free job slot and return it
# Uses:
# @Global::slots - list with free jobslots
# Returns: # Returns:
# $jobslot = number of jobslot # $jobslot = number of jobslot
my $self = shift; my $self = shift;
if(not $self->{'slot'}) { if(not $self->{'slot'}) {
if(not @slots) { if(not @Global::slots) {
# $max_slot_number will typically be $Global::max_jobs_running # $max_slot_number will typically be $Global::max_jobs_running
push @slots, ++$max_slot_number; push @Global::slots, ++$Global::max_slot_number;
} }
$self->{'slot'} = shift @slots; $self->{'slot'} = shift @Global::slots;
} }
return $self->{'slot'}; return $self->{'slot'};
} }
sub free_slot {
# Make this slot available again
my $self = shift;
push @slots, $self->{'slot'};
}
} }
sub populate { sub populate {