tange/parallel
1
0
Fork 0
mirror of https://git.savannah.gnu.org/git/parallel.git synced 2024-12-22 20:57:53 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

parallel: Less memory leak (now around 40 bytes/job).

Browse source
This commit is contained in:
Ole Tange 2016-08-13 19:17:44 +02:00
parent a7b50a054e
commit e94774df14
1 changed files with 16 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

30
src/parallel
View file

@ -1550,10 +1550,10 @@ sub find_compression_program {
# $compress_program = compress program with options
# $decompress_program = decompress program with options
# Search for these. Sorted by speed on 16 core
# parallel -j1 --joblog jl --arg-sep , parallel --compress-program \'{3}" "-{2}\' cat ::: gz '>'/dev/null , 1 2 3 , {1..3} , lz4 lzop pigz pxz gzip plzip pbzip2 lzma xz lzip bzip2
# Search for these. Sorted by speed on 20 core
# parallel -j1 --joblog jl --arg-sep , parallel --compress-program \'{3}" "-{2}\' cat ::: gz '>'/dev/null , 1 2 3 , {1..3} , zstd clzip lz4 lzop pigz pxz gzip plzip pbzip2 lzma xz lzip bzip2
# sort -nk4 jl
my @prg = qw(lz4 pigz lzop plzip pbzip2 pxz gzip lzma xz bzip2 lzip);
my @prg = qw(lz4 lzop pbzip2 pigz plzip lzip gzip pxz lzma xz bzip2);
for my $p (@prg) {
if(which($p)) {
return ("$p -c -1","$p -dc");
@ -4311,6 +4311,13 @@ sub reap_usleep {
# $ms*1.1 if no children reaped
my $ms = shift;
if(reaper()) {
if(not $Global::total_completed % 100) {
if($opt::timeout) {
# Force cleaning the timeout queue for every 1000 jobs
# Fixes potential memleak
$Global::timeoutq->process_timeouts();
}
}
# Sleep exponentially shorter (1/2^n) if a job finished
return $ms/2+0.001;
} else {
@ -6278,7 +6285,7 @@ sub slot {
sub free_slot {
my $self = shift;
return $self->{'commandline'}->free_slot();
push @Global::slots, $self->slot();
}
{
@ -8479,28 +8486,23 @@ sub set_seq {
{
my $max_slot_number;
my @slots;
sub slot {
# Find the number of a free job slot and return it
# Uses:
# @Global::slots - list with free jobslots
# Returns:
# $jobslot = number of jobslot
my $self = shift;
if(not $self->{'slot'}) {
if(not @slots) {
if(not @Global::slots) {
# $max_slot_number will typically be $Global::max_jobs_running
push @slots, ++$max_slot_number;
push @Global::slots, ++$Global::max_slot_number;
}
$self->{'slot'} = shift @slots;
$self->{'slot'} = shift @Global::slots;
}
return $self->{'slot'};
}
sub free_slot {
# Make this slot available again
my $self = shift;
push @slots, $self->{'slot'};
}
}
sub populate {