From fd6abf1f9a0aa5014912448b403bde5fb95d0616 Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Sun, 14 Aug 2016 16:54:27 +0200 Subject: [PATCH] parallel: Removed memleak. --- src/parallel | 54 ++++++++++--------- testsuite/tests-to-run/parallel-local-100s.sh | 46 ++++++++++++++++ testsuite/wanted-results/parallel-local-100s | 7 ++- testsuite/wanted-results/parallel-local-10s | 2 +- testsuite/wanted-results/parallel-local164 | 2 +- 5 files changed, 82 insertions(+), 29 deletions(-) diff --git a/src/parallel b/src/parallel index e1f1982c..fde99d47 100755 --- a/src/parallel +++ b/src/parallel @@ -3502,12 +3502,17 @@ sub reaper { # @pids_reaped = PIDs of children finished my $stiff; my @pids_reaped; + my $children_reaped = 0; debug("run", "Reaper "); # For efficiency surround with BEGIN/COMMIT when using $opt::sqlmaster $opt::sqlmaster and $Global::sql->run("BEGIN;"); while (($stiff = waitpid(-1, &WNOHANG)) > 0) { # $stiff = pid of dead process - push(@pids_reaped,$stiff); + if(wantarray) { + push(@pids_reaped,$stiff); + } else { + $children_reaped++; + } if($Global::sshmaster{$stiff}) { # This is one of the ssh -M: ignore next; @@ -3565,7 +3570,7 @@ sub reaper { } $opt::sqlmaster and $Global::sql->run("COMMIT;"); debug("run", "done "); - return @pids_reaped; + return wantarray ? @pids_reaped : $children_reaped; } sub __USAGE__ {} @@ -5313,18 +5318,20 @@ sub compute_number_of_processes { if($system_limit < $wanted_processes) { # The system_limit is less than the wanted_processes if($system_limit < 1 and not $Global::JobQueue->empty()) { - ::warning("Cannot spawn any jobs. Raising ulimit -u or /etc/security/limits.conf", + ::warning("Cannot spawn any jobs. ". + "Raising ulimit -u or /etc/security/limits.conf", "or /proc/sys/kernel/pid_max may help."); ::wait_and_exit(255); } if(not $more_filehandles) { - ::warning("Only enough file handles to run ". $system_limit. " jobs in parallel.", - "Running 'parallel -j0 -N $system_limit --pipe parallel -j0' or ", + ::warning("Only enough file handles to run ". + $system_limit. " jobs in parallel.", + "Running 'parallel -j0 -N $system_limit --pipe parallel -j0' or", "raising ulimit -n or /etc/security/limits.conf may help."); } if($max_system_proc_reached) { - ::warning("Only enough available processes to run ". $system_limit. - " jobs in parallel.", + ::warning("Only enough available processes to run ". + $system_limit. " jobs in parallel.", "Raising ulimit -u or /etc/security/limits.conf ", "or /proc/sys/kernel/pid_max may help."); } @@ -8484,25 +8491,21 @@ sub set_seq { $self->{'seq'} = shift; } -{ - my $max_slot_number; - - sub slot { - # Find the number of a free job slot and return it - # Uses: - # @Global::slots - list with free jobslots - # Returns: - # $jobslot = number of jobslot - my $self = shift; - if(not $self->{'slot'}) { - if(not @Global::slots) { - # $max_slot_number will typically be $Global::max_jobs_running - push @Global::slots, ++$Global::max_slot_number; - } - $self->{'slot'} = shift @Global::slots; +sub slot { + # Find the number of a free job slot and return it + # Uses: + # @Global::slots - list with free jobslots + # Returns: + # $jobslot = number of jobslot + my $self = shift; + if(not $self->{'slot'}) { + if(not @Global::slots) { + # $max_slot_number will typically be $Global::max_jobs_running + push @Global::slots, ++$Global::max_slot_number; } - return $self->{'slot'}; + $self->{'slot'} = shift @Global::slots; } + return $self->{'slot'}; } sub populate { @@ -10622,4 +10625,5 @@ sub unlock { # Keep perl -w happy $opt::x = $Semaphore::timeout = $Semaphore::wait = -$Job::file_descriptor_warning_printed = $Global::envdef = @Arg::arg; +$Job::file_descriptor_warning_printed = $Global::envdef = @Arg::arg = +$Global::max_slot_number; diff --git a/testsuite/tests-to-run/parallel-local-100s.sh b/testsuite/tests-to-run/parallel-local-100s.sh index ee330d8f..e2eb6f0a 100644 --- a/testsuite/tests-to-run/parallel-local-100s.sh +++ b/testsuite/tests-to-run/parallel-local-100s.sh @@ -28,5 +28,51 @@ par_over_4GB() { nice md5sum } + +measure() { + # Input: + # $1 = iterations + # $2 = sleep 1 sec for every $2 + seq $1 | + stdout time -v parallel -u sleep '{= $_=$_%'$2'?0:1 =}' | + grep Maximum | field 6; +} +export -f measure + +no_mem_leak() { + # Return false if leaking + max1000=$(parallel measure {} 100000 ::: 1000 1000 1000 1000 | + sort -n | tail -n 1) + min30000=$(parallel measure {} 100000 ::: 30000 30000 30000 30000 | + sort -n | head -n 1) + if [ $max1000 -gt $min30000 ] ; then + max1000=$(parallel measure {} 100 ::: 1000 1000 1000 1000 | + sort -n | tail -n 1) + min30000=$(parallel measure {} 100 ::: 30000 30000 30000 30000 | + sort -n | head -n 1) + if [ $max1000 -gt $min30000 ] ; then + echo $max1000 -gt $min30000 = no leak + return 0 + else + echo not $max1000 -gt $min30000 = possible leak + return 1 + fi + else + echo not $max1000 -gt $min30000 = possible leak + return 1 + fi +} +export -f no_mem_leak + +par_mem_leak() { + echo "### test for mem leak" + if no_mem_leak >/dev/null ; then + echo no mem leak detected + else + echo possible mem leak; + fi +} + + export -f $(compgen -A function | grep par_) compgen -A function | grep par_ | sort | parallel -vj0 -k --tag --joblog /tmp/jl-`basename $0` '{} 2>&1' diff --git a/testsuite/wanted-results/parallel-local-100s b/testsuite/wanted-results/parallel-local-100s index eca8b35d..f400de76 100644 --- a/testsuite/wanted-results/parallel-local-100s +++ b/testsuite/wanted-results/parallel-local-100s @@ -1,7 +1,10 @@ +par_mem_leak 2>&1 +par_mem_leak ### test for mem leak +par_mem_leak no mem leak detected par_outside_file_handle_limit 2>&1 par_outside_file_handle_limit ### Test Force outside the file handle limit, 2009-02-17 Gave fork error -par_outside_file_handle_limit parallel: Warning: Only enough file handles to run 252 jobs in parallel. -par_outside_file_handle_limit parallel: Warning: Running 'parallel -j0 -N 252 --pipe parallel -j0' or +par_outside_file_handle_limit parallel: Warning: Only enough file handles to run 251 jobs in parallel. +par_outside_file_handle_limit parallel: Warning: Running 'parallel -j0 -N 251 --pipe parallel -j0' or par_outside_file_handle_limit parallel: Warning: raising ulimit -n or /etc/security/limits.conf may help. par_outside_file_handle_limit Start par_outside_file_handle_limit end diff --git a/testsuite/wanted-results/parallel-local-10s b/testsuite/wanted-results/parallel-local-10s index 8e20e388..bb7c4cf9 100644 --- a/testsuite/wanted-results/parallel-local-10s +++ b/testsuite/wanted-results/parallel-local-10s @@ -1,7 +1,7 @@ echo '### bug #46214: Using --pipepart doesnt spawn multiple jobs in version 20150922' ### bug #46214: Using --pipepart doesnt spawn multiple jobs in version 20150922 seq 1000000 > /tmp/num1000000; stdout parallel --pipepart --progress -a /tmp/num1000000 --block 10k -j0 true |grep 1:local -1:local / 8 / 252 +1:local / 8 / 251 echo '**' ** testhalt() { echo '### testhalt --halt '$1; (yes 0 | head -n 10; seq 10) | stdout parallel -kj4 --halt $1 'sleep {= $_=$_*0.3+1 =}; exit {}'; echo $?; (seq 10; yes 0 | head -n 10) | stdout parallel -kj4 --halt $1 'sleep {= $_=$_*0.3+1 =}; exit {}'; echo $?; }; export -f testhalt; parallel -kj0 testhalt ::: now,fail=0 now,fail=1 now,fail=2 now,fail=30% now,fail=70% soon,fail=0 soon,fail=1 soon,fail=2 soon,fail=30% soon,fail=70% now,success=0 now,success=1 now,success=2 now,success=30% now,success=70% soon,success=0 soon,success=1 soon,success=2 soon,success=30% now,success=70% diff --git a/testsuite/wanted-results/parallel-local164 b/testsuite/wanted-results/parallel-local164 index dccbd0a5..f272a4d8 100644 --- a/testsuite/wanted-results/parallel-local164 +++ b/testsuite/wanted-results/parallel-local164 @@ -338,7 +338,7 @@ bug #38439: "open files" with --files --pipe blocks after a while 20 of 21 21 of 21 parallel: Warning: Only enough file handles to run 1 jobs in parallel. -parallel: Warning: Running 'parallel -j0 -N 1 --pipe parallel -j0' or +parallel: Warning: Running 'parallel -j0 -N 1 --pipe parallel -j0' or parallel: Warning: raising ulimit -n or /etc/security/limits.conf may help. echo 'bug #34241: --pipe should not spawn unneeded processes - part 2' bug #34241: --pipe should not spawn unneeded processes - part 2