mirror of
https://git.savannah.gnu.org/git/parallel.git
synced 2024-11-22 05:57:54 +00:00
parallel: Removed memleak.
This commit is contained in:
Ole Tange
parent
e94774df14
commit
fd6abf1f9a
54
src/parallel
54
src/parallel
|
|
@ -3502,12 +3502,17 @@ sub reaper {
|
|||
# @pids_reaped = PIDs of children finished
|
||||
my $stiff;
|
||||
my @pids_reaped;
|
||||
my $children_reaped = 0;
|
||||
debug("run", "Reaper ");
|
||||
# For efficiency surround with BEGIN/COMMIT when using $opt::sqlmaster
|
||||
$opt::sqlmaster and $Global::sql->run("BEGIN;");
|
||||
while (($stiff = waitpid(-1, &WNOHANG)) > 0) {
|
||||
# $stiff = pid of dead process
|
||||
push(@pids_reaped,$stiff);
|
||||
if(wantarray) {
|
||||
push(@pids_reaped,$stiff);
|
||||
} else {
|
||||
$children_reaped++;
|
||||
}
|
||||
if($Global::sshmaster{$stiff}) {
|
||||
# This is one of the ssh -M: ignore
|
||||
next;
|
||||
|
|
@ -3565,7 +3570,7 @@ sub reaper {
|
|||
}
|
||||
$opt::sqlmaster and $Global::sql->run("COMMIT;");
|
||||
debug("run", "done ");
|
||||
return @pids_reaped;
|
||||
return wantarray ? @pids_reaped : $children_reaped;
|
||||
}
|
||||
|
||||
sub __USAGE__ {}
|
||||
|
|
@ -5313,18 +5318,20 @@ sub compute_number_of_processes {
|
|||
if($system_limit < $wanted_processes) {
|
||||
# The system_limit is less than the wanted_processes
|
||||
if($system_limit < 1 and not $Global::JobQueue->empty()) {
|
||||
::warning("Cannot spawn any jobs. Raising ulimit -u or /etc/security/limits.conf",
|
||||
::warning("Cannot spawn any jobs. ".
|
||||
"Raising ulimit -u or /etc/security/limits.conf",
|
||||
"or /proc/sys/kernel/pid_max may help.");
|
||||
::wait_and_exit(255);
|
||||
}
|
||||
if(not $more_filehandles) {
|
||||
::warning("Only enough file handles to run ". $system_limit. " jobs in parallel.",
|
||||
"Running 'parallel -j0 -N $system_limit --pipe parallel -j0' or ",
|
||||
::warning("Only enough file handles to run ".
|
||||
$system_limit. " jobs in parallel.",
|
||||
"Running 'parallel -j0 -N $system_limit --pipe parallel -j0' or",
|
||||
"raising ulimit -n or /etc/security/limits.conf may help.");
|
||||
}
|
||||
if($max_system_proc_reached) {
|
||||
::warning("Only enough available processes to run ". $system_limit.
|
||||
" jobs in parallel.",
|
||||
::warning("Only enough available processes to run ".
|
||||
$system_limit. " jobs in parallel.",
|
||||
"Raising ulimit -u or /etc/security/limits.conf ",
|
||||
"or /proc/sys/kernel/pid_max may help.");
|
||||
}
|
||||
|
|
@ -8484,25 +8491,21 @@ sub set_seq {
|
|||
$self->{'seq'} = shift;
|
||||
}
|
||||
|
||||
{
|
||||
my $max_slot_number;
|
||||
|
||||
sub slot {
|
||||
# Find the number of a free job slot and return it
|
||||
# Uses:
|
||||
# @Global::slots - list with free jobslots
|
||||
# Returns:
|
||||
# $jobslot = number of jobslot
|
||||
my $self = shift;
|
||||
if(not $self->{'slot'}) {
|
||||
if(not @Global::slots) {
|
||||
# $max_slot_number will typically be $Global::max_jobs_running
|
||||
push @Global::slots, ++$Global::max_slot_number;
|
||||
}
|
||||
$self->{'slot'} = shift @Global::slots;
|
||||
sub slot {
|
||||
# Find the number of a free job slot and return it
|
||||
# Uses:
|
||||
# @Global::slots - list with free jobslots
|
||||
# Returns:
|
||||
# $jobslot = number of jobslot
|
||||
my $self = shift;
|
||||
if(not $self->{'slot'}) {
|
||||
if(not @Global::slots) {
|
||||
# $max_slot_number will typically be $Global::max_jobs_running
|
||||
push @Global::slots, ++$Global::max_slot_number;
|
||||
}
|
||||
return $self->{'slot'};
|
||||
$self->{'slot'} = shift @Global::slots;
|
||||
}
|
||||
return $self->{'slot'};
|
||||
}
|
||||
|
||||
sub populate {
|
||||
|
|
@ -10622,4 +10625,5 @@ sub unlock {
|
|||
# Keep perl -w happy
|
||||
|
||||
$opt::x = $Semaphore::timeout = $Semaphore::wait =
|
||||
$Job::file_descriptor_warning_printed = $Global::envdef = @Arg::arg;
|
||||
$Job::file_descriptor_warning_printed = $Global::envdef = @Arg::arg =
|
||||
$Global::max_slot_number;
|
||||
|
|
|
|||
|
|
@ -28,5 +28,51 @@ par_over_4GB() {
|
|||
nice md5sum
|
||||
}
|
||||
|
||||
|
||||
measure() {
|
||||
# Input:
|
||||
# $1 = iterations
|
||||
# $2 = sleep 1 sec for every $2
|
||||
seq $1 |
|
||||
stdout time -v parallel -u sleep '{= $_=$_%'$2'?0:1 =}' |
|
||||
grep Maximum | field 6;
|
||||
}
|
||||
export -f measure
|
||||
|
||||
no_mem_leak() {
|
||||
# Return false if leaking
|
||||
max1000=$(parallel measure {} 100000 ::: 1000 1000 1000 1000 |
|
||||
sort -n | tail -n 1)
|
||||
min30000=$(parallel measure {} 100000 ::: 30000 30000 30000 30000 |
|
||||
sort -n | head -n 1)
|
||||
if [ $max1000 -gt $min30000 ] ; then
|
||||
max1000=$(parallel measure {} 100 ::: 1000 1000 1000 1000 |
|
||||
sort -n | tail -n 1)
|
||||
min30000=$(parallel measure {} 100 ::: 30000 30000 30000 30000 |
|
||||
sort -n | head -n 1)
|
||||
if [ $max1000 -gt $min30000 ] ; then
|
||||
echo $max1000 -gt $min30000 = no leak
|
||||
return 0
|
||||
else
|
||||
echo not $max1000 -gt $min30000 = possible leak
|
||||
return 1
|
||||
fi
|
||||
else
|
||||
echo not $max1000 -gt $min30000 = possible leak
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
export -f no_mem_leak
|
||||
|
||||
par_mem_leak() {
|
||||
echo "### test for mem leak"
|
||||
if no_mem_leak >/dev/null ; then
|
||||
echo no mem leak detected
|
||||
else
|
||||
echo possible mem leak;
|
||||
fi
|
||||
}
|
||||
|
||||
|
||||
export -f $(compgen -A function | grep par_)
|
||||
compgen -A function | grep par_ | sort | parallel -vj0 -k --tag --joblog /tmp/jl-`basename $0` '{} 2>&1'
|
||||
|
|
|
|||
|
|
@ -1,7 +1,10 @@
|
|||
par_mem_leak 2>&1
|
||||
par_mem_leak ### test for mem leak
|
||||
par_mem_leak no mem leak detected
|
||||
par_outside_file_handle_limit 2>&1
|
||||
par_outside_file_handle_limit ### Test Force outside the file handle limit, 2009-02-17 Gave fork error
|
||||
par_outside_file_handle_limit parallel: Warning: Only enough file handles to run 252 jobs in parallel.
|
||||
par_outside_file_handle_limit parallel: Warning: Running 'parallel -j0 -N 252 --pipe parallel -j0' or
|
||||
par_outside_file_handle_limit parallel: Warning: Only enough file handles to run 251 jobs in parallel.
|
||||
par_outside_file_handle_limit parallel: Warning: Running 'parallel -j0 -N 251 --pipe parallel -j0' or
|
||||
par_outside_file_handle_limit parallel: Warning: raising ulimit -n or /etc/security/limits.conf may help.
|
||||
par_outside_file_handle_limit Start
|
||||
par_outside_file_handle_limit end
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
echo '### bug #46214: Using --pipepart doesnt spawn multiple jobs in version 20150922'
|
||||
### bug #46214: Using --pipepart doesnt spawn multiple jobs in version 20150922
|
||||
seq 1000000 > /tmp/num1000000; stdout parallel --pipepart --progress -a /tmp/num1000000 --block 10k -j0 true |grep 1:local
|
||||
1:local / 8 / 252
|
||||
1:local / 8 / 251
|
||||
echo '**'
|
||||
**
|
||||
testhalt() { echo '### testhalt --halt '$1; (yes 0 | head -n 10; seq 10) | stdout parallel -kj4 --halt $1 'sleep {= $_=$_*0.3+1 =}; exit {}'; echo $?; (seq 10; yes 0 | head -n 10) | stdout parallel -kj4 --halt $1 'sleep {= $_=$_*0.3+1 =}; exit {}'; echo $?; }; export -f testhalt; parallel -kj0 testhalt ::: now,fail=0 now,fail=1 now,fail=2 now,fail=30% now,fail=70% soon,fail=0 soon,fail=1 soon,fail=2 soon,fail=30% soon,fail=70% now,success=0 now,success=1 now,success=2 now,success=30% now,success=70% soon,success=0 soon,success=1 soon,success=2 soon,success=30% now,success=70%
|
||||
|
|
|
|||
|
|
@ -338,7 +338,7 @@ bug #38439: "open files" with --files --pipe blocks after a while
|
|||
20 of 21
|
||||
21 of 21
|
||||
parallel: Warning: Only enough file handles to run 1 jobs in parallel.
|
||||
parallel: Warning: Running 'parallel -j0 -N 1 --pipe parallel -j0' or
|
||||
parallel: Warning: Running 'parallel -j0 -N 1 --pipe parallel -j0' or
|
||||
parallel: Warning: raising ulimit -n or /etc/security/limits.conf may help.
|
||||
echo 'bug #34241: --pipe should not spawn unneeded processes - part 2'
|
||||
bug #34241: --pipe should not spawn unneeded processes - part 2
|
||||
|
|
|
|||
Loading…
Reference in a new issue