parallel: Removed memleak.

src/parallel

@@ -3502,12 +3502,17 @@ sub reaper {
     #   @pids_reaped = PIDs of children finished
     my $stiff;
     my @pids_reaped;
+    my $children_reaped = 0;
     debug("run", "Reaper ");
     # For efficiency surround with BEGIN/COMMIT when using $opt::sqlmaster
     $opt::sqlmaster and $Global::sql->run("BEGIN;");
     while (($stiff = waitpid(-1, &WNOHANG)) > 0) {
 	# $stiff = pid of dead process
-	push(@pids_reaped,$stiff);
+	if(wantarray) {
+	    push(@pids_reaped,$stiff);
+	} else {
+	    $children_reaped++;
+	}
         if($Global::sshmaster{$stiff}) {
             # This is one of the ssh -M: ignore
             next;
@@ -3565,7 +3570,7 @@ sub reaper {
     }
     $opt::sqlmaster and $Global::sql->run("COMMIT;");
     debug("run", "done ");
-    return @pids_reaped;
+    return wantarray ? @pids_reaped : $children_reaped;
 }
 
 sub __USAGE__ {}
@@ -5313,18 +5318,20 @@ sub compute_number_of_processes {
 	if($system_limit < $wanted_processes) {
 	    # The system_limit is less than the wanted_processes
 	    if($system_limit < 1 and not $Global::JobQueue->empty()) {
-		::warning("Cannot spawn any jobs. Raising ulimit -u or /etc/security/limits.conf",
+		::warning("Cannot spawn any jobs. ".
+			  "Raising ulimit -u or /etc/security/limits.conf",
 			  "or /proc/sys/kernel/pid_max may help.");
 		::wait_and_exit(255);
 	    }
 	    if(not $more_filehandles) {
-		::warning("Only enough file handles to run ". $system_limit. " jobs in parallel.",
+		::warning("Only enough file handles to run ".
-			  "Running 'parallel -j0 -N $system_limit --pipe parallel -j0' or ",
+			  $system_limit. " jobs in parallel.",
+			  "Running 'parallel -j0 -N $system_limit --pipe parallel -j0' or",
 			  "raising ulimit -n or /etc/security/limits.conf may help.");
 	    }
 	    if($max_system_proc_reached) {
-		::warning("Only enough available processes to run ". $system_limit.
+		::warning("Only enough available processes to run ".
-			  " jobs in parallel.",
+			  $system_limit. " jobs in parallel.",
 			  "Raising ulimit -u or /etc/security/limits.conf ",
 			  "or /proc/sys/kernel/pid_max may help.");
 	    }
@@ -8484,25 +8491,21 @@ sub set_seq {
     $self->{'seq'} = shift;
 }
 
-{
+sub slot {
-    my $max_slot_number;
+    # Find the number of a free job slot and return it
-
+    # Uses:
-    sub slot {
+    #   @Global::slots - list with free jobslots
-	# Find the number of a free job slot and return it
+    # Returns:
-        # Uses:
+    #   $jobslot = number of jobslot
-        #   @Global::slots - list with free jobslots
+    my $self = shift;
-	# Returns:
+    if(not $self->{'slot'}) {
-	#   $jobslot = number of jobslot
+	if(not @Global::slots) {
-	my $self = shift;
+	    # $max_slot_number will typically be $Global::max_jobs_running
-	if(not $self->{'slot'}) {
+	    push @Global::slots, ++$Global::max_slot_number;
-	    if(not @Global::slots) {
-		# $max_slot_number will typically be $Global::max_jobs_running
-		push @Global::slots, ++$Global::max_slot_number;
-	    }
-	    $self->{'slot'} = shift @Global::slots;
 	}
-	return $self->{'slot'};
+	$self->{'slot'} = shift @Global::slots;
     }
+    return $self->{'slot'};
 }
 
 sub populate {
@@ -10622,4 +10625,5 @@ sub unlock {
 # Keep perl -w happy
 
 $opt::x = $Semaphore::timeout = $Semaphore::wait =
-$Job::file_descriptor_warning_printed = $Global::envdef = @Arg::arg;
+$Job::file_descriptor_warning_printed = $Global::envdef = @Arg::arg =
+$Global::max_slot_number;

testsuite/tests-to-run/parallel-local-100s.sh

@@ -28,5 +28,51 @@ par_over_4GB() {
     nice md5sum
 }
 
+
+measure() {
+    # Input:
+    #   $1 = iterations
+    #   $2 = sleep 1 sec for every $2
+    seq $1 |
+	stdout time -v parallel -u sleep '{= $_=$_%'$2'?0:1 =}' |
+	grep Maximum | field 6;
+}
+export -f measure
+
+no_mem_leak() {
+    # Return false if leaking
+    max1000=$(parallel measure {} 100000 ::: 1000 1000 1000 1000 |
+    		       sort -n | tail -n 1)
+    min30000=$(parallel measure {} 100000 ::: 30000 30000 30000 30000 |
+    			sort -n | head -n 1)
+    if [ $max1000 -gt $min30000 ] ; then
+	max1000=$(parallel measure {} 100 ::: 1000 1000 1000 1000 |
+			   sort -n | tail -n 1)
+	min30000=$(parallel measure {} 100 ::: 30000 30000 30000 30000 |
+			    sort -n | head -n 1)
+	if [ $max1000 -gt $min30000 ] ; then
+	    echo $max1000 -gt $min30000 = no leak
+	    return 0
+	else
+	    echo not $max1000 -gt $min30000 = possible leak
+	    return 1
+	fi
+    else
+	echo not $max1000 -gt $min30000 = possible leak
+	return 1
+    fi
+}
+export -f no_mem_leak
+
+par_mem_leak() {
+    echo "### test for mem leak"
+    if no_mem_leak >/dev/null ; then
+	echo no mem leak detected
+    else
+	echo possible mem leak;
+    fi
+}
+
+
 export -f $(compgen -A function | grep par_)
 compgen -A function | grep par_ | sort | parallel -vj0 -k --tag --joblog /tmp/jl-`basename $0` '{} 2>&1'

testsuite/wanted-results/parallel-local-100s

@@ -1,7 +1,10 @@
+par_mem_leak 2>&1
+par_mem_leak	### test for mem leak
+par_mem_leak	no mem leak detected
 par_outside_file_handle_limit 2>&1
 par_outside_file_handle_limit	### Test Force outside the file handle limit, 2009-02-17 Gave fork error
-par_outside_file_handle_limit	parallel: Warning: Only enough file handles to run 252 jobs in parallel.
+par_outside_file_handle_limit	parallel: Warning: Only enough file handles to run 251 jobs in parallel.
-par_outside_file_handle_limit	parallel: Warning: Running 'parallel -j0 -N 252 --pipe parallel -j0' or 
+par_outside_file_handle_limit	parallel: Warning: Running 'parallel -j0 -N 251 --pipe parallel -j0' or
 par_outside_file_handle_limit	parallel: Warning: raising ulimit -n or /etc/security/limits.conf may help.
 par_outside_file_handle_limit	Start
 par_outside_file_handle_limit	end

testsuite/wanted-results/parallel-local-10s

@@ -1,7 +1,7 @@
 echo '### bug #46214: Using --pipepart doesnt spawn multiple jobs in version 20150922'
 ### bug #46214: Using --pipepart doesnt spawn multiple jobs in version 20150922
   seq 1000000 > /tmp/num1000000;   stdout parallel --pipepart --progress -a /tmp/num1000000 --block 10k -j0 true |grep 1:local
-1:local / 8 / 252
+1:local / 8 / 251
 echo '**'
 **
 testhalt() {   echo '### testhalt --halt '$1;   (yes 0 | head -n 10; seq 10) | stdout parallel -kj4 --halt $1 'sleep {= $_=$_*0.3+1 =}; exit {}'; echo $?;   (seq 10; yes 0 | head -n 10) | stdout parallel -kj4 --halt $1 'sleep {= $_=$_*0.3+1 =}; exit {}'; echo $?; }; export -f testhalt;   parallel -kj0 testhalt ::: now,fail=0 now,fail=1 now,fail=2 now,fail=30%  now,fail=70%     soon,fail=0 soon,fail=1 soon,fail=2 soon,fail=30% soon,fail=70%     now,success=0 now,success=1 now,success=2 now,success=30% now,success=70%     soon,success=0 soon,success=1 soon,success=2 soon,success=30% now,success=70%