tange/parallel
1
0
Fork 0
mirror of https://git.savannah.gnu.org/git/parallel.git synced 2024-11-29 09:27:56 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

parallel: limit memory consumption if only 1 input source.

Browse source
This commit is contained in:
Ole Tange 2012-01-10 21:58:39 +01:00
parent 3009913a66
commit 08e1366c5d
3 changed files with 26 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

21
src/parallel
View file

@ -4645,13 +4645,13 @@ sub nest_get {
my @record = (); my @record = ();
my $prepend = undef; my $prepend = undef;
my $empty = 1; my $empty = 1;
my $no_of_inputs = $#{$self->{'fhs'}} + 1; my $no_of_inputsources = $#{$self->{'fhs'}} + 1;
if(not $self->{'arg_matrix'}) { if(not $self->{'arg_matrix'}) {
# Initialize @arg_matrix with one arg from each file # Initialize @arg_matrix with one arg from each file
# read one line from each file # read one line from each file
my @first_arg_set; my @first_arg_set;
my $all_empty = 1; my $all_empty = 1;
for (my $fhno = 0; $fhno < $no_of_inputs ; $fhno++) { for (my $fhno = 0; $fhno < $no_of_inputsources ; $fhno++) {
my $arg = read_arg_from_fh($self->{'fhs'}[$fhno]); my $arg = read_arg_from_fh($self->{'fhs'}[$fhno]);
if(defined $arg) { if(defined $arg) {
$all_empty = 0; $all_empty = 0;
@ -4665,7 +4665,16 @@ sub nest_get {
} }
return [@first_arg_set]; return [@first_arg_set];
} }
for (my $fhno = $no_of_inputs - 1; $fhno >= 0; $fhno--) {
# Treat the case with one input source special. For multiple
# input sources we need to remember all previously read values to
# generate all combinations. But for one input source we can
# forget the value after first use.
if($no_of_inputsources == 1) {
my $arg = read_arg_from_fh($self->{'fhs'}[0]);
return [$arg];
}
for (my $fhno = $no_of_inputsources - 1; $fhno >= 0; $fhno--) {
if(eof($self->{'fhs'}[$fhno])) { if(eof($self->{'fhs'}[$fhno])) {
next; next;
} else { } else {
@ -4676,7 +4685,7 @@ sub nest_get {
$self->{'arg_matrix'}[$fhno][$len] = $arg; $self->{'arg_matrix'}[$fhno][$len] = $arg;
# make all new combinations # make all new combinations
my @combarg = (); my @combarg = ();
for (my $fhn = 0; $fhn < $no_of_inputs; $fhn++) { for (my $fhn = 0; $fhn < $no_of_inputsources; $fhn++) {
push @combarg, [0, $#{$self->{'arg_matrix'}[$fhn]}]; push @combarg, [0, $#{$self->{'arg_matrix'}[$fhn]}];
} }
$combarg[$fhno] = [$len,$len]; # Find only combinations with this new entry $combarg[$fhno] = [$len,$len]; # Find only combinations with this new entry
@ -4687,10 +4696,10 @@ sub nest_get {
my @mapped; my @mapped;
for my $c (expand_combinations(@combarg)) { for my $c (expand_combinations(@combarg)) {
my @a; my @a;
for my $n (0 .. $no_of_inputs - 1 ) { for my $n (0 .. $no_of_inputsources - 1 ) {
push @a, $self->{'arg_matrix'}[$n][$$c[$n]]; push @a, $self->{'arg_matrix'}[$n][$$c[$n]];
} }
push @mapped, [@a]; push @mapped, \@a;
} }
# append the mapped to the ungotten arguments # append the mapped to the ungotten arguments
push @{$self->{'unget'}}, @mapped; push @{$self->{'unget'}}, @mapped;

8
testsuite/tests-to-run/test65.sh
View file

@ -4,6 +4,14 @@
cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | parallel -j10 -k -L1 cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | parallel -j10 -k -L1
EOF EOF
echo "### Test memory consumption stays (almost) the same for 30 and 300 jobs";
out30=$( stdout memusg parallel -j2 true :::: <(perl -e '$a="x"x100000;for(1..30){print $a,"\n"}') );
out300=$( stdout memusg parallel -j2 true :::: <(perl -e '$a="x"x100000;for(1..300){print $a,"\n"}') );
mem30=$(echo $out30 | tr -cd 0-9);
mem300=$(echo $out300 | tr -cd 0-9);
echo "Test if memory consumption(300 jobs) < memory consumption(30 jobs) * 150% ";
echo $(($mem300*100 < $mem30 * 150))
echo '### Test --shellquote' echo '### Test --shellquote'
cat <<'_EOF' | parallel --shellquote cat <<'_EOF' | parallel --shellquote
awk -v FS="\",\"" '{print $1, $3, $4, $5, $9, $14}' | grep -v "#" | sed -e '1d' -e 's/\"//g' -e 's/\/\/\//\t/g' | cut -f1-6,11 | sed -e 's/\/\//\t/g' -e 's/ /\t/g awk -v FS="\",\"" '{print $1, $3, $4, $5, $9, $14}' | grep -v "#" | sed -e '1d' -e 's/\"//g' -e 's/\/\/\//\t/g' | cut -f1-6,11 | sed -e 's/\/\//\t/g' -e 's/ /\t/g

3
testsuite/wanted-results/test65
View file

@ -1,2 +1,5 @@
### Test memory consumption stays (almost) the same for 30 and 300 jobs
Test if memory consumption(300 jobs) < memory consumption(30 jobs) * 150%
1
### Test --shellquote ### Test --shellquote
awk\ -v\ FS=\"\\\",\\\"\"\ \'\{print\ \$1,\ \$3,\ \$4,\ \$5,\ \$9,\ \$14\}\'\ \|\ grep\ -v\ \"\#\"\ \|\ sed\ -e\ \'1d\'\ -e\ \'s/\\\"//g\'\ -e\ \'s/\\/\\/\\//\\t/g\'\ \|\ cut\ -f1-6,11\ \|\ sed\ -e\ \'s/\\/\\//\\t/g\'\ -e\ \'s/\ /\\t/g awk\ -v\ FS=\"\\\",\\\"\"\ \'\{print\ \$1,\ \$3,\ \$4,\ \$5,\ \$9,\ \$14\}\'\ \|\ grep\ -v\ \"\#\"\ \|\ sed\ -e\ \'1d\'\ -e\ \'s/\\\"//g\'\ -e\ \'s/\\/\\/\\//\\t/g\'\ \|\ cut\ -f1-6,11\ \|\ sed\ -e\ \'s/\\/\\//\\t/g\'\ -e\ \'s/\ /\\t/g