tange/parallel
1
0
Fork 0
mirror of https://git.savannah.gnu.org/git/parallel.git synced 2024-12-22 20:57:53 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fixed too many spawned jobs in --pipe --files.

Browse source
This commit is contained in:
Ole Tange 2014-02-17 22:25:42 +01:00
parent 816f851488
commit 18ce76f598
4 changed files with 86 additions and 13 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

33
src/parallel
View file

@ -4907,19 +4907,27 @@ sub print {
my $self = shift;
::debug(">>joboutput ".$self->replaced()."\n");
if($opt::pipe and $self->virgin()) {
# Nothing was printed to this job:
# cleanup tmp files if --files was set
unlink $self->fh(1,"name");
return;
}
# if($opt::pipe and $self->virgin()) {
# # Nothing was printed to this job:
# # cleanup tmp files if --files was set
# close $self->fh(0,"w");
# for my $fdno (1,2) {
# close $self->fh($fdno,"w");
# close $self->fh($fdno,"r");
# # Unlink files that are empty
# unlink $self->fh($fdno,"name");
# }
# return;
# }
if($opt::dryrun) {
# Nothing was printed to this job:
# cleanup tmp files if --files was set
unlink $self->fh(1,"name");
}
if($Global::joblog) { $self->print_joblog() }
if($opt::pipe and $self->virgin()) {
} else {
if($Global::joblog) { $self->print_joblog() }
}
# Printing is only relevant for grouped output.
$Global::grouped or return;
@ -4957,7 +4965,14 @@ sub print {
if($opt::files) {
# If --compress: $in_fh must be closed first.
close $self->fh($fdno,"w");
if($fdno == 1 and $self->fh($fdno,"name")) {
close $in_fh;
if($opt::pipe and $self->virgin()) {
# Nothing was printed to this job: # cleanup tmp files if --files was set
for my $fdno (1,2) {
unlink $self->fh($fdno,"name");
unlink $self->fh($fdno,"unlink");
}
} elsif($fdno == 1 and $self->fh($fdno,"name")) {
print $out_fd $self->fh($fdno,"name"),"\n";
}
} elsif($opt::linebuffer) {

2
testsuite/tests-to-run/parallel-local-100s.sh
View file

@ -6,7 +6,7 @@
cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -k -j4 -L1
echo '### Test if we can deal with output > 4 GB'
## echo | niceload --io 10 parallel -q perl -e '"\$a=\"x\"x1000000;for(0..4300){print \$a}"' | md5sum
echo | parallel -q perl -e '$a="x"x1000000;for(0..4300){print $a}' | md5sum
echo | parallel --tmpdir /dev/shm -q perl -e '$a="x"x1000000;for(0..4300){print $a}' | md5sum
echo '**'

28
testsuite/tests-to-run/parallel-local114.sh
View file

@ -1,14 +1,36 @@
#!/bin/bash
cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -j0 -k -L1
echo "### --line-buffer"
seq 10 | parallel -j20 --line-buffer 'seq {} 10 | pv -qL 10' > /tmp/parallel_l$$;
seq 10 | parallel -j20 'seq {} 10 | pv -qL 10' > /tmp/parallel_$$;
cat /tmp/parallel_l$$ | wc;
diff /tmp/parallel_$$ /tmp/parallel_l$$ >/dev/null ; echo These must diff: $?
echo "### --pipe --line-buffer"
seq 200| parallel -N10 -L1 --pipe -j20 --line-buffer --tagstring {#} pv -qL 10 > /tmp/parallel_$$;
cat /tmp/parallel_$$ | wc;
diff <(sort /tmp/parallel_$$) /tmp/parallel_$$ >/dev/null ; echo These must diff: $?
seq 200| parallel -N10 -L1 --pipe -j20 --line-buffer --tagstring {#} pv -qL 10 > /tmp/parallel_pl$$;
seq 200| parallel -N10 -L1 --pipe -j20 --tagstring {#} pv -qL 10 > /tmp/parallel_p$$;
cat /tmp/parallel_pl$$ | wc;
diff /tmp/parallel_p$$ /tmp/parallel_pl$$ >/dev/null ; echo These must diff: $?
echo "### --pipe --line-buffer --compress (fails)"
# seq 200| parallel -N10 -L1 --pipe -j20 --line-buffer --compress --tagstring {#} pv -qL 10 | wc
echo "### bug #41482: --pipe --compress blocks at different -j/seq combinations"
seq 1 | parallel -k -j2 --compress -N1 -L1 --pipe cat;
echo echo 1-4 + 1-4
seq 4 | parallel -k -j3 --compress -N1 -L1 -vv echo;
echo 4 times wc to stderr to stdout
(seq 4 | parallel -k -j3 --compress -N1 -L1 --pipe wc '>&2') 2>&1 >/dev/null
echo 1 2 3 4
seq 4 | parallel -k -j3 --compress echo;
echo 1 2 3 4
seq 4 | parallel -k -j1 --compress echo;
echo 1 2
seq 2 | parallel -k -j1 --compress echo;
echo 1 2 3
seq 3 | parallel -k -j2 --compress -N1 -L1 --pipe cat;
echo "### bug #41609: --compress fails"
seq 12 | parallel --compress --compress-program bzip2 -k seq {} 1000000 | md5sum
seq 12 | parallel --compress -k seq {} 1000000 | md5sum

36
testsuite/wanted-results/parallel-local114
View file

@ -1,7 +1,43 @@
### --line-buffer
55 55 120
These must diff: 1
### --pipe --line-buffer
200 400 1202
These must diff: 1
### --pipe --line-buffer --compress (fails)
### bug #41482: --pipe --compress blocks at different -j/seq combinations
1
echo 1-4 + 1-4
echo 1
1
echo 2
2
echo 3
3
echo 4
4
4 times wc to stderr to stdout
1 1 2
1 1 2
1 1 2
1 1 2
1 2 3 4
1
2
3
4
1 2 3 4
1
2
3
4
1 2
1
2
1 2 3
1
2
3
### bug #41609: --compress fails
24812dd0f24a26d08a780f988b9d5ad2 -
24812dd0f24a26d08a780f988b9d5ad2 -