mirror of
https://git.savannah.gnu.org/git/parallel.git
synced 2024-11-22 05:57:54 +00:00
Fixed too many spawned jobs in --pipe --files.
This commit is contained in:
Ole Tange
parent
816f851488
commit
18ce76f598
33
src/parallel
33
src/parallel
|
|
@ -4907,19 +4907,27 @@ sub print {
|
|||
|
||||
my $self = shift;
|
||||
::debug(">>joboutput ".$self->replaced()."\n");
|
||||
if($opt::pipe and $self->virgin()) {
|
||||
# Nothing was printed to this job:
|
||||
# cleanup tmp files if --files was set
|
||||
unlink $self->fh(1,"name");
|
||||
return;
|
||||
}
|
||||
# if($opt::pipe and $self->virgin()) {
|
||||
# # Nothing was printed to this job:
|
||||
# # cleanup tmp files if --files was set
|
||||
# close $self->fh(0,"w");
|
||||
# for my $fdno (1,2) {
|
||||
# close $self->fh($fdno,"w");
|
||||
# close $self->fh($fdno,"r");
|
||||
# # Unlink files that are empty
|
||||
# unlink $self->fh($fdno,"name");
|
||||
# }
|
||||
# return;
|
||||
# }
|
||||
if($opt::dryrun) {
|
||||
# Nothing was printed to this job:
|
||||
# cleanup tmp files if --files was set
|
||||
unlink $self->fh(1,"name");
|
||||
}
|
||||
|
||||
if($Global::joblog) { $self->print_joblog() }
|
||||
if($opt::pipe and $self->virgin()) {
|
||||
} else {
|
||||
if($Global::joblog) { $self->print_joblog() }
|
||||
}
|
||||
|
||||
# Printing is only relevant for grouped output.
|
||||
$Global::grouped or return;
|
||||
|
|
@ -4957,7 +4965,14 @@ sub print {
|
|||
if($opt::files) {
|
||||
# If --compress: $in_fh must be closed first.
|
||||
close $self->fh($fdno,"w");
|
||||
if($fdno == 1 and $self->fh($fdno,"name")) {
|
||||
close $in_fh;
|
||||
if($opt::pipe and $self->virgin()) {
|
||||
# Nothing was printed to this job: # cleanup tmp files if --files was set
|
||||
for my $fdno (1,2) {
|
||||
unlink $self->fh($fdno,"name");
|
||||
unlink $self->fh($fdno,"unlink");
|
||||
}
|
||||
} elsif($fdno == 1 and $self->fh($fdno,"name")) {
|
||||
print $out_fd $self->fh($fdno,"name"),"\n";
|
||||
}
|
||||
} elsif($opt::linebuffer) {
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@
|
|||
cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -k -j4 -L1
|
||||
echo '### Test if we can deal with output > 4 GB'
|
||||
## echo | niceload --io 10 parallel -q perl -e '"\$a=\"x\"x1000000;for(0..4300){print \$a}"' | md5sum
|
||||
echo | parallel -q perl -e '$a="x"x1000000;for(0..4300){print $a}' | md5sum
|
||||
echo | parallel --tmpdir /dev/shm -q perl -e '$a="x"x1000000;for(0..4300){print $a}' | md5sum
|
||||
|
||||
echo '**'
|
||||
|
||||
|
|
|
|||
|
|
@ -1,14 +1,36 @@
|
|||
#!/bin/bash
|
||||
|
||||
cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -j0 -k -L1
|
||||
echo "### --line-buffer"
|
||||
seq 10 | parallel -j20 --line-buffer 'seq {} 10 | pv -qL 10' > /tmp/parallel_l$$;
|
||||
seq 10 | parallel -j20 'seq {} 10 | pv -qL 10' > /tmp/parallel_$$;
|
||||
cat /tmp/parallel_l$$ | wc;
|
||||
diff /tmp/parallel_$$ /tmp/parallel_l$$ >/dev/null ; echo These must diff: $?
|
||||
|
||||
echo "### --pipe --line-buffer"
|
||||
seq 200| parallel -N10 -L1 --pipe -j20 --line-buffer --tagstring {#} pv -qL 10 > /tmp/parallel_$$;
|
||||
cat /tmp/parallel_$$ | wc;
|
||||
diff <(sort /tmp/parallel_$$) /tmp/parallel_$$ >/dev/null ; echo These must diff: $?
|
||||
seq 200| parallel -N10 -L1 --pipe -j20 --line-buffer --tagstring {#} pv -qL 10 > /tmp/parallel_pl$$;
|
||||
seq 200| parallel -N10 -L1 --pipe -j20 --tagstring {#} pv -qL 10 > /tmp/parallel_p$$;
|
||||
cat /tmp/parallel_pl$$ | wc;
|
||||
diff /tmp/parallel_p$$ /tmp/parallel_pl$$ >/dev/null ; echo These must diff: $?
|
||||
|
||||
echo "### --pipe --line-buffer --compress (fails)"
|
||||
# seq 200| parallel -N10 -L1 --pipe -j20 --line-buffer --compress --tagstring {#} pv -qL 10 | wc
|
||||
|
||||
echo "### bug #41482: --pipe --compress blocks at different -j/seq combinations"
|
||||
seq 1 | parallel -k -j2 --compress -N1 -L1 --pipe cat;
|
||||
echo echo 1-4 + 1-4
|
||||
seq 4 | parallel -k -j3 --compress -N1 -L1 -vv echo;
|
||||
echo 4 times wc to stderr to stdout
|
||||
(seq 4 | parallel -k -j3 --compress -N1 -L1 --pipe wc '>&2') 2>&1 >/dev/null
|
||||
echo 1 2 3 4
|
||||
seq 4 | parallel -k -j3 --compress echo;
|
||||
echo 1 2 3 4
|
||||
seq 4 | parallel -k -j1 --compress echo;
|
||||
echo 1 2
|
||||
seq 2 | parallel -k -j1 --compress echo;
|
||||
echo 1 2 3
|
||||
seq 3 | parallel -k -j2 --compress -N1 -L1 --pipe cat;
|
||||
|
||||
echo "### bug #41609: --compress fails"
|
||||
seq 12 | parallel --compress --compress-program bzip2 -k seq {} 1000000 | md5sum
|
||||
seq 12 | parallel --compress -k seq {} 1000000 | md5sum
|
||||
|
|
|
|||
|
|
@ -1,7 +1,43 @@
|
|||
### --line-buffer
|
||||
55 55 120
|
||||
These must diff: 1
|
||||
### --pipe --line-buffer
|
||||
200 400 1202
|
||||
These must diff: 1
|
||||
### --pipe --line-buffer --compress (fails)
|
||||
### bug #41482: --pipe --compress blocks at different -j/seq combinations
|
||||
1
|
||||
echo 1-4 + 1-4
|
||||
echo 1
|
||||
1
|
||||
echo 2
|
||||
2
|
||||
echo 3
|
||||
3
|
||||
echo 4
|
||||
4
|
||||
4 times wc to stderr to stdout
|
||||
1 1 2
|
||||
1 1 2
|
||||
1 1 2
|
||||
1 1 2
|
||||
1 2 3 4
|
||||
1
|
||||
2
|
||||
3
|
||||
4
|
||||
1 2 3 4
|
||||
1
|
||||
2
|
||||
3
|
||||
4
|
||||
1 2
|
||||
1
|
||||
2
|
||||
1 2 3
|
||||
1
|
||||
2
|
||||
3
|
||||
### bug #41609: --compress fails
|
||||
24812dd0f24a26d08a780f988b9d5ad2 -
|
||||
24812dd0f24a26d08a780f988b9d5ad2 -
|
||||
|
|
|
|||
Loading…
Reference in a new issue