tange/parallel
1
0
Fork 0
mirror of https://git.savannah.gnu.org/git/parallel.git synced 2024-11-22 14:07:55 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Passes testsuite.

Browse source
This commit is contained in:
Ole Tange 2014-02-22 11:05:59 +01:00
parent 04781ab6e9
commit 36b22df1f3
5 changed files with 33 additions and 11 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

25
src/parallel
View file

@ -626,7 +626,7 @@ sub get_options_from_array {
sub parse_options { sub parse_options {
# Returns: N/A # Returns: N/A
# Defaults: # Defaults:
$Global::version = 20140220; $Global::version = 20140221;
$Global::progname = 'parallel'; $Global::progname = 'parallel';
$Global::infinity = 2**31; $Global::infinity = 2**31;
$Global::debug = 0; $Global::debug = 0;
@ -3077,7 +3077,8 @@ sub processes_available_by_system_limit {
# Reserve filehandles # Reserve filehandles
# perl uses 7 filehandles for something? # perl uses 7 filehandles for something?
# parallel uses 1 for memory_usage # parallel uses 1 for memory_usage
for my $i (1..8) { # parallel uses 4 for ?
for my $i (1..12) {
open($fh{"init-$i"}, "<", "/dev/null"); open($fh{"init-$i"}, "<", "/dev/null");
} }
@ -3139,8 +3140,11 @@ sub processes_available_by_system_limit {
$system_limit++; $system_limit++;
# Every simultaneous process uses 2 filehandles when grouping # Every simultaneous process uses 2 filehandles when grouping
# Every simultaneous process uses 2 filehandles when compressing
$more_filehandles = open($fh{$system_limit*10}, "<", "/dev/null") $more_filehandles = open($fh{$system_limit*10}, "<", "/dev/null")
&& open($fh{$system_limit*10+2}, "<", "/dev/null"); && open($fh{$system_limit*10+2}, "<", "/dev/null")
&& open($fh{$system_limit*10+3}, "<", "/dev/null")
&& open($fh{$system_limit*10+4}, "<", "/dev/null");
# System process limit # System process limit
my $child; my $child;
@ -4982,6 +4986,17 @@ sub print {
if(defined $self->{'exitstatus'}) { if(defined $self->{'exitstatus'}) {
# If the job is dead: close printing fh. Needed for --compress # If the job is dead: close printing fh. Needed for --compress
close $self->fh($fdno,"w"); close $self->fh($fdno,"w");
if($opt::compress && $opt::linebuffer) {
# Blocked reading in final round
$Global::use{"Fcntl"} ||= eval "use Fcntl qw(:DEFAULT :flock); 1;";
for my $fdno (1,2) {
my $fdr = $self->fh($fdno,'r');
my $flags;
fcntl($fdr, &F_GETFL, $flags) || die $!; # Get the current flags on the filehandle
$flags &= ~&O_NONBLOCK; # Remove non-blocking to the flags
fcntl($fdr, &F_SETFL, $flags) || die $!; # Set the flags on the filehandle
}
}
} }
# This seek will clear EOF # This seek will clear EOF
seek $in_fh, tell($in_fh), 0; seek $in_fh, tell($in_fh), 0;
@ -5013,12 +5028,12 @@ sub print {
} }
} }
if($self->fh($fdno,"rpid") and kill 0, $self->fh($fdno,"rpid")) { if($self->fh($fdno,"rpid") and CORE::kill 0, $self->fh($fdno,"rpid")) {
# decompress still running # decompress still running
} elsif(defined $self->{'exitstatus'}) { } elsif(defined $self->{'exitstatus'}) {
# If the job is dead: print the remaining partial line # If the job is dead: print the remaining partial line
# read remaining # read remaining
if($opt::tag or defined $opt::tagstring) { if($$partial and ($opt::tag or defined $opt::tagstring)) {
my $tag = $self->tag(); my $tag = $self->tag();
$$partial =~ s/^/$tag/gm; $$partial =~ s/^/$tag/gm;
} }

7
testsuite/tests-to-run/parallel-local114.sh
View file

@ -13,8 +13,8 @@ echo "### --pipe --line-buffer"
cat /tmp/parallel_pl$$ | wc; cat /tmp/parallel_pl$$ | wc;
diff /tmp/parallel_p$$ /tmp/parallel_pl$$ >/dev/null ; echo These must diff: $? diff /tmp/parallel_p$$ /tmp/parallel_pl$$ >/dev/null ; echo These must diff: $?
echo "### --pipe --line-buffer --compress (fails)" echo "### --pipe --line-buffer --compress"
# seq 200| parallel -N10 -L1 --pipe -j20 --line-buffer --compress --tagstring {#} pv -qL 10 | wc seq 200| parallel -N10 -L1 --pipe -j20 --line-buffer --compress --tagstring {#} pv -qL 10 | wc
echo "### bug #41482: --pipe --compress blocks at different -j/seq combinations" echo "### bug #41482: --pipe --compress blocks at different -j/seq combinations"
seq 1 | parallel -k -j2 --compress -N1 -L1 --pipe cat; seq 1 | parallel -k -j2 --compress -N1 -L1 --pipe cat;
@ -41,6 +41,9 @@ echo "### --compress race condition (use nice): Fewer than 400 would run"
echo "### -v --pipe: Dont spawn too many - 1 is enough" echo "### -v --pipe: Dont spawn too many - 1 is enough"
seq 1 | parallel -j10 -v --pipe cat seq 1 | parallel -j10 -v --pipe cat
echo "### Test -N0 and --tagstring (fails)"
echo tagstring arg | parallel --tag -N0 echo foo
echo "### Test -I"; echo "### Test -I";
seq 1 10 | parallel -k 'seq 1 {} | parallel -k -I :: echo {} ::' seq 1 10 | parallel -k 'seq 1 {} | parallel -k -I :: echo {} ::'

2
testsuite/wanted-results/parallel-local-100s
View file

@ -2,7 +2,7 @@
46a318993dfc8e2afd71ff2bc6f605f1 - 46a318993dfc8e2afd71ff2bc6f605f1 -
** **
### Test Force outside the file handle limit, 2009-02-17 Gave fork error ### Test Force outside the file handle limit, 2009-02-17 Gave fork error
parallel: Warning: Only enough file handles to run 506 jobs in parallel. parallel: Warning: Only enough file handles to run 252 jobs in parallel.
Raising ulimit -n or /etc/security/limits.conf may help. Raising ulimit -n or /etc/security/limits.conf may help.
Start Start
end end

5
testsuite/wanted-results/parallel-local114
View file

@ -4,7 +4,8 @@ These must diff: 1
### --pipe --line-buffer ### --pipe --line-buffer
200 400 1202 200 400 1202
These must diff: 1 These must diff: 1
### --pipe --line-buffer --compress (fails) ### --pipe --line-buffer --compress
200 400 1202
### bug #41482: --pipe --compress blocks at different -j/seq combinations ### bug #41482: --pipe --compress blocks at different -j/seq combinations
1 1
echo 1-4 + 1-4 echo 1-4 + 1-4
@ -46,6 +47,8 @@ echo 4
### -v --pipe: Dont spawn too many - 1 is enough ### -v --pipe: Dont spawn too many - 1 is enough
cat cat
1 1
### Test -N0 and --tagstring (fails)
foo
### Test -I ### Test -I
1 1 1 1
2 1 2 1

5
testsuite/wanted-results/test13
View file

@ -1,5 +1,5 @@
### Test -k ### Test -k
parallel: Warning: Only enough file handles to run 18 jobs in parallel. parallel: Warning: Only enough file handles to run 8 jobs in parallel.
Raising ulimit -n or /etc/security/limits.conf may help. Raising ulimit -n or /etc/security/limits.conf may help.
begin begin
1 1
@ -59,9 +59,10 @@ job2
7 7
8 8
9 9
Raising ulimit -n or /etc/security/limits.conf may help.
parallel: SIGTERM received. No new jobs will be started. parallel: SIGTERM received. No new jobs will be started.
parallel: Waiting for these 8 jobs to finish. Send SIGTERM again to stop now. parallel: Waiting for these 8 jobs to finish. Send SIGTERM again to stop now.
parallel: Warning: No more file handles. Raising ulimit -n or /etc/security/limits.conf may help. parallel: Warning: Only enough file handles to run 8 jobs in parallel.
parallel: sleep 3; echo 10 parallel: sleep 3; echo 10
parallel: sleep 3; echo 11 parallel: sleep 3; echo 11
parallel: sleep 3; echo 12 parallel: sleep 3; echo 12