diff --git a/src/parallel b/src/parallel index 3d96b77d..31083807 100755 --- a/src/parallel +++ b/src/parallel @@ -626,7 +626,7 @@ sub get_options_from_array { sub parse_options { # Returns: N/A # Defaults: - $Global::version = 20140220; + $Global::version = 20140221; $Global::progname = 'parallel'; $Global::infinity = 2**31; $Global::debug = 0; @@ -3077,7 +3077,8 @@ sub processes_available_by_system_limit { # Reserve filehandles # perl uses 7 filehandles for something? # parallel uses 1 for memory_usage - for my $i (1..8) { + # parallel uses 4 for ? + for my $i (1..12) { open($fh{"init-$i"}, "<", "/dev/null"); } @@ -3139,8 +3140,11 @@ sub processes_available_by_system_limit { $system_limit++; # Every simultaneous process uses 2 filehandles when grouping + # Every simultaneous process uses 2 filehandles when compressing $more_filehandles = open($fh{$system_limit*10}, "<", "/dev/null") - && open($fh{$system_limit*10+2}, "<", "/dev/null"); + && open($fh{$system_limit*10+2}, "<", "/dev/null") + && open($fh{$system_limit*10+3}, "<", "/dev/null") + && open($fh{$system_limit*10+4}, "<", "/dev/null"); # System process limit my $child; @@ -4982,6 +4986,17 @@ sub print { if(defined $self->{'exitstatus'}) { # If the job is dead: close printing fh. Needed for --compress close $self->fh($fdno,"w"); + if($opt::compress && $opt::linebuffer) { + # Blocked reading in final round + $Global::use{"Fcntl"} ||= eval "use Fcntl qw(:DEFAULT :flock); 1;"; + for my $fdno (1,2) { + my $fdr = $self->fh($fdno,'r'); + my $flags; + fcntl($fdr, &F_GETFL, $flags) || die $!; # Get the current flags on the filehandle + $flags &= ~&O_NONBLOCK; # Remove non-blocking to the flags + fcntl($fdr, &F_SETFL, $flags) || die $!; # Set the flags on the filehandle + } + } } # This seek will clear EOF seek $in_fh, tell($in_fh), 0; @@ -5013,12 +5028,12 @@ sub print { } } - if($self->fh($fdno,"rpid") and kill 0, $self->fh($fdno,"rpid")) { + if($self->fh($fdno,"rpid") and CORE::kill 0, $self->fh($fdno,"rpid")) { # decompress still running } elsif(defined $self->{'exitstatus'}) { # If the job is dead: print the remaining partial line # read remaining - if($opt::tag or defined $opt::tagstring) { + if($$partial and ($opt::tag or defined $opt::tagstring)) { my $tag = $self->tag(); $$partial =~ s/^/$tag/gm; } diff --git a/testsuite/tests-to-run/parallel-local114.sh b/testsuite/tests-to-run/parallel-local114.sh index 6af4d9ea..aed24914 100755 --- a/testsuite/tests-to-run/parallel-local114.sh +++ b/testsuite/tests-to-run/parallel-local114.sh @@ -13,8 +13,8 @@ echo "### --pipe --line-buffer" cat /tmp/parallel_pl$$ | wc; diff /tmp/parallel_p$$ /tmp/parallel_pl$$ >/dev/null ; echo These must diff: $? -echo "### --pipe --line-buffer --compress (fails)" -# seq 200| parallel -N10 -L1 --pipe -j20 --line-buffer --compress --tagstring {#} pv -qL 10 | wc +echo "### --pipe --line-buffer --compress" + seq 200| parallel -N10 -L1 --pipe -j20 --line-buffer --compress --tagstring {#} pv -qL 10 | wc echo "### bug #41482: --pipe --compress blocks at different -j/seq combinations" seq 1 | parallel -k -j2 --compress -N1 -L1 --pipe cat; @@ -41,6 +41,9 @@ echo "### --compress race condition (use nice): Fewer than 400 would run" echo "### -v --pipe: Dont spawn too many - 1 is enough" seq 1 | parallel -j10 -v --pipe cat +echo "### Test -N0 and --tagstring (fails)" + echo tagstring arg | parallel --tag -N0 echo foo + echo "### Test -I"; seq 1 10 | parallel -k 'seq 1 {} | parallel -k -I :: echo {} ::' diff --git a/testsuite/wanted-results/parallel-local-100s b/testsuite/wanted-results/parallel-local-100s index 487c456d..64d3d3a5 100644 --- a/testsuite/wanted-results/parallel-local-100s +++ b/testsuite/wanted-results/parallel-local-100s @@ -2,7 +2,7 @@ 46a318993dfc8e2afd71ff2bc6f605f1 - ** ### Test Force outside the file handle limit, 2009-02-17 Gave fork error -parallel: Warning: Only enough file handles to run 506 jobs in parallel. +parallel: Warning: Only enough file handles to run 252 jobs in parallel. Raising ulimit -n or /etc/security/limits.conf may help. Start end diff --git a/testsuite/wanted-results/parallel-local114 b/testsuite/wanted-results/parallel-local114 index 1eee6e28..7ac7987f 100644 --- a/testsuite/wanted-results/parallel-local114 +++ b/testsuite/wanted-results/parallel-local114 @@ -4,7 +4,8 @@ These must diff: 1 ### --pipe --line-buffer 200 400 1202 These must diff: 1 -### --pipe --line-buffer --compress (fails) +### --pipe --line-buffer --compress + 200 400 1202 ### bug #41482: --pipe --compress blocks at different -j/seq combinations 1 echo 1-4 + 1-4 @@ -46,6 +47,8 @@ echo 4 ### -v --pipe: Dont spawn too many - 1 is enough cat 1 +### Test -N0 and --tagstring (fails) + foo ### Test -I 1 1 2 1 diff --git a/testsuite/wanted-results/test13 b/testsuite/wanted-results/test13 index 9f9a5e9e..f3a98e86 100644 --- a/testsuite/wanted-results/test13 +++ b/testsuite/wanted-results/test13 @@ -1,5 +1,5 @@ ### Test -k -parallel: Warning: Only enough file handles to run 18 jobs in parallel. +parallel: Warning: Only enough file handles to run 8 jobs in parallel. Raising ulimit -n or /etc/security/limits.conf may help. begin 1 @@ -59,9 +59,10 @@ job2 7 8 9 +Raising ulimit -n or /etc/security/limits.conf may help. parallel: SIGTERM received. No new jobs will be started. parallel: Waiting for these 8 jobs to finish. Send SIGTERM again to stop now. -parallel: Warning: No more file handles. Raising ulimit -n or /etc/security/limits.conf may help. +parallel: Warning: Only enough file handles to run 8 jobs in parallel. parallel: sleep 3; echo 10 parallel: sleep 3; echo 11 parallel: sleep 3; echo 12