Fixed bug #48658: --linebuffer --files.

This commit is contained in:
Ole Tange 2016-08-27 23:24:30 +02:00
parent 49690b97e5
commit e0ac24f213
5 changed files with 59 additions and 15 deletions

View file

@ -8086,21 +8086,29 @@ sub print {
} }
sub files_print { sub files_print {
# Print the name of the file containing stdout on stdout
# Uses:
# $opt::pipe
# $opt::group = Print when job is done
# $opt::linebuffer = Print ASAP
# Returns: N/A
my $self = shift; my $self = shift;
my ($fdno,$in_fh,$out_fd) = @_; my ($fdno,$in_fh,$out_fd) = @_;
# If the job is dead: close printing fh. Needed for --compress if($opt::group and defined $self->{'exitstatus'}) {
close $self->fh($fdno,"w"); # If the job is dead: close printing fh. Needed for --compress
if($? and $opt::compress) { close $self->fh($fdno,"w");
::error($opt::compress_program." failed."); if($? and $opt::compress) {
$self->set_exitstatus(255); ::error($opt::compress_program." failed.");
$self->set_exitstatus(255);
}
if($opt::compress) {
# Kill the decompressor which will not be needed
CORE::kill "TERM", $self->fh($fdno,"rpid");
}
close $in_fh;
} }
if($opt::compress) {
# Kill the decompressor which will not be needed
CORE::kill "TERM", $self->fh($fdno,"rpid");
}
close $in_fh;
if($opt::pipe and $self->virgin()) { if($opt::pipe and $self->virgin()) {
# Nothing was printed to this job: # Nothing was printed to this job:
# cleanup unused tmp files if --files was set # cleanup unused tmp files if --files was set
@ -8115,6 +8123,8 @@ sub files_print {
$self->tag().$self->fh($fdno,"name")); $self->tag().$self->fh($fdno,"name"));
} }
$self->add_returnsize(-s $self->fh($fdno,"name")); $self->add_returnsize(-s $self->fh($fdno,"name"));
# Mark as printed - do not print again
$self->set_fh($fdno,"name",undef);
} }
} }

View file

@ -4798,6 +4798,28 @@ B<orgalorg> has a progress indicator for the transferring of a
file. GNU B<parallel> does not. file. GNU B<parallel> does not.
=head2 DIFFERENCES BETWEEN Rust parallel AND GNU Parallel
Rust parallel implements a few features from GNU B<parallel>, but
lacks many functions. All of these fail:
# -q to protect quoted $ and space
parallel -q perl -e '$a=shift; print "$a"x10000000' ::: a b c
# Argument is command
parallel {} 10 ::: echo seq
# Generation of combination of inputs
parallel echo {1} {2} ::: red green blue ::: S M L XL XXL
# Composed commands
parallel echo {} '|' wc ::: a
# Output is mixed
parallel (-q) perl -e 'print"{}"x10000000' ::: a b c |
perl -pe '$_=join "\n",split //, $_' | uniq -c
# Show what would be executed
parallel --dry-run echo ::: a
Rust parallel has no remote facilities.
=head2 DIFFERENCES BETWEEN ClusterSSH AND GNU Parallel =head2 DIFFERENCES BETWEEN ClusterSSH AND GNU Parallel
ClusterSSH solves a different problem than GNU B<parallel>. ClusterSSH solves a different problem than GNU B<parallel>.

View file

@ -696,8 +696,8 @@ into the file by the size of a block and then reading until it meets
end of a block. The seeking explains why GNU B<parallel> does not know end of a block. The seeking explains why GNU B<parallel> does not know
the line number and why B<-L/-l> and B<-N> do not work. the line number and why B<-L/-l> and B<-N> do not work.
With a reasonable block and file size this seeking is often more than With a reasonable block and file size this seeking is more than 1000
1000 faster than reading the full file. The byte positions are then time faster than reading the full file. The byte positions are then
given to a small script that reads from position X to Y and sends given to a small script that reads from position X to Y and sends
output to standard output (stdout). This small script is prepended to output to standard output (stdout). This small script is prepended to
the command and the full command is executed just as if GNU the command and the full command is executed just as if GNU
@ -787,8 +787,8 @@ The results are here:
GNU B<parallel> is pretty much limited by the speed of the disk: Up to GNU B<parallel> is pretty much limited by the speed of the disk: Up to
6 GB data is written to disk but cached, so reading is fast. Above 6 6 GB data is written to disk but cached, so reading is fast. Above 6
GB data are both written and read from disk. When the 30000MB job is GB data are both written and read from disk. When the 30000MB job is
running, the disk system is slow, but not completely unusable: If you are running, the disk system is slow, but usable: If you are not using the
not using the disk, you almost do not feel it. disk, you almost do not feel it.
B<gargs> has a speed advantage up until 2500M where it hits a B<gargs> has a speed advantage up until 2500M where it hits a
wall. Then the system starts swapping like crazy and is completely wall. Then the system starts swapping like crazy and is completely

View file

@ -641,6 +641,12 @@ echo '### bug #48745: :::+ bug'
echo '**' echo '**'
echo '### bug #48658: --linebuffer --files'
parallel --files --linebuffer 'sleep .1;seq {};sleep .1' ::: {1..10} | wc -l
echo '**'
EOF EOF
echo '### 1 .par file from --files expected' echo '### 1 .par file from --files expected'
find /tmp{/*,}/*.{par,tms,tmx} 2>/dev/null -mmin -10 | wc -l find /tmp{/*,}/*.{par,tms,tmx} 2>/dev/null -mmin -10 | wc -l

View file

@ -1759,5 +1759,11 @@ echo '### bug #48745: :::+ bug'
3 23 c cc 3 23 c cc
echo '**' echo '**'
** **
echo '### bug #48658: --linebuffer --files'
### bug #48658: --linebuffer --files
parallel --files --linebuffer 'sleep .1;seq {};sleep .1' ::: {1..10} | wc -l
10
echo '**'
**
### 1 .par file from --files expected ### 1 .par file from --files expected
0 0