From e0ac24f213957c344c6fa1446ee05811e93b1c46 Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Sat, 27 Aug 2016 23:24:30 +0200 Subject: [PATCH] Fixed bug #48658: --linebuffer --files. --- src/parallel | 32 ++++++++++++------- src/parallel.pod | 22 +++++++++++++ src/parallel_design.pod | 8 ++--- testsuite/tests-to-run/parallel-local-0.3s.sh | 6 ++++ testsuite/wanted-results/parallel-local-0.3s | 6 ++++ 5 files changed, 59 insertions(+), 15 deletions(-) diff --git a/src/parallel b/src/parallel index 9b0744bd..dafaee5e 100755 --- a/src/parallel +++ b/src/parallel @@ -8086,21 +8086,29 @@ sub print { } sub files_print { + # Print the name of the file containing stdout on stdout + # Uses: + # $opt::pipe + # $opt::group = Print when job is done + # $opt::linebuffer = Print ASAP + # Returns: N/A my $self = shift; my ($fdno,$in_fh,$out_fd) = @_; - # If the job is dead: close printing fh. Needed for --compress - close $self->fh($fdno,"w"); - if($? and $opt::compress) { - ::error($opt::compress_program." failed."); - $self->set_exitstatus(255); + if($opt::group and defined $self->{'exitstatus'}) { + # If the job is dead: close printing fh. Needed for --compress + close $self->fh($fdno,"w"); + if($? and $opt::compress) { + ::error($opt::compress_program." failed."); + $self->set_exitstatus(255); + } + if($opt::compress) { + # Kill the decompressor which will not be needed + CORE::kill "TERM", $self->fh($fdno,"rpid"); + } + close $in_fh; } - if($opt::compress) { - # Kill the decompressor which will not be needed - CORE::kill "TERM", $self->fh($fdno,"rpid"); - } - close $in_fh; - + if($opt::pipe and $self->virgin()) { # Nothing was printed to this job: # cleanup unused tmp files if --files was set @@ -8115,6 +8123,8 @@ sub files_print { $self->tag().$self->fh($fdno,"name")); } $self->add_returnsize(-s $self->fh($fdno,"name")); + # Mark as printed - do not print again + $self->set_fh($fdno,"name",undef); } } diff --git a/src/parallel.pod b/src/parallel.pod index b7cf8a72..097a9fc5 100644 --- a/src/parallel.pod +++ b/src/parallel.pod @@ -4798,6 +4798,28 @@ B has a progress indicator for the transferring of a file. GNU B does not. +=head2 DIFFERENCES BETWEEN Rust parallel AND GNU Parallel + +Rust parallel implements a few features from GNU B, but +lacks many functions. All of these fail: + + # -q to protect quoted $ and space + parallel -q perl -e '$a=shift; print "$a"x10000000' ::: a b c + # Argument is command + parallel {} 10 ::: echo seq + # Generation of combination of inputs + parallel echo {1} {2} ::: red green blue ::: S M L XL XXL + # Composed commands + parallel echo {} '|' wc ::: a + # Output is mixed + parallel (-q) perl -e 'print"{}"x10000000' ::: a b c | + perl -pe '$_=join "\n",split //, $_' | uniq -c + # Show what would be executed + parallel --dry-run echo ::: a + +Rust parallel has no remote facilities. + + =head2 DIFFERENCES BETWEEN ClusterSSH AND GNU Parallel ClusterSSH solves a different problem than GNU B. diff --git a/src/parallel_design.pod b/src/parallel_design.pod index ff96b1c1..b9693049 100644 --- a/src/parallel_design.pod +++ b/src/parallel_design.pod @@ -696,8 +696,8 @@ into the file by the size of a block and then reading until it meets end of a block. The seeking explains why GNU B does not know the line number and why B<-L/-l> and B<-N> do not work. -With a reasonable block and file size this seeking is often more than -1000 faster than reading the full file. The byte positions are then +With a reasonable block and file size this seeking is more than 1000 +time faster than reading the full file. The byte positions are then given to a small script that reads from position X to Y and sends output to standard output (stdout). This small script is prepended to the command and the full command is executed just as if GNU @@ -787,8 +787,8 @@ The results are here: GNU B is pretty much limited by the speed of the disk: Up to 6 GB data is written to disk but cached, so reading is fast. Above 6 GB data are both written and read from disk. When the 30000MB job is -running, the disk system is slow, but not completely unusable: If you are -not using the disk, you almost do not feel it. +running, the disk system is slow, but usable: If you are not using the +disk, you almost do not feel it. B has a speed advantage up until 2500M where it hits a wall. Then the system starts swapping like crazy and is completely diff --git a/testsuite/tests-to-run/parallel-local-0.3s.sh b/testsuite/tests-to-run/parallel-local-0.3s.sh index c90b43b5..b1ef1c9d 100644 --- a/testsuite/tests-to-run/parallel-local-0.3s.sh +++ b/testsuite/tests-to-run/parallel-local-0.3s.sh @@ -641,6 +641,12 @@ echo '### bug #48745: :::+ bug' echo '**' +echo '### bug #48658: --linebuffer --files' + + parallel --files --linebuffer 'sleep .1;seq {};sleep .1' ::: {1..10} | wc -l + +echo '**' + EOF echo '### 1 .par file from --files expected' find /tmp{/*,}/*.{par,tms,tmx} 2>/dev/null -mmin -10 | wc -l diff --git a/testsuite/wanted-results/parallel-local-0.3s b/testsuite/wanted-results/parallel-local-0.3s index 38deddfc..c68ee7a9 100644 --- a/testsuite/wanted-results/parallel-local-0.3s +++ b/testsuite/wanted-results/parallel-local-0.3s @@ -1759,5 +1759,11 @@ echo '### bug #48745: :::+ bug' 3 23 c cc echo '**' ** +echo '### bug #48658: --linebuffer --files' +### bug #48658: --linebuffer --files + parallel --files --linebuffer 'sleep .1;seq {};sleep .1' ::: {1..10} | wc -l +10 +echo '**' +** ### 1 .par file from --files expected 0