parallel: Fixed bug #54328: --pipe --recend '' blocks.

doc/release_new_version

@@ -199,9 +199,9 @@ to:parallel@gnu.org, bug-parallel@gnu.org
 stable-bcc: Jesse Alama <jessealama@fastmail.fm>
 
 
-Subject: GNU Parallel 20180622 ('Kim Trump/GDPR/Guatamala') released <<[stable]>>
+Subject: GNU Parallel 20180722 ('?') released <<[stable]>>
 
-GNU Parallel 20180622 ('') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/
+GNU Parallel 20180722 ('') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/
 
 <<No new functionality was introduced so this is a good candidate for a stable release.>>
 
@@ -214,6 +214,11 @@ Quote of the month:
 
 New in this release:
 
+* The CPU calculation has changed. By default GNU Parallel uses the number of CPU threads as the number of CPUs. This can be change to the number of CPU cores or number of CPU sockets with --use-cores-instead-of-threads or --use-sockets-instead-of-threads.
+
+* The detected number of sockets, cores, and threads can be shown with --number-of-sockets, --number-of-cores, and --number-of-threads.
+
+* env_parallel now support mksh using env_parallel.mksh.
 
 * GNU Parallel is distributed as part of Snippy https://github.com/tseemann/snippy
 

src/parallel

@@ -519,12 +519,13 @@ sub spreadstdin {
 		  $Global::max_number_of_args * ($Global::max_lines || 1);
 	      # (?!negative lookahead) is needed to avoid backtracking
 	      # See: https://unix.stackexchange.com/questions/439356/
-	      while($buf =~ /# From start up till recend
+	      while($buf =~
-                             ^((?:(?!$recend$recstart).)*?$recend
+		    /# From start up till recend
-                             # Then n-1 times recstart.*recend
+		    ^((?:(?!$recend$recstart).)*?$recend
-                             (?:$recstart(?:(?!$recend$recstart).)*?$recend){$read_n_lines})
+		    # Then n-1 times recstart.*recend
-                             # Followed by recstart
+		    (?:$recstart(?:(?!$recend$recstart).)*?$recend){$read_n_lines})
-                             (?=$recstart)/osx) {
+		    # Followed by recstart
+		    (?=$recstart)/osx) {
 		  $anything_written +=
 		      write_record_to_pipe($chunk_number++,\$header,\$buf,
 					   $recstart,$recend,length $1);
@@ -561,7 +562,9 @@ sub spreadstdin {
 	      my $i = 0;
 	      my $read_n_lines =
 		  $Global::max_number_of_args * ($Global::max_lines || 1);
-	      while(($i = nindex(\$buf,$recendrecstart,$read_n_lines)) != -1) {
+	      while(($i = nindex(\$buf,$recendrecstart,$read_n_lines)) != -1
+		    and
+		    length $buf) {
 		  $i += length $recend; # find the actual splitting location
 		  $anything_written +=
 		      write_record_to_pipe($chunk_number++,\$header,\$buf,
@@ -657,6 +660,10 @@ sub recstartrecend {
 	# If --recend is given then it must match end of record
 	$recstart = "";
 	$recend = $opt::recend;
+	if($opt::regexp and $recend eq '') {
+	    # --regexp --recend ''
+	    $recend = '.';
+	}
     }
 
     if($opt::regexp) {
@@ -1385,10 +1392,10 @@ sub parse_options {
     if(defined $opt::eta) { $opt::progress = $opt::eta; }
     if(defined $opt::bar) { $opt::progress = $opt::bar; }
 
-    # Funding a free software project is hard. GNU parallel is no
+    # Funding a free software project is hard. GNU Parallel is no
     # exception. On top of that it seems the less visible a project
     # is, the harder it is to get funding. And the nature of GNU
-    # parallel is that it will never be seen by "the guy with the
+    # Parallel is that it will never be seen by "the guy with the
     # checkbook", but only by the people doing the actual work.
     #
     # This problem has been covered by others - though no solution has
@@ -1402,7 +1409,7 @@ sub parse_options {
     #
     # Having to spend 10 seconds on running 'parallel --citation' once
     # is no doubt not an ideal solution, but no one has so far come up
-    # with an ideal solution - neither for funding GNU parallel nor
+    # with an ideal solution - neither for funding GNU Parallel nor
     # other free software.
     #
     # If you believe you have the perfect solution, you should try it
@@ -1418,9 +1425,11 @@ sub parse_options {
     #
     # Description of the xt:Commerce case in OLG Duesseldorf
     # http://www.inta.org/INTABulletin/Pages/GERMANYGeneralPublicLicenseDoesNotPermitUseofThird-PartyTrademarksforAdvertisingModifiedVersionsofOpen-SourceSoftware.aspx
+    # https://web.archive.org/web/20180715073746/http://www.inta.org/INTABulletin/Pages/GERMANYGeneralPublicLicenseDoesNotPermitUseofThird-PartyTrademarksforAdvertisingModifiedVersionsofOpen-SourceSoftware.aspx
     #
     # The verdict in German
     # https://www.admody.com/urteilsdatenbank/cafe6fdaeed3/OLG-Duesseldorf_Urteil_vom_28-September-2010_Az_I-20-U-41-09
+    # https://web.archive.org/web/20180715073717/https://www.admody.com/urteilsdatenbank/cafe6fdaeed3/OLG-Duesseldorf_Urteil_vom_28-September-2010_Az_I-20-U-41-09
     #
     # Other free software limiting derivates by the same name
     # https://en.wikipedia.org/wiki/Red_Hat_Enterprise_Linux_derivatives#Legal_aspects
@@ -4263,6 +4272,26 @@ sub citation_notice {
 	     "To silence this citation notice: run 'parallel --citation' once.",
 	     ""
 	    );
+	mkdir $Global::config_dir;
+	# Number of times the user has run GNU Parallel without showing
+	# willingness to cite
+	my $runs = 0;
+	if(open (my $fh, "<", $Global::config_dir.
+		 "/runs-without-willing-to-cite")) {
+	    $runs = <$fh>;
+	    close $fh;
+	}
+	$runs++;
+	if(open (my $fh, ">", $Global::config_dir.
+		 "/runs-without-willing-to-cite")) {
+	    print $fh $runs;
+	    close $fh;
+	    if($runs >= 10) {
+		::status("Come on: You have run parallel $runs times. Isn't it about time ",
+			 "you run 'parallel --citation' once to silence the citation notice?",
+			 "");
+	    }
+	}
     }
 }
 
@@ -9175,6 +9204,7 @@ sub print_linebuffer {
 	    my ($buf,$i,$rv);
 	    while($rv = sysread($in_fh, $buf, 131072)) {
 		$outputlength += $rv;
+		# TODO --recend
 		# Treat both \n and \r as line end
 		$i = (rindex($buf,"\n")+1) || (rindex($buf,"\r")+1);
 		if($i) {
@@ -9182,6 +9212,7 @@ sub print_linebuffer {
 		    if($opt::tag or defined $opt::tagstring) {
 			# Replace ^ with $tag within the full line
 			my $tag = $self->tag();
+			# TODO --recend
 			substr($buf,0,$i-1) =~ s/(?<=[\n\r])/$tag/gm;
 			# The length changed, so find the new ending pos
 			$i = (rindex($buf,"\n")+1) || (rindex($buf,"\r")+1);

src/parallel.pod

@@ -1056,11 +1056,11 @@ If used with B<--pipe --roundrobin> and the same input, the jobslots
 will get the same blocks in the same order in every run.
 
 
-=item B<-L> I<max-lines>
+=item B<-L> I<recsize>
 
-When used with B<--pipe>: Read records of I<max-lines>.
+When used with B<--pipe>: Read records of I<recsize>.
 
-When used otherwise: Use at most I<max-lines> nonblank input lines per
+When used otherwise: Use at most I<recsize> nonblank input lines per
 command line.  Trailing blanks cause an input line to be logically
 continued on the next input line.
 
@@ -1070,14 +1070,14 @@ line.
 Implies B<-X> unless B<-m>, B<--xargs>, or B<--pipe> is set.
 
 
-=item B<--max-lines>[=I<max-lines>]
+=item B<--max-lines>[=I<recsize>]
 
-=item B<-l>[I<max-lines>]
+=item B<-l>[I<recsize>]
 
-When used with B<--pipe>: Read records of I<max-lines>.
+When used with B<--pipe>: Read records of I<recsize> lines.
 
 When used otherwise: Synonym for the B<-L> option.  Unlike B<-L>, the
-I<max-lines> argument is optional.  If I<max-lines> is not specified,
+I<recsize> argument is optional.  If I<recsize> is not specified,
 it defaults to one.  The B<-l> option is deprecated since the POSIX
 standard specifies B<-L> instead.
 
@@ -3027,6 +3027,19 @@ B<parallel> finally uses B<wget> to fetch the images.
     parallel wget
 
 
+=head1 EXAMPLE: Download video playlist in parallel
+
+B<youtube-dl> is an excellent tool to download videos. It can,
+however, not download videos in parallel. This takes a playlist and
+downloads 10 videos in parallel.
+
+  url='https://youtu.be/watch?v=0wOf2Fgi3DE&list=UU_cznB5YZZmvAmeq7Y3EriQ'
+  export url
+  youtube-dl --flat-playlist "$url" |
+    parallel --tagstring {#} --lb -j10 \
+      LC_ALL= youtube-dl --playlist-start {#} --playlist-end {#} '"$url"'
+
+
 =head1 EXAMPLE: Copy files as last modified date (ISO8601) with added
 random digits
 

testsuite/tests-to-run/parallel-local-0.3s.sh

@@ -819,6 +819,12 @@ par_PARALLEL_ENV() {
     rm $PARALLEL_ENV
 }
 
+par_pipe_recend() {
+    echo 'bug #54328: --pipe --recend '' blocks'
+    seq 3 | parallel -k --pipe --regexp --recend '' -n 1 xxd
+    seq 3 | parallel -k --pipe --recend '' -n 1 xxd
+}
+
 export -f $(compgen -A function | grep par_)
 compgen -A function | grep par_ | sort |
     parallel -j6 --tag -k --joblog +/tmp/jl-`basename $0` '{} 2>&1'

testsuite/wanted-results/parallel-local-0.3s

@@ -1388,6 +1388,19 @@ par_parcat_rm	OK1
 par_parcat_rm	OK file removed
 par_pipe_no_command	### --pipe without command
 par_pipe_no_command	parallel: Error: --pipe/--pipepart must have a command to pipe into (e.g. 'cat').
+par_pipe_recend	bug #54328: --pipe --recend  blocks
+par_pipe_recend	00000000: 31                                       1
+par_pipe_recend	00000000: 0a                                       .
+par_pipe_recend	00000000: 32                                       2
+par_pipe_recend	00000000: 0a                                       .
+par_pipe_recend	00000000: 33                                       3
+par_pipe_recend	00000000: 0a                                       .
+par_pipe_recend	00000000: 31                                       1
+par_pipe_recend	00000000: 0a                                       .
+par_pipe_recend	00000000: 32                                       2
+par_pipe_recend	00000000: 0a                                       .
+par_pipe_recend	00000000: 33                                       3
+par_pipe_recend	00000000: 0a                                       .
 par_pipe_record_size_in_lines	bug #34958: --pipe with record size measured in lines
 par_pipe_record_size_in_lines	1
 par_pipe_record_size_in_lines	2