Fixed bug #52740: Bash redirection with process substitution blocks.

doc/release_new_version

@@ -210,20 +210,6 @@ Haiku of the month:
 
 New in this release:
 
-* env_parset for ash, dash, ksh, sh, zsh
-
-* Automatically create hostgroups if argument ends in @sshlogin
-
-* Tips for running jobs on your VM https://support.ehelp.edu.au/support/solutions/articles/6000089713-tips-for-running-jobs-on-your-vm#parallel
-
-* Node-local scheduling https://mogonwiki.zdv.uni-mainz.de/dokuwiki/node_local_scheduling
-
-* GNU Parallel https://docs.computecanada.ca/wiki/GNU_Parallel
-
-* GNU Parallel のエラー処理 https://qiita.com/inouet/items/bfc208668c86caf8ff74
-
-* Using GNU Parallel https://wiki.ncsa.illinois.edu/display/ROGER/Using+GNU+Parallel
-
 
 <<Citation not OK: BAMClipper: removing primers from alignments to minimize false-negative mutations in amplicon next-generation sequencing https://www.nature.com/articles/s41598-017-01703-6>>
 

src/niceload

@@ -24,7 +24,7 @@
 use strict;
 use Getopt::Long;
 $Global::progname="niceload";
-$Global::version = 20171222;
+$Global::version = 20171224;
 Getopt::Long::Configure("bundling","require_order");
 get_options_from_array(\@ARGV) || die_usage();
 if($opt::version) {

src/parallel

@@ -1100,8 +1100,13 @@ sub get_options_from_array {
 sub parse_options {
     # Returns: N/A
     init_globals();
+    my @argv_before = @ARGV;
     @ARGV = read_options();
 
+    if(defined $opt::citation) {
+	citation(\@argv_before,\@ARGV);
+	wait_and_exit(0);
+    }
     # no-* overrides *
     if($opt::nokeeporder) { $opt::keeporder = undef; }
 
@@ -1162,7 +1167,6 @@ sub parse_options {
         print Limits::Command::real_max_length(),"\n"; wait_and_exit(0);
     }
     if(defined $opt::version) { version(); wait_and_exit(0); }
-    if(defined $opt::citation) { citation(); wait_and_exit(0); }
     if(defined $opt::record_env) { record_env(); wait_and_exit(0); }
     if(defined $opt::show_limits) { show_limits(); }
     if(@opt::sshlogin) { @Global::sshlogin = @opt::sshlogin; }
@@ -1387,7 +1391,7 @@ sub check_invalid_option_combinations {
 
 sub init_globals {
     # Defaults:
-    $Global::version = 20171222;
+    $Global::version = 20171224;
     $Global::progname = 'parallel';
     $Global::infinity = 2**31;
     $Global::debug = 0;
@@ -2750,8 +2754,8 @@ sub drain_job_queue {
 	    if($Global::start_sqlworker) {
 		# Start an SQL worker as we are now sure there is work to do
 		$Global::start_sqlworker = 0;
-		if(fork()) {
+		if(my $pid = fork()) {
-		    # skip
+		    $Global::unkilled_sqlworker = $pid;
 		} else {
 		    # Replace --sql/--sqlandworker with --sqlworker
 		    my @ARGV = map { s/^--sql(andworker)?$/--sqlworker/; $_ } @Global::options_in_argv;
@@ -4014,7 +4018,9 @@ sub wait_and_exit {
         waitpid($_,0);
         delete $Global::unkilled_children{$_};
     }
-    wait();
+    if($Global::unkilled_sqlworker) {
+	waitpid($Global::unkilled_sqlworker,0);
+    }
     exit($error);
 }
 
@@ -4160,6 +4166,15 @@ sub version {
 
 sub citation {
     # Returns: N/A
+    my ($all_argv_ref,$argv_options_removed_ref) = @_;
+    my $all_argv = "@$all_argv_ref";
+    my $no_opts = "@$argv_options_removed_ref";
+    $all_argv=~s/--citation//;
+    if($all_argv ne $no_opts) {
+	::warning("--citation ignores all other options and arguments.");
+	::status("");
+    }
+
     ::status(
 	  "Academic tradition requires you to cite works you base your article on.",
 	  "If you use programs that use GNU Parallel to process data for an article in a",

src/parallel.pod

@@ -470,8 +470,8 @@ Implies B<--semaphore>.
 
 =item B<--citation>
 
-Print the BibTeX entry for GNU B<parallel> and silence citation
+Print the BibTeX entry for GNU B<parallel>, silence citation
-notice.
+notice and exit. It will not run any commands.
 
 If it is impossible for you to run B<--bibtex> you can use
 B<--will-cite>.

src/parallel_alternatives.pod

@@ -1327,6 +1327,47 @@ It can be emulated with GNU B<parallel> using this Bash function:
 
 This works execpt for the B<--exclude> option.
 
+=head2 DIFFERENCES BETWEEN pyargs AND GNU Parallel
+
+B<pyargs> deals badly with input containing spaces. It buffers stdout,
+but not stderr. It buffers in RAM. {} does not work as replacement
+string. It does not support running functions.
+
+B<pyargs> does not support composed commands if run with B<--lines>,
+and fails on B<pyargs traceroute pi.dk fi.dk>.
+
+=head3 Examples
+
+  seq 5 | pyargs -P50 -L seq
+  seq 5 | parallel -P50 --lb seq
+
+  seq 5 | pyargs -P50 --mark -L seq
+  seq 5 | parallel -P50 --lb \
+    --tagstring OUTPUT'[{= $_=$job->replaced()=}]' seq
+  # Similar, but not precisely the same
+  seq 5 | parallel -P50 --lb --tag seq
+
+  seq 5 | pyargs -P50  --mark command
+  # Somewhat longer with GNU Parallel due to the special
+  #   --mark formatting
+  cmd="$(echo "command" | parallel --shellquote)"
+  wrap_cmd() {
+     echo "MARK $cmd $@================================" >&3
+     echo "OUTPUT START[$cmd $@]:"
+     eval $cmd "$@"
+     echo "OUTPUT END[$cmd $@]"
+  }
+  (seq 5 | env_parallel -P2 wrap_cmd) 3>&1
+  # Similar, but not exactly the same
+  seq 5 | parallel -t --tag command
+
+  (echo '1  2  3';echo 4 5 6) | pyargs  --stream seq
+  (echo '1  2  3';echo 4 5 6) | perl -pe 's/\n/ /' |
+    parallel -r -d' ' seq
+  # Similar, but not exactly the same
+  parallel seq ::: 1 2 3 4 5 6
+
+
 =head2 Todo
 
 machma. Requires Go >= 1.7.
@@ -1344,6 +1385,8 @@ https://github.com/Julian/Verge
 
 https://github.com/amattn/paral
 
+pyargs
+
 
 =head1 TESTING OTHER TOOLS
 
@@ -1363,18 +1406,22 @@ will most likely mix.
   cat <<-EOF > mycommand
   #!/bin/bash
 
-  # If 'a', 'b' and 'c' mix: Very bad
+  # If a, b, c, d, e, and f mix: Very bad
-  perl -e 'print "a"x3000_000," "'
+  perl -e 'print STDOUT "a"x3000_000," "'
-  perl -e 'print "b"x3000_000," "'
+  perl -e 'print STDERR "b"x3000_000," "'
-  perl -e 'print "c"x3000_000," "'
+  perl -e 'print STDOUT "c"x3000_000," "'
+  perl -e 'print STDERR "d"x3000_000," "'
+  perl -e 'print STDOUT "e"x3000_000," "'
+  perl -e 'print STDERR "f"x3000_000," "'
   echo
+  echo >&2
   EOF
   chmod +x mycommand
 
   # Run 30 jobs in parallel
-  seq 30 | $paralleltool -j0 ./mycommand | tr -s abc
+  seq 30 | $paralleltool -j0 ./mycommand > >(tr -s abcdef) 2> >(tr -s abcdef >&2)
 
-  # 'a b c' should always stay together
+  # 'a c e' and 'b d f' should always stay together
   # and there should only be a single line per job
 
 =head2 B: Output limited by RAM

src/sql

@@ -576,7 +576,7 @@ $Global::Initfile && unlink $Global::Initfile;
 exit ($err);
 
 sub parse_options {
-    $Global::version = 20171222;
+    $Global::version = 20171224;
     $Global::progname = 'sql';
 
     # This must be done first as this may exec myself

testsuite/Makefile

@@ -94,7 +94,7 @@ portable:
 	time bash Portable.sh
 
 timings: tests-to-run/* ../src/parallel
-	ls tests-to-run/*.sh | parallel echo /usr/bin/time -f %e bash >/tmp/timing.script
+	ls tests-to-run/*.sh | parallel echo /usr/bin/time -f "'%e\ %P'" bash >/tmp/timing.script
 	stdout bash -x /tmp/timing.script | tee /tmp/timing.out
 	echo + .usr.bin.time_END >>/tmp/timing.out
 	perl -ne '/\+ .usr.bin.time/ and do { print $$last.$$h; $$h=$$_ }; chomp; s/.*\0//;$$last = $$_' /tmp/timing.out |sort -n >timings

testsuite/tests-to-run/parallel-local-0.3s.sh

@@ -804,6 +804,17 @@ par_halt_one_job() {
     parallel -j1 --halt soon,fail=1 'echo {#};exit {}' ::: 1 0 1
 }
 
+par_blocking_redir() {
+    (
+    echo 'bug #52740: Bash redirection with process substitution blocks'
+    echo Test stdout
+    echo 3 | parallel seq > >(echo stdout;wc) 2> >(echo stderr >&2; wc >&2)
+    echo Test stderr
+    echo nOfilE | parallel ls > >(echo stdout;wc) 2> >(echo stderr >&2; wc >&2)
+    ) 2>&1 | sort
+}
+
+
 export -f $(compgen -A function | grep par_)
 compgen -A function | grep par_ | sort |
     parallel -j6 --tag -k --joblog +/tmp/jl-`basename $0` '{} 2>&1'

testsuite/wanted-results/parallel-local-0.3s

@@ -1344,6 +1344,17 @@ par_basic_halt	true
 par_basic_halt	parallel: Error: --halt must have 'never', 'soon', or 'now'.
 par_basic_halt	parallel: Error: --halt soon must be followed by ,success or ,fail.
 par_basic_halt	parallel: Error: --halt now must be followed by ,success or ,fail.
+par_blocking_redir	      0       0       0
+par_blocking_redir	      0       0       0
+par_blocking_redir	      1       9      54
+par_blocking_redir	      3       3       6
+par_blocking_redir	Test stderr
+par_blocking_redir	Test stdout
+par_blocking_redir	bug #52740: Bash redirection with process substitution blocks
+par_blocking_redir	stderr
+par_blocking_redir	stderr
+par_blocking_redir	stdout
+par_blocking_redir	stdout
 par_colsep_0	bug --colsep 0
 par_colsep_0	OK
 par_colsep_0	OK