env_parallel.*sh: Parsing 'type' depends on $LANG. Set LANG=C.

--tee: No longer spawns cat, but uses redirection instead.

doc/release_new_version

@@ -199,7 +199,7 @@ to:parallel@gnu.org, bug-parallel@gnu.org
 stable-bcc: Jesse Alama <jessealama@fastmail.fm>
 
 
-Subject: GNU Parallel 20180422 ('Trèbes') released <<[stable]>>
+Subject: GNU Parallel 20180422 ('Trèbes/Tiangong-1/Winnie Mandela/ASIM/Algeria') released <<[stable]>>
 
 GNU Parallel 20180422 ('Trèbes') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/
 
@@ -212,6 +212,13 @@ Quote of the month:
 
 New in this release:
 
+Launch at DIKU
+
+https://userinfo.surfsara.nl/documentation/running-many-serial-jobs-efficiently
+
+http://fliplinux.com/gnu-parallel-6.html
+
+https://www.youtube.com/watch?v=sHpTywpb4_4
 
 <<Citation not OK: BAMClipper: removing primers from alignments to minimize false-negative mutations in amplicon next-generation sequencing https://www.nature.com/articles/s41598-017-01703-6>>
 

src/env_parallel.ash

@@ -51,11 +51,11 @@ env_parallel() {
     }
     _names_of_FUNCTIONS() {
 	# myfunc is a function
-	type `_names_of_maybe_FUNCTIONS` |
+	LANG=C type `_names_of_maybe_FUNCTIONS` |
 	    perl -ne '/^(\S+) is a function$/ and not $seen{$1}++ and print "$1\n"'
     }
     _bodies_of_FUNCTIONS() {
-	type "$@" | perl -ne '/^(\S+) is a function$/ or print'
+	LANG=C type "$@" | perl -ne '/^(\S+) is a function$/ or print'
     }
     _names_of_VARIABLES() {
 	# This may screw up if variables contain \n and =
@@ -131,7 +131,7 @@ env_parallel() {
 	#   aliased to `alias | /usr/bin/which --tty-only --read-alias --show-dot --show-tilde'
 	#   parallel is a tracked alias for /usr/local/bin/parallel (ksh)
 	# Return 0 if found, 1 otherwise
-	type "$@" |
+	LANG=C type "$@" |
 	    perl -pe '$exit += (s/ is an alias for .*// ||
 	                        s/ is aliased to .*// ||
                                 s/ is a function// ||

src/env_parallel.bash

@@ -152,7 +152,7 @@ env_parallel() {
 	#   which is hashed (/usr/bin/which)
 	#   aliased to `alias | /usr/bin/which --tty-only --read-alias --show-dot --show-tilde'
 	# Return 0 if found, 1 otherwise
-	type "$@" |
+	LANG=C type "$@" |
 	    perl -pe '$exit += (s/ is an alias for .*// ||
 	                        s/ is aliased to .*// ||
                                 s/ is a function// ||

src/env_parallel.dash

@@ -51,11 +51,11 @@ env_parallel() {
     }
     _names_of_FUNCTIONS() {
 	# myfunc is a function
-	type `_names_of_maybe_FUNCTIONS` |
+	LANG=C type `_names_of_maybe_FUNCTIONS` |
 	    perl -ne '/^(\S+) is a function$/ and not $seen{$1}++ and print "$1\n"'
     }
     _bodies_of_FUNCTIONS() {
-	type "$@" | perl -ne '/^(\S+) is a function$/ or print'
+	LANC=C type "$@" | perl -ne '/^(\S+) is a function$/ or print'
     }
     _names_of_VARIABLES() {
 	# This may screw up if variables contain \n and =
@@ -131,7 +131,7 @@ env_parallel() {
 	#   aliased to `alias | /usr/bin/which --tty-only --read-alias --show-dot --show-tilde'
 	#   parallel is a tracked alias for /usr/local/bin/parallel (ksh)
 	# Return 0 if found, 1 otherwise
-	type "$@" |
+	LANG=C type "$@" |
 	    perl -pe '$exit += (s/ is an alias for .*// ||
 	                        s/ is aliased to .*// ||
                                 s/ is a function// ||

src/env_parallel.ksh

@@ -111,7 +111,7 @@ env_parallel() {
 	#   aliased to `alias | /usr/bin/which --tty-only --read-alias --show-dot --show-tilde'
 	#   parallel is a tracked alias for /usr/local/bin/parallel (ksh)
 	# Return 0 if found, 1 otherwise
-	type "$@" |
+	LANG=C type "$@" |
 	    perl -pe '$exit += (s/ is an alias for .*// ||
 	                        s/ is aliased to .*// ||
                                 s/ is a function// ||

src/env_parallel.sh

@@ -51,11 +51,11 @@ env_parallel() {
     }
     _names_of_FUNCTIONS() {
 	# myfunc is a function
-	type `_names_of_maybe_FUNCTIONS` |
+	LANG=C type `_names_of_maybe_FUNCTIONS` |
 	    perl -ne '/^(\S+) is a function$/ and not $seen{$1}++ and print "$1\n"'
     }
     _bodies_of_FUNCTIONS() {
-	type "$@" | perl -ne '/^(\S+) is a function$/ or print'
+	LANG=C type "$@" | perl -ne '/^(\S+) is a function$/ or print'
     }
     _names_of_VARIABLES() {
 	# This may screw up if variables contain \n and =
@@ -131,7 +131,7 @@ env_parallel() {
 	#   aliased to `alias | /usr/bin/which --tty-only --read-alias --show-dot --show-tilde'
 	#   parallel is a tracked alias for /usr/local/bin/parallel (ksh)
 	# Return 0 if found, 1 otherwise
-	type "$@" |
+	LANG=C type "$@" |
 	    perl -pe '$exit += (s/ is an alias for .*// ||
 	                        s/ is aliased to .*// ||
                                 s/ is a function// ||

src/env_parallel.zsh

@@ -105,7 +105,7 @@ env_parallel() {
 	#   which is hashed (/usr/bin/which)
 	#   aliased to `alias | /usr/bin/which --tty-only --read-alias --show-dot --show-tilde'
 	# Return 0 if found, 1 otherwise
-	type "$@" |
+	LANG=C type "$@" |
 	    perl -pe '$exit += (s/ is an alias for .*// ||
 	                        s/ is aliased to .*// ||
                                 s/ is a function// ||

src/parallel

@@ -48,7 +48,6 @@ if($Global::max_number_of_args) {
 }
 
 my @command = @ARGV;
-
 my @input_source_fh;
 if($opt::pipepart) {
     if($opt::tee) {
@@ -226,10 +225,11 @@ sub pipepart_setup {
     #   $Global::JobQueue
     if($opt::tee) {
 	# Prepend each command with
-	#   cat file
-	my $cat_string = "cat ".::shell_quote_scalar($opt::a[0]);
+	#   < file
+	my $cat_string = "< ".::shell_quote_scalar($opt::a[0]);
 	for(1..$Global::JobQueue->total_jobs()) {
-	    push @Global::cat_prepends, $cat_string;
+	    push @Global::cat_appends, $cat_string;
+	    push @Global::cat_prepends, "";
 	}
     } else {
 	if(not $opt::blocksize) {
@@ -286,15 +286,13 @@ sub pipe_tee_setup {
 	open STDOUT, ">","/dev/null";
 	exec "tee",@fifos;
     }
-    # Make cats for each fifo
-    #   cat fifo1 | grep 1
-    #   cat fifo2 | grep 2
-    #   cat fifo3 | grep 3
-    #   cat fifo4 | grep 4
-    #   cat fifo5 | grep 5
+    # For each fifo
+    #   (rm fifo1; grep 1) < fifo1
+    #   (rm fifo2; grep 2) < fifo2
+    #   (rm fifo3; grep 3) < fifo3
     # Remove the tmpfifo as soon as it is open
-    # 'exec' saves a process
-    @Global::cat_prepends = map { "(rm $_; exec cat) < $_" } @fifos;
+    @Global::cat_prepends = map { "(rm $_;" } @fifos;
+    @Global::cat_appends = map { ") < $_" } @fifos;
 }
 
 sub pipe_part_files {
@@ -407,7 +405,10 @@ sub find_split_positions {
 	    }
 	}
     }
+    if($pos[$#pos] != $size) {
+	# Last splitpoint was not at end of the file: add it
 	push(@pos,$size);
+    }
     close $fh;
     return @pos;
 }
@@ -440,7 +441,7 @@ sub cat_partial {
 	     }
 	 });
     return "<". shell_quote_scalar($file) .
-	" perl -e '$script' @start_len";
+	" perl -e '$script' @start_len |";
 }
 
 sub spreadstdin {
@@ -4097,8 +4098,8 @@ sub usage {
 	 "If you use programs that use GNU Parallel to process data for an article in a",
 	 "scientific publication, please cite:",
 	 "",
-	 "  O. Tange (2011): GNU Parallel - The Command-Line Power Tool,",
-	 "  ;login: The USENIX Magazine, February 2011:42-47.",
+	 "  O. Tange (2018): GNU Parallel 2018, Apr 2018, ISBN 9781387509881,",
+	 "  DOI https://doi.org/10.5281/zenodo.1146014",
 	 "",
 	 "This helps funding further development; AND IT WON'T COST YOU A CENT.",
 	 "If you pay 10000 EUR you should feel free to use GNU Parallel without citing.",
@@ -4125,12 +4126,15 @@ sub citation_notice {
 	     "If you use programs that use GNU Parallel to process data for an article in a",
 	     "scientific publication, please cite:",
 	     "",
-	     "  O. Tange (2011): GNU Parallel - The Command-Line Power Tool,",
-	     "  ;login: The USENIX Magazine, February 2011:42-47.",
+	     "  O. Tange (2018): GNU Parallel 2018, Apr 2018, ISBN 9781387509881,",
+	     "  DOI https://doi.org/10.5281/zenodo.1146014",
 	     "",
 	     "This helps funding further development; AND IT WON'T COST YOU A CENT.",
 	     "If you pay 10000 EUR you should feel free to use GNU Parallel without citing.",
 	     "",
+	     "More about funding GNU Parallel and the citation notice:",
+	     "https://www.gnu.org/software/parallel/parallel_design.html#Citation-notice",
+	     "",
 	     "To silence this citation notice: run 'parallel --citation' once.",
 	     ""
 	    );
@@ -4216,8 +4220,9 @@ sub citation {
 	"      author       = {Tange, Ole},",
 	"      title        = {GNU Parallel 2018},",
 	"      publisher    = {Ole Tange},",
+	"      month        = Apr,",
 	"      year         = 2018,",
-	"      month        = apr,",
+	"      ISBN         = {9781387509881},",
 	"      doi          = {10.5281/zenodo.1146014},",
 	"      url          = {https://doi.org/10.5281/zenodo.1146014}",
 	"}",
@@ -4227,6 +4232,9 @@ sub citation {
 	"This helps funding further development; AND IT WON'T COST YOU A CENT.",
 	"If you pay 10000 EUR you should feel free to use GNU Parallel without citing.",
 	"",
+	"More about funding GNU Parallel and the citation notice:",
+	"https://www.gnu.org/software/parallel/parallel_design.html#Citation-notice",
+	"",
 	"If you send a copy of your published article to tange\@gnu.org, it will be",
 	"mentioned in the release notes of next version of GNU Parallel.",
 	""
@@ -7765,11 +7773,12 @@ sub wrapped {
 	    # }'  0 0 0 11 |
 	    #
 	    # --pipepart --tee: prepend:
-	    #   cat dash-a-file |
+	    #   < dash-a-file
 	    #
-	    # --pipe --tee: prepend:
-	    #   cat fifo |
-	    $command = (shift @Global::cat_prepends). " | ($command)";
+	    # --pipe --tee: wrap:
+	    #   (rm fifo; ... ) < fifo
+	    $command = (shift @Global::cat_prepends). "($command)".
+		(shift @Global::cat_appends);
 	} elsif($opt::pipe) {
 	    # Wrap with EOF-detector to avoid starting $command if EOF.
 	    $command = empty_input_wrapper($command);

src/parallel.pod

@@ -2917,7 +2917,35 @@ for Blue Marble world map and create a 10240x20480 map.
   convert -append line{0..19}.jpg world.jpg
 
 
-=head1 EXAMPLE: Copy files as last modified date (ISO8601) with added random digits
+=head1 EXAMPLE: Download Apollo-11 images from NASA using jq
+
+Search NASA using their API to get JSON for images related to 'apollo
+11' and has 'moon landing' in the description.
+
+The search query returns JSON containing URLs to JSON containing
+collections of pictures. One of the pictures in each of these
+collection is I<large>.
+
+B<wget> is used to get the JSON for the search query. B<jq> is then
+used to extract the URLs of the collections. B<parallel> then calls
+B<wget> to get each collection, which is passed to B<jq> to extract
+the URLs of all images. B<grep> filters out the I<large> images, and
+B<parallel> finally uses B<wget> to fetch the images.
+
+  base="https://images-api.nasa.gov/search"
+  q="q=apollo 11"
+  description="description=moon landing"
+  media_type="media_type=image"
+  wget -O - "$base?$q&$description&$media_type" |
+    jq -r .collection.items[].href |
+    parallel wget -O - |
+    jq -r .[] |
+    grep large |
+    parallel wget
+
+
+=head1 EXAMPLE: Copy files as last modified date (ISO8601) with added
+random digits
 
   find . | parallel cp {} '../destdir/{= $a=int(10000*rand); $_=pQ($_);
     $_=`date -r "$_" +%FT%T"$a"`; chomp; =}'
@@ -3205,7 +3233,8 @@ significant. Compare the speed of these:
 If your program cannot take multiple arguments, then you can use GNU
 B<parallel> to spawn multiple GNU B<parallel>s:
 
-  seq -w 0 999999 | parallel -j10 --pipe parallel -j0 touch pict{}.jpg
+  seq -w 0 9999999 |
+    parallel -j10 -q -I,, --pipe parallel -j0 touch pict{}.jpg
 
 If B<-j0> normally spawns 252 jobs, then the above will try to spawn
 2520 jobs. On a normal GNU/Linux system you can spawn 32000 jobs using

src/parallel_alternatives.pod

@@ -1294,7 +1294,8 @@ While you I<can> use multi-lined commands in GNU B<parallel>, to
 improve readibilty GNU B<parallel> discourages the use of multi-line
 commands. In most cases it can be written as a function:
 
-  seq 1 3 | parallel --timeout 2 --joblog my.log 'sleep {}; echo {}; \
+  seq 1 3 |
+    parallel --timeout 2 --joblog my.log 'sleep {}; echo {}; \
       echo finish {}'
 
 Could be written as:

src/parallel_tutorial.pod

@@ -5,11 +5,20 @@
 This tutorial shows off much of GNU B<parallel>'s functionality. The
 tutorial is meant to learn the options in and syntax of GNU
 B<parallel>.  The tutorial is B<not> to show realistic examples from the
-real world. For realistic examples see B<man parallel> in the EXAMPLE
-section.
+real world.
+
+=head2 Reader's guide
+
+Start by watching the intro videos for a quick introduction:
+http://www.youtube.com/playlist?list=PL284C9FF2488BC6D1
+
+Then look at the B<EXAMPLE>s after the list of B<OPTIONS> in B<man
+parallel> (Use B<LESS=+/EXAMPLE\: man parallel>). That will give you
+an idea of what GNU B<parallel> is capable of.
+
+Then spend a couple of hours walking through this tutorial (B<man
+parallel_tutorial>). Your command line will love you for it.
 
-Spend an hour walking through the tutorial. Your command line will
-love you for it.
 
 =head1 Prerequisites
 

testsuite/wanted-results/parallel-local-ssh7

@@ -315,8 +315,8 @@ par_tcsh_man	Academic tradition requires you to cite works you base your article
 par_tcsh_man	If you use programs that use GNU Parallel to process data for an article in a
 par_tcsh_man	scientific publication, please cite:
 par_tcsh_man	
-par_tcsh_man	  O. Tange (2011): GNU Parallel - The Command-Line Power Tool,
-par_tcsh_man	  ;login: The USENIX Magazine, February 2011:42-47.
+par_tcsh_man	  O. Tange (2018): GNU Parallel 2018, Apr 2018, ISBN 9781387509881,
+par_tcsh_man	  DOI https://doi.org/10.5281/zenodo.1146014
 par_tcsh_man	
 par_tcsh_man	This helps funding further development; AND IT WON'T COST YOU A CENT.
 par_tcsh_man	If you pay 10000 EUR you should feel free to use GNU Parallel without citing.
@@ -1019,8 +1019,8 @@ par_csh_man
 par_csh_man	
 par_csh_man	
 par_csh_man	                {+/}/{/..}.{+..} = {...}.{+...} = {+/}/{/...}.{+...}
-par_csh_man	  ;login: The USENIX Magazine, February 2011:42-47.
-par_csh_man	  O. Tange (2011): GNU Parallel - The Command-Line Power Tool,
+par_csh_man	  DOI https://doi.org/10.5281/zenodo.1146014
+par_csh_man	  O. Tange (2018): GNU Parallel 2018, Apr 2018, ISBN 9781387509881,
 par_csh_man	### From man env_parallel
 par_csh_man	--colsep regexp Split input on regexp for positional replacements
 par_csh_man	--nonall        Run the given command with no arguments on all sshlogins

testsuite/wanted-results/parallel-local12

@@ -4,12 +4,15 @@ Academic tradition requires you to cite works you base your article on.
 If you use programs that use GNU Parallel to process data for an article in a
 scientific publication, please cite:
 
-  O. Tange (2011): GNU Parallel - The Command-Line Power Tool,
-  ;login: The USENIX Magazine, February 2011:42-47.
+  O. Tange (2018): GNU Parallel 2018, Apr 2018, ISBN 9781387509881,
+  DOI https://doi.org/10.5281/zenodo.1146014
 
 This helps funding further development; AND IT WON'T COST YOU A CENT.
 If you pay 10000 EUR you should feel free to use GNU Parallel without citing.
 
+More about funding GNU Parallel and the citation notice:
+https://www.gnu.org/software/parallel/parallel_design.html#Citation-notice
+
 To silence this citation notice: run 'parallel --citation' once.
 
 echo a
@@ -24,12 +27,15 @@ Academic tradition requires you to cite works you base your article on.
 If you use programs that use GNU Parallel to process data for an article in a
 scientific publication, please cite:
 
-  O. Tange (2011): GNU Parallel - The Command-Line Power Tool,
-  ;login: The USENIX Magazine, February 2011:42-47.
+  O. Tange (2018): GNU Parallel 2018, Apr 2018, ISBN 9781387509881,
+  DOI https://doi.org/10.5281/zenodo.1146014
 
 This helps funding further development; AND IT WON'T COST YOU A CENT.
 If you pay 10000 EUR you should feel free to use GNU Parallel without citing.
 
+More about funding GNU Parallel and the citation notice:
+https://www.gnu.org/software/parallel/parallel_design.html#Citation-notice
+
 To silence this citation notice: run 'parallel --citation' once.
 
 10 files to edit
@@ -41,8 +47,9 @@ scientific publication, please cite:
       author       = {Tange, Ole},
       title        = {GNU Parallel 2018},
       publisher    = {Ole Tange},
+      month        = Apr,
       year         = 2018,
-      month        = apr,
+      ISBN         = {9781387509881},
       doi          = {10.5281/zenodo.1146014},
       url          = {https://doi.org/10.5281/zenodo.1146014}
 }
@@ -52,6 +59,9 @@ scientific publication, please cite:
 This helps funding further development; AND IT WON'T COST YOU A CENT.
 If you pay 10000 EUR you should feel free to use GNU Parallel without citing.
 
+More about funding GNU Parallel and the citation notice:
+https://www.gnu.org/software/parallel/parallel_design.html#Citation-notice
+
 If you send a copy of your published article to tange@gnu.org, it will be
 mentioned in the release notes of next version of GNU Parallel.
 

testsuite/wanted-results/parallel-local9

@@ -171,8 +171,8 @@ Academic tradition requires you to cite works you base your article on.
 If you use programs that use GNU Parallel to process data for an article in a
 scientific publication, please cite:
 
-  O. Tange (2011): GNU Parallel - The Command-Line Power Tool,
-  ;login: The USENIX Magazine, February 2011:42-47.
+  O. Tange (2018): GNU Parallel 2018, Apr 2018, ISBN 9781387509881,
+  DOI https://doi.org/10.5281/zenodo.1146014
 
 This helps funding further development; AND IT WON'T COST YOU A CENT.
 If you pay 10000 EUR you should feel free to use GNU Parallel without citing.