Added env_parallel.csh env_parallel.pdksh.

env_parallel.fish now supports arrays.
Documentation updates.

README

@@ -75,7 +75,7 @@ system is old or Microsoft Windows):
 
 After this you should be able to do:
 
-    parallel -j0 ping -nc 3 ::: foss.org.my gnu.org freenetproject.org
+    parallel -j0 ping -nc 3 ::: qubes-os.org gnu.org freenetproject.org
 
 This will send 3 ping packets to 3 different hosts in parallel and print
 the output when they complete.

doc/release_new_version

@@ -248,6 +248,12 @@ for Big Data Applications https://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumb
 
 * <<Citation needed: Introspecting for RSA Key Material to Assist Intrusion Detection http://ieeexplore.ieee.org/xpl/login.jsp?tp=&arnumber=7331177&url=http%3A%2F%2Fieeexplore.ieee.org%2Fxpls%2Fabs_all.jsp%3Farnumber%3D7331177>>
 
+* GNU Parallel was cited in: Tools and techniques for computational reproducibility http://biorxiv.org/content/biorxiv/early/2016/03/17/022707.full.pdf
+
+* qbatch uses GNU Parallel: https://pypi.python.org/pypi/qbatch/1.0rc2
+
+* FaceCrop uses GNU Parallel: https://github.com/EderSantana/FaceCrop
+
 * Bug fixes and man page updates.
 
 GNU Parallel - For people who live life in the parallel lane.

src/Makefile.am

@@ -1,15 +1,20 @@
 bin_SCRIPTS = parallel sql niceload \
-	env_parallel env_parallel.bash env_parallel.zsh env_parallel.fish env_parallel.ksh
+	env_parallel env_parallel.bash env_parallel.zsh env_parallel.fish \
+	env_parallel.ksh env_parallel.pdksh env_parallel.csh
 
 install-exec-hook:
 	rm $(DESTDIR)$(bindir)/sem || true
 	$(LN_S) parallel $(DESTDIR)$(bindir)/sem
 
 if DOCUMENTATION
-man_MANS = parallel.1 env_parallel.1 sem.1 sql.1 niceload.1 parallel_tutorial.7 parallel_design.7
+man_MANS = parallel.1 env_parallel.1 sem.1 sql.1 niceload.1 \
-doc_DATA = parallel.html env_parallel.html sem.html sql.html niceload.html parallel_tutorial.html parallel_design.html \
+	parallel_tutorial.7 parallel_design.7
-	parallel.texi env_parallel.texi sem.texi sql.texi niceload.texi parallel_tutorial.texi parallel_design.texi \
+doc_DATA = parallel.html env_parallel.html sem.html sql.html niceload.html \
-	parallel.pdf env_parallel.pdf sem.pdf sql.pdf niceload.pdf parallel_tutorial.pdf parallel_design.pdf
+	parallel_tutorial.html parallel_design.html \
+	parallel.texi env_parallel.texi sem.texi sql.texi niceload.texi \
+	parallel_tutorial.texi parallel_design.texi \
+	parallel.pdf env_parallel.pdf sem.pdf sql.pdf niceload.pdf \
+	parallel_tutorial.pdf parallel_design.pdf
 endif
 
 # Build documentation file if the tool to build exists.
@@ -174,5 +179,7 @@ DISTCLEANFILES = parallel.1 env_parallel.1 sem.1 sql.1 niceload.1 \
 
 EXTRA_DIST = parallel sem sql niceload env_parallel \
 	env_parallel.bash env_parallel.zsh env_parallel.fish env_parallel.ksh \
-	sem.pod parallel.pod env_parallel.pod niceload.pod parallel_tutorial.pod parallel_design.pod \
+	env_parallel.pdksh env_parallel.csh \
+	sem.pod parallel.pod env_parallel.pod niceload.pod parallel_tutorial.pod \
+	parallel_design.pod \
 	$(DISTCLEANFILES)

src/Makefile.in

@@ -218,12 +218,18 @@ top_build_prefix = @top_build_prefix@
 top_builddir = @top_builddir@
 top_srcdir = @top_srcdir@
 bin_SCRIPTS = parallel sql niceload \
-	env_parallel env_parallel.bash env_parallel.zsh env_parallel.fish env_parallel.ksh
+	env_parallel env_parallel.bash env_parallel.zsh env_parallel.fish \
+	env_parallel.ksh env_parallel.pdksh env_parallel.csh
 
-@DOCUMENTATION_TRUE@man_MANS = parallel.1 env_parallel.1 sem.1 sql.1 niceload.1 parallel_tutorial.7 parallel_design.7
+@DOCUMENTATION_TRUE@man_MANS = parallel.1 env_parallel.1 sem.1 sql.1 niceload.1 \
-@DOCUMENTATION_TRUE@doc_DATA = parallel.html env_parallel.html sem.html sql.html niceload.html parallel_tutorial.html parallel_design.html \
+@DOCUMENTATION_TRUE@	parallel_tutorial.7 parallel_design.7
-@DOCUMENTATION_TRUE@	parallel.texi env_parallel.texi sem.texi sql.texi niceload.texi parallel_tutorial.texi parallel_design.texi \
+
-@DOCUMENTATION_TRUE@	parallel.pdf env_parallel.pdf sem.pdf sql.pdf niceload.pdf parallel_tutorial.pdf parallel_design.pdf
+@DOCUMENTATION_TRUE@doc_DATA = parallel.html env_parallel.html sem.html sql.html niceload.html \
+@DOCUMENTATION_TRUE@	parallel_tutorial.html parallel_design.html \
+@DOCUMENTATION_TRUE@	parallel.texi env_parallel.texi sem.texi sql.texi niceload.texi \
+@DOCUMENTATION_TRUE@	parallel_tutorial.texi parallel_design.texi \
+@DOCUMENTATION_TRUE@	parallel.pdf env_parallel.pdf sem.pdf sql.pdf niceload.pdf \
+@DOCUMENTATION_TRUE@	parallel_tutorial.pdf parallel_design.pdf
 
 DISTCLEANFILES = parallel.1 env_parallel.1 sem.1 sql.1 niceload.1 \
 	parallel_tutorial.7 parallel_design.7 \
@@ -236,7 +242,9 @@ DISTCLEANFILES = parallel.1 env_parallel.1 sem.1 sql.1 niceload.1 \
 
 EXTRA_DIST = parallel sem sql niceload env_parallel \
 	env_parallel.bash env_parallel.zsh env_parallel.fish env_parallel.ksh \
-	sem.pod parallel.pod env_parallel.pod niceload.pod parallel_tutorial.pod parallel_design.pod \
+	env_parallel.pdksh env_parallel.csh \
+	sem.pod parallel.pod env_parallel.pod niceload.pod parallel_tutorial.pod \
+	parallel_design.pod \
 	$(DISTCLEANFILES)
 
 all: all-am

src/env_parallel

@@ -27,14 +27,14 @@ bash:  Put this in $HOME/.bashrc:  . `which env_parallel.bash`
        Supports: aliases, functions, variables, arrays
 
 zsh:   Put this in $HOME/.zshrc:  . `which env_parallel.zsh`
-       E.g. by doing:  echo '. `which env_parallel.zsh`' >> $HOME/.zshrc
+       E.g. by doing:  echo '. `which env_parallel.zsh`' >> $HOME/.zshenv
        Supports: functions, variables, arrays
 
 fish:  Put this in $HOME/.config/fish/config.fish:
          source (which env_parallel.fish)
        E.g. by doing:
          echo 'source (which env_parallel.fish)' >> $HOME/.config/fish/config.fish
-       Supports: aliases, functions, variables
+       Supports: aliases, functions, variables, arrays
 
 ksh:   Put this in $HOME/.kshrc:  source `which env_parallel.ksh`
        E.g. by doing:  echo 'source `which env_parallel.ksh`' >> $HOME/.kshrc
@@ -44,6 +44,10 @@ pdksh: Put this in $HOME/.profile:  source `which env_parallel.pdksh`
        E.g. by doing:  echo 'source `which env_parallel.pdksh`' >> $HOME/.profile
        Supports: aliases, functions, variables, arrays
 
+csh:   Put this in $HOME/.cshrc:  source `which env_parallel.csh`
+       E.g. by doing:  echo 'source `which env_parallel.csh`' >> $HOME/.cshrc
+       Supports: aliases
+
 For details: see  man env_parallel
 
 _EOS

src/env_parallel.bash

@@ -1,3 +1,5 @@
+#!/bin/bash
+
 # This file must be sourced in bash:
 #
 #   source `which env_parallel.bash`
@@ -5,7 +7,7 @@
 # after which 'env_parallel' works
 #
 #
-# Copyright (C) 2007,2008,2009,2010,2011,2012,2013,2014,2015,2016
+# Copyright (C) 2016
 # Ole Tange and Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or modify

src/env_parallel.fish

@@ -1,3 +1,5 @@
+#!/usr/bin/fish
+
 # This file must be sourced in fish:
 #
 #   source (which env_parallel.fish)
@@ -5,7 +7,7 @@
 # after which 'env_parallel' works
 #
 #
-# Copyright (C) 2007,2008,2009,2010,2011,2012,2013,2014,2015,2016
+# Copyright (C) 2016
 # Ole Tange and Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or modify
@@ -23,8 +25,70 @@
 # or write to the Free Software Foundation, Inc., 51 Franklin St,
 # Fifth Floor, Boston, MA 02110-1301 USA
 
+# If you are a fisherman feel free to improve the code
+#
+# The code needs to deal with variables like:
+#   set funky (perl -e 'print pack "c*", 2..254')
+#
+# Problem:
+#   Tell the difference between:
+#     set tmp "a'  'b'  'c"
+#     set tmparr1 "a'  'b"  'c'
+#     set tmparr2 'a'  "b'  'c"
+#   The output from `set` is exactly the same.
+# Solution:
+#   for-loop for each variable. Each value is separated with a
+#   separator.
+
 function env_parallel
-  setenv PARALLEL_ENV (begin; functions -n | perl -pe 's/,/\n/g' | while read d; functions $d; end; perl -e 'print map { "$_///$ENV{$_}\n"} grep !/^(PWD|SHLVL|PATH)$/, keys %ENV'| sh -c 'parallel --shellquote' | perl -pe 's:^([^/]+)///:setenv $1 :'; end |perl -pe 's/\001/\\cb/g;s/\n/\001/')
+  setenv PARALLEL_ENV (
+    begin;
+      # Export function definitions
+      functions -n | perl -pe 's/,/\n/g' | while read d; functions $d; end;
+      # Convert scalar vars to fish \XX quoting
+      eval (set -L | perl -ne 'chomp;
+        ($name,$val)=split(/ /,$_,2);
+        $name=~/^(COLUMNS|FISH_VERSION|LINES|PWD|SHLVL|_|history|status|version)$/ and next;
+        if($val=~/^'"'"'/) { next; }
+        print "set $name \"\$$name\";\n";
+      ')
+      # Generate commands to set scalar variables
+      begin;
+        for v in (set -n);
+          # Separate variables with the string: \000
+          eval "for i in \$$v;
+            echo -n $v \$i;
+    	    perl -e print\\\"\\\\0\\\";
+          end;"
+        end;
+        # A final line to flush the last variable in Perl
+        perl -e print\"\\0\";
+      end | perl -0 -ne '
+          # Remove separator string
+          chop;
+          ($name,$val)=split(/ /,$_,2);
+          # Ignore read-only vars
+          $name=~/^(COLUMNS|FISH_VERSION|LINES|PWD|SHLVL|_|history|status|version)$/ and next;
+          # Quote $val
+          $val=~s/[\002-\011\013-\032\\\#\?\`\(\)\{\}\[\]\^\*\<\=\>\~\|\; \"\!\$\&\202-\377]/\\\$&/go;
+          # Quote single quote
+          $val=~s/'"'"'/\\\$&/go;
+          # Quote newline as '\n'
+          $val =~ s/[\n]/\\\n/go;
+          if($name ne $last and $last) {
+            # The $name is different, so this is a new variable.
+            # Print the last one.
+            # Separate list elements by 2 spaces
+            $"="  ";
+            print "set $last @qval;\n";
+            @qval=();
+          }
+          push @qval,$val;
+          $last=$name;
+        ';
+    end |perl -pe 's/\001/\\cb/g and print STDERR "env_parallel: Warning: ASCII value 1 in variables is not supported\n";
+                   s/\n/\001/'
+    )
   parallel $argv;
   set -e PARALLEL_ENV
 end

src/env_parallel.ksh

@@ -1,3 +1,5 @@
+#!/usr/bin/ksh
+
 # This file must be sourced in ksh:
 #
 #   source `which env_parallel.ksh`
@@ -5,7 +7,7 @@
 # after which 'env_parallel' works
 #
 #
-# Copyright (C) 2007,2008,2009,2010,2011,2012,2013,2014,2015,2016
+# Copyright (C) 2016
 # Ole Tange and Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or modify

src/env_parallel.pdksh

@@ -1,3 +1,5 @@
+#!/bin/pdksh
+
 # This file must be sourced in ksh:
 #
 #   source `which env_parallel.ksh`
@@ -5,7 +7,7 @@
 # after which 'env_parallel' works
 #
 #
-# Copyright (C) 2007,2008,2009,2010,2011,2012,2013,2014,2015,2016
+# Copyright (C) 2016
 # Ole Tange and Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or modify

src/env_parallel.pod

@@ -20,6 +20,11 @@ environment to GNU Parallel.
 If the shell function is not loaded, a dummy script will be run
 instead that explains how to install the function.
 
+B<env_parallel> only works if the size of the current environment is
+smaller than half of the max size and smaller than 25% of the max if
+running remotely. E.g. The max size of Bash's environment is 256 KB,
+so B<env_parallel> will fail if B<set | wc -c> is bigger than 128 KB.
+
 
 =head1 OPTIONS
 
@@ -30,63 +35,102 @@ Same as GNU B<parallel>.
 
 =head2 Bash
 
+Installation
+
+Put this in $HOME/.bashrc:
+
+  . `which env_parallel.bash`
+
+E.g. by doing:
+
+  echo '. `which env_parallel.bash`' >> $HOME/.bashrc
+
 =over 8
 
 =item aliases
 
   alias myecho=echo
   env_parallel myecho ::: test
+  env_parallel -S server myecho ::: test
 
 =item functions
 
   myfunc() { echo $*; }
   env_parallel myfunc ::: test
+  env_parallel -S server myfunc ::: test
 
 =item variables
 
   myvar=test
   env_parallel echo '$myvar' ::: test
+  env_parallel -S server echo '$myvar' ::: test
 
 =item arrays
 
   myarray=(foo bar baz)
   env_parallel echo '${myarray[{}]}' ::: 0 1 2
+  env_parallel -S server echo '${myarray[{}]}' ::: 0 1 2
 
 =back
 
 =head2 Zsh
 
+Installation
+
+Put this in $HOME/.zshrc:
+
+  . `which env_parallel.zsh`
+
+E.g. by doing:
+
+  echo '. `which env_parallel.zsh`' >> $HOME/.zshenv
+
 =over 8
 
 =item aliases
 
-Not supported
+Not supported - Zsh does not support aliases defined in the same
+parsing as they are used.
 
 =item functions
 
   myfunc() { echo $*; }
   env_parallel myfunc ::: test
+  env_parallel -S server myfunc ::: test
 
 =item variables
 
   myvar=test
   env_parallel echo '$myvar' ::: test
+  env_parallel -S server echo '$myvar' ::: test
 
 =item arrays
 
   myarray=(foo bar baz)
   env_parallel echo '${myarray[{}]}' ::: 1 2 3
+  env_parallel -S server echo '${myarray[{}]}' ::: 1 2 3
 
 =back
 
 =head2 fish
 
+Installation
+
+Put this in $HOME/.config/fish/config.fish:
+
+  source (which env_parallel.fish)
+
+E.g. by doing:
+
+  echo 'source (which env_parallel.fish)' >> $HOME/.config/fish/config.fish
+
 =over 8
 
 =item aliases
 
   alias myecho=echo
   env_parallel myecho ::: test
+  env_parallel -S server myecho ::: test
 
 =item functions
 
@@ -94,67 +138,133 @@ Not supported
     echo $argv
   end
   env_parallel myfunc ::: test
+  env_parallel -S server myfunc ::: test
 
 =item variables
 
   set myvar test
   env_parallel echo '$myvar' ::: test
+  env_parallel -S server echo '$myvar' ::: test
 
 =item arrays
 
-Not supported
+  set myarray foo bar baz
+  env_parallel echo '$myarray[{}]' ::: 1 2 3
+  env_parallel -S server echo '$myarray[{}]' ::: 1 2 3
 
 =back
 
 =head2 ksh
 
+Installation
+
+Put this in $HOME/.kshrc:
+
+  source `which env_parallel.ksh`
+
+E.g. by doing:
+
+  echo 'source `which env_parallel.ksh`' >> $HOME/.kshrc
+
 =over 8
 
 =item aliases
 
   alias myecho=echo
   env_parallel myecho ::: test
+  env_parallel -S server myecho ::: test
 
 =item functions
 
   myfunc() { echo $*; }
   env_parallel myfunc ::: test
+  env_parallel -S server myfunc ::: test
 
 =item variables
 
   myvar=test
   env_parallel echo '$myvar' ::: test
+  env_parallel -S server echo '$myvar' ::: test
 
 =item arrays
 
   myarray=(foo bar baz)
   env_parallel echo '${myarray[{}]}' ::: 0 1 2
+  env_parallel -S server echo '${myarray[{}]}' ::: 0 1 2
 
 =back
 
 =head2 pdksh
 
+Installation
+
+Put this in $HOME/.profile:
+
+  source `which env_parallel.pdksh`
+
+E.g. by doing:
+
+  echo 'source `which env_parallel.pdksh`' >> $HOME/.profile
+
 =over 8
 
 =item aliases
 
   alias myecho=echo
   env_parallel myecho ::: test
+  env_parallel -S server myecho ::: test
 
 =item functions
 
   myfunc() { echo $*; }
   env_parallel myfunc ::: test
+  env_parallel -S server myfunc ::: test
 
 =item variables
 
   myvar=test
   env_parallel echo '$myvar' ::: test
+  env_parallel -S server echo '$myvar' ::: test
 
 =item arrays
 
   myarray=(foo bar baz)
   env_parallel echo '${myarray[{}]}' ::: 0 1 2
+  env_parallel -S server echo '${myarray[{}]}' ::: 0 1 2
+
+=back
+
+=head2 csh
+
+Installation
+
+Put this in $HOME/.cshrc:
+
+  source `which env_parallel.csh`
+
+E.g. by doing:
+
+  echo 'source `which env_parallel.csh`' >> $HOME/.cshrc
+
+=over 8
+
+=item aliases
+
+  alias myecho echo
+  env_parallel myecho ::: test
+  env_parallel -S server myecho ::: test
+
+=item functions
+
+Not supported by B<csh>.
+
+=item variables
+
+Not supported
+
+=item arrays
+
+Not supported
 
 =back
 
@@ -187,9 +297,9 @@ the manual of B<xargs> from GNU findutils 4.4.2.
 
 =head1 LICENSE
 
-Copyright (C) 2007,2008,2009,2010,2011,2012,2013,2014,2015 Free
+Copyright (C) 2016
-Software Foundation, Inc.
+Ole Tange and Free Software Foundation, Inc.
-v
+
 This program is free software; you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
 the Free Software Foundation; either version 3 of the License, or

src/env_parallel.zsh

@@ -1,3 +1,5 @@
+#!/usr/bin/zsh
+
 # This file must be sourced in zsh:
 #
 #   source =env_parallel.zsh
@@ -5,7 +7,7 @@
 # after which 'env_parallel' works
 #
 #
-# Copyright (C) 2007,2008,2009,2010,2011,2012,2013,2014,2015,2016
+# Copyright (C) 2016
 # Ole Tange and Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or modify
@@ -24,7 +26,7 @@
 # Fifth Floor, Boston, MA 02110-1301 USA
 
 env_parallel() {
-  export PARALLEL_ENV="$(alias | perl -pe 's/^/alias /';typeset -p |
+  export PARALLEL_ENV="$(alias | perl -pe 's/^/alias /'; typeset -p |
     grep -aFvf <(typeset -pr) |
     egrep -iav 'ZSH_EVAL_CONTEXT|LINENO=| _=|aliases|^typeset [a-z_]+$'|
     egrep -av '^(typeset -A (commands|functions|options)|typeset IFS=|..$)|cyan';

src/parallel

@@ -7491,7 +7491,6 @@ sub start {
     $ENV{'PARALLEL_TMP'} = ::tmpname("par");
     ::debug("run", $Global::total_running, " processes . Starting (",
 	    $job->seq(), "): $command\n");
-
     if($opt::pipe) {
 	my ($stdin_fh) = ::gensym();
 	$pid = open3_setpgrp($stdin_fh,$stdout_fh,$stderr_fh,$command);
@@ -9166,6 +9165,8 @@ sub get {
     my $ret = $self->{'arg_sub_queue'}->get();
     if($ret and
        grep { index($_->orig(),"\0") > 0 } map { @$_ } $ret) {
+	# Allow for \0 in position 0 because GNU Parallel uses "\0"
+	# to mean no-string
 	::warning("a NUL character occurred in the input.",
 		  "It cannot be passed through in the argument list.",
 		  "Did you mean to use the --null option?");

src/parallel.pod

@@ -11,8 +11,8 @@ parallel - build and execute shell command lines from standard input in parallel
 
 B<parallel> [options] [I<command> [arguments]] < list_of_arguments
 
-B<parallel> [options] [I<command> [arguments]] ( B<:::> arguments |
+B<parallel> [options] [I<command> [arguments]] ( B<:::> arguments | B<:::+> arguments |
-B<::::> argfile(s) ) ...
+B<::::> argfile(s) | B<::::+> argfile(s) ) ...
 
 B<parallel> --semaphore [options] I<command>
 
@@ -326,6 +326,18 @@ B<:::> and B<::::> can be mixed. So these are equivalent:
     ::: 1 2 3
 
 
+=item B<:::+> I<arguments> (alpha testing)
+
+Like B<:::> but linked like B<--xapply> to the previous input source.
+
+Contrary to B<--xapply> values do not wrap: The shortest input source
+determines the length.
+
+Example:
+
+  parallel echo ::: a b c :::+ 1 2 3 ::: X Y :::+ 11 22
+
+
 =item B<::::> I<argfiles>
 
 Another way to write B<-a> I<argfile1> B<-a> I<argfile2> ...
@@ -335,6 +347,14 @@ B<:::> and B<::::> can be mixed.
 See B<-a>, B<:::> and B<--xapply>.
 
 
+=item B<::::+> I<argfiles> (alpha testing)
+
+Like B<::::+> but linked like B<--xapply> to the previous input source.
+
+Contrary to B<--xapply> values do not wrap: The shortest input source
+determines the length.
+
+
 =item B<--null>
 
 =item B<-0>
@@ -1665,7 +1685,7 @@ Like this:
 
   #!/usr/bin/parallel --shebang -r traceroute
 
-  foss.org.my
+  qubes-os.org
   debian.org
   freenetproject.org
 
@@ -1675,7 +1695,7 @@ On FreeBSD B<env> is needed:
 
   #!/usr/bin/env -S parallel --shebang -r traceroute
 
-  foss.org.my
+  qubes-os.org
   debian.org
   freenetproject.org
 
@@ -2770,9 +2790,9 @@ printed as soon as possible you can use B<-u>.
 
 Compare the output of:
 
-  parallel traceroute ::: foss.org.my debian.org freenetproject.org
+  parallel traceroute ::: qubes-os.org debian.org freenetproject.org
-  parallel --line-buffer traceroute ::: foss.org.my debian.org freenetproject.org
+  parallel --line-buffer traceroute ::: qubes-os.org debian.org freenetproject.org
-  parallel -u traceroute ::: foss.org.my debian.org freenetproject.org
+  parallel -u traceroute ::: qubes-os.org debian.org freenetproject.org
 
 
 =head1 EXAMPLE: Tag output lines
@@ -2781,12 +2801,12 @@ GNU B<parallel> groups the output lines, but it can be hard to see
 where the different jobs begin. B<--tag> prepends the argument to make
 that more visible:
 
-  parallel --tag traceroute ::: foss.org.my debian.org freenetproject.org
+  parallel --tag traceroute ::: qubes-os.org debian.org freenetproject.org
 
 B<--tag> works with B<--line-buffer> but not with B<-u>:
 
   parallel --tag --line-buffer traceroute \
-    ::: foss.org.my debian.org freenetproject.org
+    ::: qubes-os.org debian.org freenetproject.org
 
 Check the uptime of the servers in I<~/.parallel/sshloginfile>:
 
@@ -2810,17 +2830,17 @@ If you remove B<-k> some of the lines may come out in the wrong order.
 
 Another example is B<traceroute>:
 
-  parallel traceroute ::: foss.org.my debian.org freenetproject.org
+  parallel traceroute ::: qubes-os.org debian.org freenetproject.org
 
-will give traceroute of foss.org.my, debian.org and
+will give traceroute of qubes-os.org, debian.org and
 freenetproject.org, but it will be sorted according to which job
 completed first.
 
 To keep the order the same as input run:
 
-  parallel -k traceroute ::: foss.org.my debian.org freenetproject.org
+  parallel -k traceroute ::: qubes-os.org debian.org freenetproject.org
 
-This will make sure the traceroute to foss.org.my will be printed
+This will make sure the traceroute to qubes-os.org will be printed
 first.
 
 A bit more complex example is downloading a huge file in chunks in
@@ -4730,7 +4750,7 @@ Copyright (C) 2007-10-18 Ole Tange, http://ole.tange.dk
 
 Copyright (C) 2008,2009,2010 Ole Tange, http://ole.tange.dk
 
-Copyright (C) 2010,2011,2012,2013,2014,2015 Ole Tange,
+Copyright (C) 2010,2011,2012,2013,2014,2015,2016 Ole Tange,
 http://ole.tange.dk and Free Software Foundation, Inc.
 
 Parts of the manual concerning B<xargs> compatibility is inspired by
@@ -4739,7 +4759,7 @@ the manual of B<xargs> from GNU findutils 4.4.2.
 
 =head1 LICENSE
 
-Copyright (C) 2007,2008,2009,2010,2011,2012,2013,2014,2015 Free
+Copyright (C) 2007,2008,2009,2010,2011,2012,2013,2014,2015,2016 Free
 Software Foundation, Inc.
 
 This program is free software; you can redistribute it and/or modify

src/parallel_design.pod

@@ -6,8 +6,8 @@
 
 This document describes design decisions made in the development of
 GNU B<parallel> and the reasoning behind them. It will give an
-overview of why some of the code looks like it does, and help new
+overview of why some of the code looks the way it does, and will help
-maintainers understand the code better.
+new maintainers understand the code better.
 
 
 =head2 One file program
@@ -30,10 +30,10 @@ is CentOS 3.9 and Perl 5.8.0.
 =head2 Exponentially back off
 
 GNU B<parallel> busy waits. This is because the reason why a job is
-not started may be due to load average, and thus it will not make
+not started may be due to load average (when using B<--load>), and
-sense to wait for a job to finish. Instead the load average must be
+thus it will not make sense to wait for a job to finish. Instead the
-checked again. Load average is not the only reason: B<--timeout> has a
+load average must be checked again. Load average is not the only
-similar problem.
+reason: B<--timeout> has a similar problem.
 
 To not burn up too much CPU GNU B<parallel> sleeps exponentially
 longer and longer if nothing happens, maxing out at 1 second.
@@ -169,6 +169,20 @@ to complete; remove {} and exit with the exit code from I<input>.
 
 It is made this way to be compatible with B<*csh>.
 
+=item --pipepart
+
+  < file perl -e 'while(@ARGV) {
+      sysseek(STDIN,shift,0) || die;
+      $left = shift;
+      while($read = sysread(STDIN,$buf, ($left > 131072 ? 131072 : $left))){
+        $left -= $read;
+        syswrite(STDOUT,$buf);
+      }
+    }' startposition length
+
+This will read I<length> bytes from I<file> starting at
+I<startposition> and send it to STDOUT.
+
 =item --sshlogin I<sln>
 
 ssh I<sln> I<shell quoted input>
@@ -446,14 +460,13 @@ The wrapper looks like this:
 
 Transferring of variables and functions given by B<--env> is done by
 running a Perl script remotely that calls the actual command. The Perl
-script sets B<$ENV{>I<variable>B<}> to the correct value before exec'ing the a
+script sets B<$ENV{>I<variable>B<}> to the correct value before
-shell that runs the function definition followed by the actual
+exec'ing the a shell that runs the function definition followed by the
-command.
+actual command.
 
-B<env_parallel> (mentioned in the man page) copies the full current
+The function B<env_parallel> copies the full current environment into
-environment into the environment variable B<PARALLEL_ENV>. This
+the environment variable B<PARALLEL_ENV>. This variable is picked up
-variable is picked up by GNU B<parallel> and used to create the Perl
+by GNU B<parallel> and used to create the Perl script mentioned above.
-script mentioned above.
 
 
 =head2 Base64 encoded bzip2
@@ -747,7 +760,8 @@ What can be done about it?
 
 =back
 
-Unfortunately it is not always possible to predict the root cause of the error.
+Unfortunately it is not always possible to predict the root cause of
+the error.
 
 
 =head2 Computation of load
@@ -766,8 +780,8 @@ B<ps> and put the result in a file, which is then used next time.
 GNU B<parallel> kills jobs. It can be due to B<--memfree>, B<--halt>,
 or when GNU B<parallel> meets a condition from which it cannot
 recover. Every job is started as its own process group. This way any
-(grand)*children will get killed, too. The process group is killed with
+(grand)*children will get killed, too. The process group is killed
-the specification mentioned in B<--termseq>.
+with the specification mentioned in B<--termseq>.
 
 
 =head2 SQL interface
@@ -789,6 +803,10 @@ respectively.
 The Signal column has been renamed to _Signal due to Signal being a
 reserved word in MySQL.
 
+=head2 Logo
+
+The logo is inspired by the Cafe Wall illusion. The font is DejaVu
+Sans.
 
 
 =head1 Ideas for new design
@@ -816,16 +834,13 @@ Will that require 2x record size memory?
 
 Will that require 2x block size memory?
 
-=head2 Logo
-
-The logo is inpired by the Cafe Wall illusion. The font is DejaVu Sans.
-
 
 =head1 Historical decisions
 
 =head2 --tollef
 
-You can read about the history of GNU B<parallel> on https://www.gnu.org/software/parallel/history.html
+You can read about the history of GNU B<parallel> on
+https://www.gnu.org/software/parallel/history.html
 
 B<--tollef> was included to make GNU B<parallel> switch compatible
 with the parallel from moreutils (which is made by Tollef Fog
@@ -833,11 +848,12 @@ Heen). This was done so that users of that parallel easily could port
 their use to GNU B<parallel>: Simply set B<PARALLEL="--tollef"> and
 that would be it.
 
-But several distributions chose to make B<--tollef> global (by putting it
+But several distributions chose to make B<--tollef> global (by putting
-into /etc/parallel/config), and that caused much confusion when people
+it into /etc/parallel/config) without making the users aware of this,
-tried out the examples from GNU B<parallel>'s man page and these did
+and that caused much confusion when people tried out the examples from
-not work.  The users became frustrated because the distribution did
+GNU B<parallel>'s man page and these did not work.  The users became
-not make it clear to them that it has made B<--tollef> global.
+frustrated because the distribution did not make it clear to them that
+it has made B<--tollef> global.
 
 So to lessen the frustration and the resulting support, B<--tollef>
 was obsoleted 20130222 and removed one year later.

src/parallel_tutorial.html

@@ -1746,7 +1746,25 @@
 
 <h2 id="Transferring-environment-variables-and-functions">Transferring environment variables and functions</h2>
 
-<p>Using <b>--env</b> GNU <b>parallel</b> can transfer an environment variable to the remote system.</p>
+<p><b>env_parallel</b> is a shell function that transfers all aliases, functions, variables, and arrays. You active it by running:</p>
+
+<pre><code>  source `which env_parallel.bash`</code></pre>
+
+<p>Replace <b>bash</b> with the shell you use.</p>
+
+<p>Now you can use <b>env_parallel</b> instead of <b>parallel</b> and still have your environment:</p>
+
+<pre><code>  alias myecho=echo
+  myvar=&quot;Joe&#39;s var is&quot;
+  env_parallel -S $SERVER1 &#39;myecho $myvar&#39; ::: green</code></pre>
+
+<p>Output:</p>
+
+<pre><code>  Joe&#39;s var is green</code></pre>
+
+<p>The disadvantage is that if your environment is huge <b>env_parallel</b> will fail.</p>
+
+<p>When <b>env_parallel</b> fails, you can still use <b>--env</b> to tell GNU <b>parallel</b> to transfer an environment variable to the remote system.</p>
 
 <pre><code>  MYVAR=&#39;foo bar&#39;
   export MYVAR
@@ -1797,26 +1815,19 @@
 
 <h2 id="Showing-what-is-actually-run">Showing what is actually run</h2>
 
-<p><b>--verbose</b> will show the command that would be run on the local machine. When a job is run on a remote machine, this is wrapped with <b>ssh</b> and possibly transferring files and environment variables, setting the workdir, and setting <b>--nice</b> value. <b>-vv</b> shows all of this.</p>
+<p><b>--verbose</b> will show the command that would be run on the local machine.</p>
 
-<pre><code>  parallel -vv -S $SERVER1 echo ::: bar</code></pre>
+<p>When using <b>--cat</b>, <b>--pipepart</b>, or when a job is run on a remote machine, the command is wrapped with helper scripts. <b>-vv</b> shows all of this.</p>
+
+<pre><code>  parallel -vv --pipepart wc :::: num30000</code></pre>
 
 <p>Output:</p>
 
-<pre><code>  ssh lo -- exec perl -e \&#39;&#39;@GNU_Parallel=(&quot;use&quot;,&quot;IPC::Open3;&quot;,&quot;use&quot;,&quot;MIME::Base64&quot;);
+<pre><code>  &lt;num30000 perl -e &#39;while(@ARGV) { sysseek(STDIN,shift,0) || die;
-  eval&quot;@GNU_Parallel&quot;;my$eval=decode_base64(join&quot;&quot;,@ARGV);eval$eval;&#39;\&#39;
+  $left = shift; while($read = sysread(STDIN,$buf, ($left &gt; 131072
-  JEVOVnsiUEFSQUxMRUxfUElEIn09IjI3MzQiOyRFTlZ7IlBBUkFMTEVMX1NFUSJ9PSIx
+  ? 131072 : $left))){ $left -= $read; syswrite(STDOUT,$buf); } }&#39;
-  IjskYmFzaGZ1bmMgPSAiIjtAQVJHVj0iZWNobyBiYXIiOyRzaGVsbD0iJEVOVntTSEVM
+  0 0 0 168894 | (wc)
-  TH0iOyR0bXBkaXI9Ii90bXAiOyRuaWNlPTA7ZG97JEVOVntQQVJBTExFTF9UTVB9PSR0
+    30000   30000  168894</code></pre>
-  bXBkaXIuIi9wYXIiLmpvaW4iIixtYXB7KDAuLjksImEiLi4ieiIsIkEiLi4iWiIpW3Jh
-  bmQoNjIpXX0oMS4uNSk7fXdoaWxlKC1lJEVOVntQQVJBTExFTF9UTVB9KTskU0lHe0NI
-  TER9PXN1YnskZG9uZT0xO307JHBpZD1mb3JrO3VubGVzcygkcGlkKXtzZXRwZ3JwO2V2
-  YWx7c2V0cHJpb3JpdHkoMCwwLCRuaWNlKX07ZXhlYyRzaGVsbCwiLWMiLCgkYmFzaGZ1
-  bmMuIkBBUkdWIik7ZGllImV4ZWM6JCFcbiI7fWRveyRzPSRzPDE/MC4wMDErJHMqMS4w
-  MzokcztzZWxlY3QodW5kZWYsdW5kZWYsdW5kZWYsJHMpO311bnRpbCgkZG9uZXx8Z2V0
-  cHBpZD09MSk7a2lsbChTSUdIVVAsLSR7cGlkfSl1bmxlc3MkZG9uZTt3YWl0O2V4aXQo
-  JD8mMTI3PzEyOCsoJD8mMTI3KToxKyQ/Pj44KQ==;
-  bar</code></pre>
 
 <p>When the command gets more complex, the output is so hard to read, that it is only useful for debugging:</p>
 
@@ -1828,44 +1839,40 @@
 
 <p>Output will be similar to:</p>
 
-<pre><code>  ( ssh lo -- mkdir -p ./.parallel/tmp/hk-3492-1;rsync --protocol 30
+<pre><code>  ( ssh server -- mkdir -p ./.parallel/tmp/aspire-1928520-1;rsync
-  -rlDzR -essh ./abc-file lo:./.parallel/tmp/hk-3492-1 );ssh lo --
+  --protocol 30 -rlDzR -essh ./abc-file 
-  exec perl -e \&#39;&#39;@GNU_Parallel=(&quot;use&quot;,&quot;IPC::Open3;&quot;,&quot;use&quot;,&quot;MIME::Base64&quot;);
+  server:./.parallel/tmp/aspire-1928520-1 );ssh server -- exec perl -e 
+  \&#39;&#39;@GNU_Parallel=(&quot;use&quot;,&quot;IPC::Open3;&quot;,&quot;use&quot;,&quot;MIME::Base64&quot;);
   eval&quot;@GNU_Parallel&quot;;my$eval=decode_base64(join&quot;&quot;,@ARGV);eval$eval;&#39;\&#39;
-  c3lzdGVtKCJta2RpciIsIi1wIiwiLS0iLCIucGFyYWxsZWwvdG1wL2hrLTM0OTItMSIp
+  c3lzdGVtKCJta2RpciIsIi1wIiwiLS0iLCIucGFyYWxsZWwvdG1wL2FzcGlyZS0xOTI4N
-  OyBjaGRpciAiLnBhcmFsbGVsL3RtcC9oay0zNDkyLTEiIHx8cHJpbnQoU1RERVJSICJw
+  TsgY2hkaXIgIi5wYXJhbGxlbC90bXAvYXNwaXJlLTE5Mjg1MjAtMSIgfHxwcmludChTVE
-  YXJhbGxlbDogQ2Fubm90IGNoZGlyIHRvIC5wYXJhbGxlbC90bXAvaGstMzQ5Mi0xXG4i
+  BhcmFsbGVsOiBDYW5ub3QgY2hkaXIgdG8gLnBhcmFsbGVsL3RtcC9hc3BpcmUtMTkyODU
-  KSAmJiBleGl0IDI1NTskRU5WeyJHUEdfQUdFTlRfSU5GTyJ9PSIvdG1wL2dwZy10WjVI
+  iKSAmJiBleGl0IDI1NTskRU5WeyJPTERQV0QifT0iL2hvbWUvdGFuZ2UvcHJpdmF0L3Bh
-  U0QvUy5ncGctYWdlbnQ6MjM5NzoxIjskRU5WeyJQQVJBTExFTF9TRVEifT0iMSI7JEVO
+  IjskRU5WeyJQQVJBTExFTF9QSUQifT0iMTkyODUyMCI7JEVOVnsiUEFSQUxMRUxfU0VRI
-  VnsiU1FMSVRFVEJMIn09InNxbGl0ZTM6Ly8vJTJGdG1wJTJGcGFyYWxsZWwuZGIyL3Bh
+  0BiYXNoX2Z1bmN0aW9ucz1xdyhteV9mdW5jMyk7IGlmKCRFTlZ7IlNIRUxMIn09fi9jc2
-  cnNxbDIiOyRFTlZ7IlBBUkFMTEVMX1BJRCJ9PSIzNDkyIjskRU5WeyJTUUxJVEUifT0i
+  ByaW50IFNUREVSUiAiQ1NIL1RDU0ggRE8gTk9UIFNVUFBPUlQgbmV3bGluZXMgSU4gVkF
-  c3FsaXRlMzovLy8lMkZ0bXAlMkZwYXJhbGxlbC5kYjIiOyRFTlZ7IlBBUkFMTEVMX1BJ
+  TL0ZVTkNUSU9OUy4gVW5zZXQgQGJhc2hfZnVuY3Rpb25zXG4iOyBleGVjICJmYWxzZSI7
-  RCJ9PSIzNDkyIjskRU5WeyJQQVJBTExFTF9TRVEifT0iMSI7QGJhc2hfZnVuY3Rpb25z
+  YXNoZnVuYyA9ICJteV9mdW5jMygpIHsgIGVjaG8gaW4gbXlfZnVuYyBcJDEgPiBcJDEub
-  PXF3KG15X2Z1bmMzKTsgaWYoJEVOVnsiU0hFTEwifT1+L2NzaC8pIHsgcHJpbnQgU1RE
+  Xhwb3J0IC1mIG15X2Z1bmMzID4vZGV2L251bGw7IjtAQVJHVj0ibXlfZnVuYzMgYWJjLW
-  RVJSICJDU0gvVENTSCBETyBOT1QgU1VQUE9SVCBuZXdsaW5lcyBJTiBWQVJJQUJMRVMv
+  RzaGVsbD0iJEVOVntTSEVMTH0iOyR0bXBkaXI9Ii90bXAiOyRuaWNlPTE3O2RveyRFTlZ
-  RlVOQ1RJT05TLiBVbnNldCBAYmFzaF9mdW5jdGlvbnNcbiI7IGV4ZWMgImZhbHNlIjsg
+  MRUxfVE1QfT0kdG1wZGlyLiIvcGFyIi5qb2luIiIsbWFweygwLi45LCJhIi4uInoiLCJB
-  fSAKJGJhc2hmdW5jID0gIm15X2Z1bmMzKCkgeyAgZWNobyBpbiBteV9mdW5jIFwkMSA+
+  KVtyYW5kKDYyKV19KDEuLjUpO313aGlsZSgtZSRFTlZ7UEFSQUxMRUxfVE1QfSk7JFNJ
-  IFwkMS5vdXQKfTtleHBvcnQgLWYgbXlfZnVuYzMgPi9kZXYvbnVsbDsiO0BBUkdWPSJt
+  fT1zdWJ7JGRvbmU9MTt9OyRwaWQ9Zm9yazt1bmxlc3MoJHBpZCl7c2V0cGdycDtldmFse
-  eV9mdW5jMyBhYmMtZmlsZSI7JHNoZWxsPSIkRU5We1NIRUxM
+  W9yaXR5KDAsMCwkbmljZSl9O2V4ZWMkc2hlbGwsIi1jIiwoJGJhc2hmdW5jLiJAQVJHVi
-  fSI7JHRtcGRpcj0iL3RtcCI7JG5pY2U9MTc7ZG97JEVOVntQQVJBTExFTF9UTVB9PSR0
+  JleGVjOiQhXG4iO31kb3skcz0kczwxPzAuMDAxKyRzKjEuMDM6JHM7c2VsZWN0KHVuZGV
-  bXBkaXIuIi9wYXIiLmpvaW4iIixtYXB7KDAuLjksImEiLi4ieiIsIkEiLi4iWiIpW3Jh
+  mLHVuZGVmLCRzKTt9dW50aWwoJGRvbmV8fGdldHBwaWQ9PTEpO2tpbGwoU0lHSFVQLC0k
-  bmQoNjIpXX0oMS4uNSk7fXdoaWxlKC1lJEVOVntQQVJBTExFTF9UTVB9KTskU0lHe0NI
+  dW5sZXNzJGRvbmU7d2FpdDtleGl0KCQ/JjEyNz8xMjgrKCQ/JjEyNyk6MSskPz4+OCk=;
-  TER9PXN1YnskZG9uZT0xO307JHBpZD1mb3JrO3VubGVzcygkcGlkKXtzZXRwZ3JwO2V2
+  _EXIT_status=$?; mkdir -p ./.; rsync --protocol 30 --rsync-path=cd\
-  YWx7c2V0cHJpb3JpdHkoMCwwLCRuaWNlKX07ZXhlYyRzaGVsbCwiLWMiLCgkYmFzaGZ1
+  ./.parallel/tmp/aspire-1928520-1/./.\;\ rsync -rlDzR -essh
-  bmMuIkBBUkdWIik7ZGllImV4ZWM6JCFcbiI7fWRveyRzPSRzPDE/MC4wMDErJHMqMS4w
+  server:./abc-file.out ./.;ssh server -- \(rm\ -f\
-  MzokcztzZWxlY3QodW5kZWYsdW5kZWYsdW5kZWYsJHMpO311bnRpbCgkZG9uZXx8Z2V0
+  ./.parallel/tmp/aspire-1928520-1/abc-file\;\ sh\ -c\ \&#39;rmdir\
-  cHBpZD09MSk7a2lsbChTSUdIVVAsLSR7cGlkfSl1bmxlc3MkZG9uZTt3YWl0O2V4aXQo
+  ./.parallel/tmp/aspire-1928520-1/\ ./.parallel/tmp/\ ./.parallel/\
-  JD8mMTI3PzEyOCsoJD8mMTI3KToxKyQ/Pj44KQ==;_EXIT_status=$?;
+  2\&gt;/dev/null\&#39;\;rm\ -rf\ ./.parallel/tmp/aspire-1928520-1\;\);ssh
-  mkdir -p ./.; rsync --protocol 30 --rsync-path=cd\
+  server -- \(rm\ -f\ ./.parallel/tmp/aspire-1928520-1/abc-file.out\;\
-  ./.parallel/tmp/hk-3492-1/./.\;\ rsync -rlDzR -essh
+  sh\ -c\ \&#39;rmdir\ ./.parallel/tmp/aspire-1928520-1/\ ./.parallel/tmp/\
-  lo:./abc-file.out ./.;ssh lo -- \(rm\ -f\
+  ./.parallel/\ 2\&gt;/dev/null\&#39;\;rm\ -rf\ 
-  ./.parallel/tmp/hk-3492-1/abc-file\;\ sh\ -c\ \&#39;rmdir\
+  ./.parallel/tmp/aspire-1928520-1\;\);ssh server -- rm -rf 
-  ./.parallel/tmp/hk-3492-1/\ ./.parallel/tmp/\ ./.parallel/\
+  .parallel/tmp/aspire-1928520-1; exit $_EXIT_status;</code></pre>
-  2\&gt;/dev/null\&#39;\;rm\ -rf\ ./.parallel/tmp/hk-3492-1\;\);ssh lo --
-  \(rm\ -f\ ./.parallel/tmp/hk-3492-1/abc-file.out\;\ sh\ -c\ \&#39;rmdir\
-  ./.parallel/tmp/hk-3492-1/\ ./.parallel/tmp/\ ./.parallel/\
-  2\&gt;/dev/null\&#39;\;rm\ -rf\ ./.parallel/tmp/hk-3492-1\;\);ssh lo -- rm
-  -rf .parallel/tmp/hk-3492-1; exit $_EXIT_status;</code></pre>
 
 <h1 id="Saving-to-an-SQL-base-advanced">Saving to an SQL base (advanced)</h1>
 
@@ -1901,6 +1908,18 @@
 
 <p>A proper CSV reader (like LibreOffice or R&#39;s read.csv) will read this format correctly - even with fields containing newlines as above.</p>
 
+<p>If the output is big you may want to put it into files using <b>--results</b>:</p>
+
+<pre><code>  parallel --results outdir --sqlandworker csv:////%2Ftmp%2Flog2.csv seq ::: 10 ::: 12 13 14
+  cat /tmp/log2.csv</code></pre>
+
+<p>Output will be similar to:</p>
+
+<pre><code>  Seq,Host,Starttime,JobRuntime,Send,Receive,Exitval,_Signal,Command,V1,V2,Stdout,Stderr
+  1,:,1458824738.287,0.029,0,9,0,0,&quot;seq 10 12&quot;,10,12,outdir/1/10/2/12/stdout,outdir/1/10/2/12/stderr
+  2,:,1458824738.298,0.025,0,12,0,0,&quot;seq 10 13&quot;,10,13,outdir/1/10/2/13/stdout,outdir/1/10/2/13/stderr
+  3,:,1458824738.309,0.026,0,15,0,0,&quot;seq 10 14&quot;,10,14,outdir/1/10/2/14/stdout,outdir/1/10/2/14/stderr</code></pre>
+
 <h2 id="DBURL-as-table">DBURL as table</h2>
 
 <p>The CSV file is an example of a DBURL.</p>
@@ -1984,7 +2003,7 @@
   149796  149796 1048572
    85349   85349  597444</code></pre>
 
-<p>The size of the chunk is not exactly 1 MB because GNU <b>parallel</b> only passes full lines - never half a line, thus the blocksize is only average 1 MB. You can change the block size to 2 MB with <b>--block</b>:</p>
+<p>The size of the chunk is not exactly 1 MB because GNU <b>parallel</b> only passes full lines - never half a line, thus the blocksize is only 1 MB on average. You can change the block size to 2 MB with <b>--block</b>:</p>
 
 <pre><code>  cat num1000000 | parallel --pipe --block 2M wc</code></pre>
 
@@ -1995,7 +2014,7 @@
   299593  299593 2097151
    85349   85349  597444</code></pre>
 
-<p>GNU <b>parallel</b> treats each line as a record. If the order of record is unimportant (e.g. you need all lines processed, but you do not care which is processed first), then you can use <b>--round-robin</b>. Without <b>--round-robin</b> GNU <b>parallel</b> will start a command per block; with <b>--round-robin</b> only the requested number of jobs will be started (<b>--jobs</b>). The records will then be distributed between the running jobs:</p>
+<p>GNU <b>parallel</b> treats each line as a record. If the order of records is unimportant (e.g. you need all lines processed, but you do not care which is processed first), then you can use <b>--round-robin</b>. Without <b>--round-robin</b> GNU <b>parallel</b> will start a command per block; with <b>--round-robin</b> only the requested number of jobs will be started (<b>--jobs</b>). The records will then be distributed between the running jobs:</p>
 
 <pre><code>  cat num1000000 | parallel --pipe -j4 --round-robin wc</code></pre>
 
@@ -2027,7 +2046,7 @@
   140000  140000  980000
    20000   20000  140001</code></pre>
 
-<p>Notice that the last job could not get the full 140000 lines, but only 20000 lines.</p>
+<p>Note how that the last job could not get the full 140000 lines, but only 20000 lines.</p>
 
 <p>If a record is 75 lines <b>-L</b> can be used:</p>
 
@@ -2044,7 +2063,7 @@
    85350   85350  597450
       25      25     176</code></pre>
 
-<p>Notice GNU <b>parallel</b> still reads a block of around 1 MB; but instead of passing full lines to <b>wc</b> it passes full 75 lines at a time. This of course does not hold for the last job (which in this case got 25 lines).</p>
+<p>Note how GNU <b>parallel</b> still reads a block of around 1 MB; but instead of passing full lines to <b>wc</b> it passes full 75 lines at a time. This of course does not hold for the last job (which in this case got 25 lines).</p>
 
 <h2 id="Record-separators">Record separators</h2>
 
@@ -2386,7 +2405,7 @@
 
 <p>GNU <b>parallel</b> can work as a counting semaphore. This is slower and less efficient than its normal mode.</p>
 
-<p>A counting semaphore is like a row of toilets. People needing a toilet can use any toilet, but if there are more people than toilets, they will have to wait for one of the toilets to be available.</p>
+<p>A counting semaphore is like a row of toilets. People needing a toilet can use any toilet, but if there are more people than toilets, they will have to wait for one of the toilets to become available.</p>
 
 <p>An alias for <b>parallel --semaphore</b> is <b>sem</b>.</p>
 
@@ -2427,7 +2446,7 @@
 
 <p>The difference between this and just running the command, is that a mutex is set, so if other <b>sem</b>s were running in the background only one would run at a time.</p>
 
-<p>To tell the difference between which semaphore is used, use <b>--semaphorename</b>/<b>--id</b>. Run this in one terminal:</p>
+<p>To control which semaphore is used, use <b>--semaphorename</b>/<b>--id</b>. Run this in one terminal:</p>
 
 <pre><code>  sem --id my_id -u &#39;echo First started; sleep 10; echo The first finished&#39;</code></pre>
 
@@ -2490,52 +2509,59 @@
 <p>Output:</p>
 
 <pre><code>  Usage:
+  
   parallel [options] [command [arguments]] &lt; list_of_arguments
   parallel [options] [command [arguments]] (::: arguments|:::: argfile(s))...
   cat ... | parallel --pipe [options] [command [arguments]]
   
-  -j n           Run n jobs in parallel
+  -j n            Run n jobs in parallel
-  -k             Keep same order
+  -k              Keep same order
-  -X             Multiple arguments with context replace
+  -X              Multiple arguments with context replace
-  --colsep regexp      Split input on regexp for positional replacements
+  --colsep regexp Split input on regexp for positional replacements
-  {} {.} {/} {/.} {#}  Replacement strings
+  {} {.} {/} {/.} {#} {%} {= perl code =} Replacement strings
-  {3} {3.} {3/} {3/.}  Positional replacement strings
+  {3} {3.} {3/} {3/.} {=3 perl code =}    Positional replacement strings
+  With --plus:    {} = {+/}/{/} = {.}.{+.} = {+/}/{/.}.{+.} = {..}.{+..} =
+                  {+/}/{/..}.{+..} = {...}.{+...} = {+/}/{/...}.{+...}
   
-  -S sshlogin    Example: foo@server.example.com
+  -S sshlogin     Example: foo@server.example.com
-  --slf ..       Use ~/.parallel/sshloginfile as the list of sshlogins
+  --slf ..        Use ~/.parallel/sshloginfile as the list of sshlogins
-  --trc {}.bar   Shorthand for --transfer --return {}.bar --cleanup
+  --trc {}.bar    Shorthand for --transfer --return {}.bar --cleanup
-  --onall        Run the given command with argument on all sshlogins
+  --onall         Run the given command with argument on all sshlogins
-  --nonall       Run the given command with no arguments on all sshlogins
+  --nonall        Run the given command with no arguments on all sshlogins
   
-  --pipe         Split stdin (standard input) to multiple jobs.
+  --pipe          Split stdin (standard input) to multiple jobs.
-  --recend str   Record end separator for --pipe.
+  --recend str    Record end separator for --pipe.
-  --recstart str Record start separator for --pipe.
+  --recstart str  Record start separator for --pipe.
   
   See &#39;man parallel&#39; for details
   
-  When using GNU Parallel for a publication please cite:
+  Academic tradition requires you to cite works you base your article on.
+  When using programs that use GNU Parallel to process data for publication
+  please cite:
   
-  O. Tange (2011): GNU Parallel - The Command-Line Power Tool,
+    O. Tange (2011): GNU Parallel - The Command-Line Power Tool,
-  ;login: The USENIX Magazine, February 2011:42-47.
+    ;login: The USENIX Magazine, February 2011:42-47.
   
-    When asking for help, always report the full output of this:
+  This helps funding further development; AND IT WON&#39;T COST YOU A CENT.
+  If you pay 10000 EUR you should feel free to use GNU Parallel without citing.</code></pre>
 
-  parallel --version</code></pre>
+<p>When asking for help, always report the full output of this:</p>
+
+<pre><code>  parallel --version</code></pre>
 
 <p>Output:</p>
 
-<pre><code>  GNU parallel 20130822
+<pre><code>  GNU parallel 20160323
-  Copyright (C) 2007,2008,2009,2010,2011,2012,2013 Ole Tange and Free Software Foundation, Inc.
+  Copyright (C) 2007,2008,2009,2010,2011,2012,2013,2014,2015,2016
+  Ole Tange and Free Software Foundation, Inc.
   License GPLv3+: GNU GPL version 3 or later &lt;http://gnu.org/licenses/gpl.html&gt;
   This is free software: you are free to change and redistribute it.
   GNU parallel comes with no warranty.
   
   Web site: http://www.gnu.org/software/parallel
   
-  When using GNU Parallel for a publication please cite:
+  When using programs that use GNU Parallel to process data for publication
-  
+  please cite as described in &#39;parallel --bibtex&#39;.</code></pre>
-  O. Tange (2011): GNU Parallel - The Command-Line Power Tool,
-  ;login: The USENIX Magazine, February 2011:42-47.</code></pre>
 
 <p>In scripts <b>--minversion</b> can be used to ensure the user has at least this version:</p>
 
@@ -2543,7 +2569,7 @@
 
 <p>Output:</p>
 
-<pre><code>  20130722
+<pre><code>  20160322
   Your version is at least 20130722.</code></pre>
 
 <p>If using GNU <b>parallel</b> for research the BibTeX citation can be generated using <b>--bibtex</b>:</p>
@@ -2552,18 +2578,32 @@
 
 <p>Output:</p>
 
-<pre><code>  @article{Tange2011a,
+<pre><code>  Academic tradition requires you to cite works you base your article on.
-   title = {GNU Parallel - The Command-Line Power Tool},
+  When using programs that use GNU Parallel to process data for publication
-   author = {O. Tange},
+  please cite:
-   address = {Frederiksberg, Denmark},
+  
-   journal = {;login: The USENIX Magazine},
+  @article{Tange2011a,
-   month = {Feb},
+    title = {GNU Parallel - The Command-Line Power Tool},
-   number = {1},
+    author = {O. Tange},
-   volume = {36},
+    address = {Frederiksberg, Denmark},
-   url = {http://www.gnu.org/s/parallel},
+    journal = {;login: The USENIX Magazine},
-   year = {2011},
+    month = {Feb},
-   pages = {42-47}
+    number = {1},
-  }</code></pre>
+    volume = {36},
+    url = {http://www.gnu.org/s/parallel},
+    year = {2011},
+    pages = {42-47},
+    doi = {10.5281/zenodo.16303}
+  }
+  
+  (Feel free to use \nocite{Tange2011a})
+  
+  This helps funding further development; AND IT WON&#39;T COST YOU A CENT.
+  If you pay 10000 EUR you should feel free to use GNU Parallel without citing.
+  
+  If you send a copy of your published article to tange@gnu.org, it will be
+  mentioned in the release notes of next version of GNU Parallel.
+  </code></pre>
 
 <p>With <b>--max-line-length-allowed</b> GNU <b>parallel</b> will report the maximal size of the command line:</p>
 
@@ -2606,9 +2646,9 @@
 
 <p>Output:</p>
 
-<pre><code>  \nice -n17 /bin/bash -c echo\ A
+<pre><code>  echo A
-  \nice -n17 /bin/bash -c echo\ B
+  echo B
-  \nice -n17 /bin/bash -c echo\ C</code></pre>
+  echo C</code></pre>
 
 <h1 id="Spread-the-word">Spread the word</h1>
 

src/parallel_tutorial.pod

@@ -1731,8 +1731,29 @@ B<--jobs> sets the number of servers to log in to in parallel.
 
 =head2 Transferring environment variables and functions
 
-Using B<--env> GNU B<parallel> can transfer an environment variable to the
+B<env_parallel> is a shell function that transfers all aliases,
-remote system.
+functions, variables, and arrays. You active it by running:
+
+  source `which env_parallel.bash`
+
+Replace B<bash> with the shell you use.
+
+Now you can use B<env_parallel> instead of B<parallel> and still have
+your environment:
+
+  alias myecho=echo
+  myvar="Joe's var is"
+  env_parallel -S $SERVER1 'myecho $myvar' ::: green
+
+Output:
+
+  Joe's var is green
+
+The disadvantage is that if your environment is huge B<env_parallel>
+will fail.
+
+When B<env_parallel> fails, you can still use B<--env> to tell GNU
+B<parallel> to transfer an environment variable to the remote system.
 
   MYVAR='foo bar'
   export MYVAR
@@ -1784,31 +1805,26 @@ Output:
   foo
   in my_func2 foo bar
 
+
+
 =head2 Showing what is actually run
 
 B<--verbose> will show the command that would be run on the local
-machine. When a job is run on a remote machine, this is wrapped with
+machine.
-B<ssh> and possibly transferring files and environment variables, setting
-the workdir, and setting B<--nice> value. B<-vv> shows all of this.
 
-  parallel -vv -S $SERVER1 echo ::: bar
+When using B<--cat>, B<--pipepart>, or when a job is run on a remote
+machine, the command is wrapped with helper scripts. B<-vv> shows all
+of this.
+
+  parallel -vv --pipepart wc :::: num30000
 
 Output:
 
-  ssh lo -- exec perl -e \''@GNU_Parallel=("use","IPC::Open3;","use","MIME::Base64");
+  <num30000 perl -e 'while(@ARGV) { sysseek(STDIN,shift,0) || die;
-  eval"@GNU_Parallel";my$eval=decode_base64(join"",@ARGV);eval$eval;'\'
+  $left = shift; while($read = sysread(STDIN,$buf, ($left > 131072
-  JEVOVnsiUEFSQUxMRUxfUElEIn09IjI3MzQiOyRFTlZ7IlBBUkFMTEVMX1NFUSJ9PSIx
+  ? 131072 : $left))){ $left -= $read; syswrite(STDOUT,$buf); } }'
-  IjskYmFzaGZ1bmMgPSAiIjtAQVJHVj0iZWNobyBiYXIiOyRzaGVsbD0iJEVOVntTSEVM
+  0 0 0 168894 | (wc)
-  TH0iOyR0bXBkaXI9Ii90bXAiOyRuaWNlPTA7ZG97JEVOVntQQVJBTExFTF9UTVB9PSR0
+    30000   30000  168894
-  bXBkaXIuIi9wYXIiLmpvaW4iIixtYXB7KDAuLjksImEiLi4ieiIsIkEiLi4iWiIpW3Jh
-  bmQoNjIpXX0oMS4uNSk7fXdoaWxlKC1lJEVOVntQQVJBTExFTF9UTVB9KTskU0lHe0NI
-  TER9PXN1YnskZG9uZT0xO307JHBpZD1mb3JrO3VubGVzcygkcGlkKXtzZXRwZ3JwO2V2
-  YWx7c2V0cHJpb3JpdHkoMCwwLCRuaWNlKX07ZXhlYyRzaGVsbCwiLWMiLCgkYmFzaGZ1
-  bmMuIkBBUkdWIik7ZGllImV4ZWM6JCFcbiI7fWRveyRzPSRzPDE/MC4wMDErJHMqMS4w
-  MzokcztzZWxlY3QodW5kZWYsdW5kZWYsdW5kZWYsJHMpO311bnRpbCgkZG9uZXx8Z2V0
-  cHBpZD09MSk7a2lsbChTSUdIVVAsLSR7cGlkfSl1bmxlc3MkZG9uZTt3YWl0O2V4aXQo
-  JD8mMTI3PzEyOCsoJD8mMTI3KToxKyQ/Pj44KQ==;
-  bar
 
 When the command gets more complex, the output is so hard to read,
 that it is only useful for debugging:
@@ -1821,44 +1837,41 @@ that it is only useful for debugging:
 
 Output will be similar to:
 
-  ( ssh lo -- mkdir -p ./.parallel/tmp/hk-3492-1;rsync --protocol 30
+
-  -rlDzR -essh ./abc-file lo:./.parallel/tmp/hk-3492-1 );ssh lo --
+  ( ssh server -- mkdir -p ./.parallel/tmp/aspire-1928520-1;rsync
-  exec perl -e \''@GNU_Parallel=("use","IPC::Open3;","use","MIME::Base64");
+  --protocol 30 -rlDzR -essh ./abc-file 
+  server:./.parallel/tmp/aspire-1928520-1 );ssh server -- exec perl -e 
+  \''@GNU_Parallel=("use","IPC::Open3;","use","MIME::Base64");
   eval"@GNU_Parallel";my$eval=decode_base64(join"",@ARGV);eval$eval;'\'
-  c3lzdGVtKCJta2RpciIsIi1wIiwiLS0iLCIucGFyYWxsZWwvdG1wL2hrLTM0OTItMSIp
+  c3lzdGVtKCJta2RpciIsIi1wIiwiLS0iLCIucGFyYWxsZWwvdG1wL2FzcGlyZS0xOTI4N
-  OyBjaGRpciAiLnBhcmFsbGVsL3RtcC9oay0zNDkyLTEiIHx8cHJpbnQoU1RERVJSICJw
+  TsgY2hkaXIgIi5wYXJhbGxlbC90bXAvYXNwaXJlLTE5Mjg1MjAtMSIgfHxwcmludChTVE
-  YXJhbGxlbDogQ2Fubm90IGNoZGlyIHRvIC5wYXJhbGxlbC90bXAvaGstMzQ5Mi0xXG4i
+  BhcmFsbGVsOiBDYW5ub3QgY2hkaXIgdG8gLnBhcmFsbGVsL3RtcC9hc3BpcmUtMTkyODU
-  KSAmJiBleGl0IDI1NTskRU5WeyJHUEdfQUdFTlRfSU5GTyJ9PSIvdG1wL2dwZy10WjVI
+  iKSAmJiBleGl0IDI1NTskRU5WeyJPTERQV0QifT0iL2hvbWUvdGFuZ2UvcHJpdmF0L3Bh
-  U0QvUy5ncGctYWdlbnQ6MjM5NzoxIjskRU5WeyJQQVJBTExFTF9TRVEifT0iMSI7JEVO
+  IjskRU5WeyJQQVJBTExFTF9QSUQifT0iMTkyODUyMCI7JEVOVnsiUEFSQUxMRUxfU0VRI
-  VnsiU1FMSVRFVEJMIn09InNxbGl0ZTM6Ly8vJTJGdG1wJTJGcGFyYWxsZWwuZGIyL3Bh
+  0BiYXNoX2Z1bmN0aW9ucz1xdyhteV9mdW5jMyk7IGlmKCRFTlZ7IlNIRUxMIn09fi9jc2
-  cnNxbDIiOyRFTlZ7IlBBUkFMTEVMX1BJRCJ9PSIzNDkyIjskRU5WeyJTUUxJVEUifT0i
+  ByaW50IFNUREVSUiAiQ1NIL1RDU0ggRE8gTk9UIFNVUFBPUlQgbmV3bGluZXMgSU4gVkF
-  c3FsaXRlMzovLy8lMkZ0bXAlMkZwYXJhbGxlbC5kYjIiOyRFTlZ7IlBBUkFMTEVMX1BJ
+  TL0ZVTkNUSU9OUy4gVW5zZXQgQGJhc2hfZnVuY3Rpb25zXG4iOyBleGVjICJmYWxzZSI7
-  RCJ9PSIzNDkyIjskRU5WeyJQQVJBTExFTF9TRVEifT0iMSI7QGJhc2hfZnVuY3Rpb25z
+  YXNoZnVuYyA9ICJteV9mdW5jMygpIHsgIGVjaG8gaW4gbXlfZnVuYyBcJDEgPiBcJDEub
-  PXF3KG15X2Z1bmMzKTsgaWYoJEVOVnsiU0hFTEwifT1+L2NzaC8pIHsgcHJpbnQgU1RE
+  Xhwb3J0IC1mIG15X2Z1bmMzID4vZGV2L251bGw7IjtAQVJHVj0ibXlfZnVuYzMgYWJjLW
-  RVJSICJDU0gvVENTSCBETyBOT1QgU1VQUE9SVCBuZXdsaW5lcyBJTiBWQVJJQUJMRVMv
+  RzaGVsbD0iJEVOVntTSEVMTH0iOyR0bXBkaXI9Ii90bXAiOyRuaWNlPTE3O2RveyRFTlZ
-  RlVOQ1RJT05TLiBVbnNldCBAYmFzaF9mdW5jdGlvbnNcbiI7IGV4ZWMgImZhbHNlIjsg
+  MRUxfVE1QfT0kdG1wZGlyLiIvcGFyIi5qb2luIiIsbWFweygwLi45LCJhIi4uInoiLCJB
-  fSAKJGJhc2hmdW5jID0gIm15X2Z1bmMzKCkgeyAgZWNobyBpbiBteV9mdW5jIFwkMSA+
+  KVtyYW5kKDYyKV19KDEuLjUpO313aGlsZSgtZSRFTlZ7UEFSQUxMRUxfVE1QfSk7JFNJ
-  IFwkMS5vdXQKfTtleHBvcnQgLWYgbXlfZnVuYzMgPi9kZXYvbnVsbDsiO0BBUkdWPSJt
+  fT1zdWJ7JGRvbmU9MTt9OyRwaWQ9Zm9yazt1bmxlc3MoJHBpZCl7c2V0cGdycDtldmFse
-  eV9mdW5jMyBhYmMtZmlsZSI7JHNoZWxsPSIkRU5We1NIRUxM
+  W9yaXR5KDAsMCwkbmljZSl9O2V4ZWMkc2hlbGwsIi1jIiwoJGJhc2hmdW5jLiJAQVJHVi
-  fSI7JHRtcGRpcj0iL3RtcCI7JG5pY2U9MTc7ZG97JEVOVntQQVJBTExFTF9UTVB9PSR0
+  JleGVjOiQhXG4iO31kb3skcz0kczwxPzAuMDAxKyRzKjEuMDM6JHM7c2VsZWN0KHVuZGV
-  bXBkaXIuIi9wYXIiLmpvaW4iIixtYXB7KDAuLjksImEiLi4ieiIsIkEiLi4iWiIpW3Jh
+  mLHVuZGVmLCRzKTt9dW50aWwoJGRvbmV8fGdldHBwaWQ9PTEpO2tpbGwoU0lHSFVQLC0k
-  bmQoNjIpXX0oMS4uNSk7fXdoaWxlKC1lJEVOVntQQVJBTExFTF9UTVB9KTskU0lHe0NI
+  dW5sZXNzJGRvbmU7d2FpdDtleGl0KCQ/JjEyNz8xMjgrKCQ/JjEyNyk6MSskPz4+OCk=;
-  TER9PXN1YnskZG9uZT0xO307JHBpZD1mb3JrO3VubGVzcygkcGlkKXtzZXRwZ3JwO2V2
+  _EXIT_status=$?; mkdir -p ./.; rsync --protocol 30 --rsync-path=cd\
-  YWx7c2V0cHJpb3JpdHkoMCwwLCRuaWNlKX07ZXhlYyRzaGVsbCwiLWMiLCgkYmFzaGZ1
+  ./.parallel/tmp/aspire-1928520-1/./.\;\ rsync -rlDzR -essh
-  bmMuIkBBUkdWIik7ZGllImV4ZWM6JCFcbiI7fWRveyRzPSRzPDE/MC4wMDErJHMqMS4w
+  server:./abc-file.out ./.;ssh server -- \(rm\ -f\
-  MzokcztzZWxlY3QodW5kZWYsdW5kZWYsdW5kZWYsJHMpO311bnRpbCgkZG9uZXx8Z2V0
+  ./.parallel/tmp/aspire-1928520-1/abc-file\;\ sh\ -c\ \'rmdir\
-  cHBpZD09MSk7a2lsbChTSUdIVVAsLSR7cGlkfSl1bmxlc3MkZG9uZTt3YWl0O2V4aXQo
+  ./.parallel/tmp/aspire-1928520-1/\ ./.parallel/tmp/\ ./.parallel/\
-  JD8mMTI3PzEyOCsoJD8mMTI3KToxKyQ/Pj44KQ==;_EXIT_status=$?;
+  2\>/dev/null\'\;rm\ -rf\ ./.parallel/tmp/aspire-1928520-1\;\);ssh
-  mkdir -p ./.; rsync --protocol 30 --rsync-path=cd\
+  server -- \(rm\ -f\ ./.parallel/tmp/aspire-1928520-1/abc-file.out\;\
-  ./.parallel/tmp/hk-3492-1/./.\;\ rsync -rlDzR -essh
+  sh\ -c\ \'rmdir\ ./.parallel/tmp/aspire-1928520-1/\ ./.parallel/tmp/\
-  lo:./abc-file.out ./.;ssh lo -- \(rm\ -f\
+  ./.parallel/\ 2\>/dev/null\'\;rm\ -rf\ 
-  ./.parallel/tmp/hk-3492-1/abc-file\;\ sh\ -c\ \'rmdir\
+  ./.parallel/tmp/aspire-1928520-1\;\);ssh server -- rm -rf 
-  ./.parallel/tmp/hk-3492-1/\ ./.parallel/tmp/\ ./.parallel/\
+  .parallel/tmp/aspire-1928520-1; exit $_EXIT_status;
-  2\>/dev/null\'\;rm\ -rf\ ./.parallel/tmp/hk-3492-1\;\);ssh lo --
-  \(rm\ -f\ ./.parallel/tmp/hk-3492-1/abc-file.out\;\ sh\ -c\ \'rmdir\
-  ./.parallel/tmp/hk-3492-1/\ ./.parallel/tmp/\ ./.parallel/\
-  2\>/dev/null\'\;rm\ -rf\ ./.parallel/tmp/hk-3492-1\;\);ssh lo -- rm
-  -rf .parallel/tmp/hk-3492-1; exit $_EXIT_status;
 
 =head1 Saving to an SQL base (advanced)
 
@@ -1897,6 +1910,19 @@ Output will be similar to:
 A proper CSV reader (like LibreOffice or R's read.csv) will read this
 format correctly - even with fields containing newlines as above.
 
+If the output is big you may want to put it into files using B<--results>:
+
+  parallel --results outdir --sqlandworker csv:////%2Ftmp%2Flog2.csv seq ::: 10 ::: 12 13 14
+  cat /tmp/log2.csv
+
+Output will be similar to:
+
+  Seq,Host,Starttime,JobRuntime,Send,Receive,Exitval,_Signal,Command,V1,V2,Stdout,Stderr
+  1,:,1458824738.287,0.029,0,9,0,0,"seq 10 12",10,12,outdir/1/10/2/12/stdout,outdir/1/10/2/12/stderr
+  2,:,1458824738.298,0.025,0,12,0,0,"seq 10 13",10,13,outdir/1/10/2/13/stdout,outdir/1/10/2/13/stderr
+  3,:,1458824738.309,0.026,0,15,0,0,"seq 10 14",10,14,outdir/1/10/2/14/stdout,outdir/1/10/2/14/stderr
+
+
 =head2 DBURL as table
 
 The CSV file is an example of a DBURL.
@@ -1996,7 +2022,7 @@ Output (the order may be different):
 
 The size of the chunk is not exactly 1 MB because GNU B<parallel> only
 passes full lines - never half a line, thus the blocksize is only
-average 1 MB. You can change the block size to 2 MB with B<--block>:
+1 MB on average. You can change the block size to 2 MB with B<--block>:
 
   cat num1000000 | parallel --pipe --block 2M wc
 
@@ -2007,8 +2033,8 @@ Output (the order may be different):
   299593  299593 2097151
    85349   85349  597444
 
-GNU B<parallel> treats each line as a record. If the order of record is
+GNU B<parallel> treats each line as a record. If the order of records
-unimportant (e.g. you need all lines processed, but you do not care
+is unimportant (e.g. you need all lines processed, but you do not care
 which is processed first), then you can use B<--round-robin>. Without
 B<--round-robin> GNU B<parallel> will start a command per block; with
 B<--round-robin> only the requested number of jobs will be started
@@ -2047,8 +2073,8 @@ Output (the order may be different):
   140000  140000  980000
    20000   20000  140001
 
-Notice that the last job could not get the full 140000 lines, but only
+Note how that the last job could not get the full 140000 lines, but
-20000 lines.
+only 20000 lines.
 
 If a record is 75 lines B<-L> can be used:
 
@@ -2065,10 +2091,10 @@ Output (the order may be different):
    85350   85350  597450
       25      25     176
 
-Notice GNU B<parallel> still reads a block of around 1 MB; but instead of
+Note how GNU B<parallel> still reads a block of around 1 MB; but
-passing full lines to B<wc> it passes full 75 lines at a time. This
+instead of passing full lines to B<wc> it passes full 75 lines at a
-of course does not hold for the last job (which in this case got 25
+time. This of course does not hold for the last job (which in this
-lines).
+case got 25 lines).
 
 =head2 Record separators
 
@@ -2405,7 +2431,7 @@ efficient than its normal mode.
 
 A counting semaphore is like a row of toilets. People needing a toilet
 can use any toilet, but if there are more people than toilets, they
-will have to wait for one of the toilets to be available.
+will have to wait for one of the toilets to become available.
 
 An alias for B<parallel --semaphore> is B<sem>.
 
@@ -2455,7 +2481,7 @@ The difference between this and just running the command, is that a
 mutex is set, so if other B<sem>s were running in the background only one
 would run at a time.
 
-To tell the difference between which semaphore is used, use
+To control which semaphore is used, use
 B<--semaphorename>/B<--id>. Run this in one terminal:
 
   sem --id my_id -u 'echo First started; sleep 10; echo The first finished'
@@ -2525,52 +2551,59 @@ B<--help> will print a summary of the most important options:
 Output:
 
   Usage:
+  
   parallel [options] [command [arguments]] < list_of_arguments
   parallel [options] [command [arguments]] (::: arguments|:::: argfile(s))...
   cat ... | parallel --pipe [options] [command [arguments]]
   
-  -j n           Run n jobs in parallel
+  -j n            Run n jobs in parallel
-  -k             Keep same order
+  -k              Keep same order
-  -X             Multiple arguments with context replace
+  -X              Multiple arguments with context replace
-  --colsep regexp      Split input on regexp for positional replacements
+  --colsep regexp Split input on regexp for positional replacements
-  {} {.} {/} {/.} {#}  Replacement strings
+  {} {.} {/} {/.} {#} {%} {= perl code =} Replacement strings
-  {3} {3.} {3/} {3/.}  Positional replacement strings
+  {3} {3.} {3/} {3/.} {=3 perl code =}    Positional replacement strings
+  With --plus:    {} = {+/}/{/} = {.}.{+.} = {+/}/{/.}.{+.} = {..}.{+..} =
+                  {+/}/{/..}.{+..} = {...}.{+...} = {+/}/{/...}.{+...}
   
-  -S sshlogin    Example: foo@server.example.com
+  -S sshlogin     Example: foo@server.example.com
-  --slf ..       Use ~/.parallel/sshloginfile as the list of sshlogins
+  --slf ..        Use ~/.parallel/sshloginfile as the list of sshlogins
-  --trc {}.bar   Shorthand for --transfer --return {}.bar --cleanup
+  --trc {}.bar    Shorthand for --transfer --return {}.bar --cleanup
-  --onall        Run the given command with argument on all sshlogins
+  --onall         Run the given command with argument on all sshlogins
-  --nonall       Run the given command with no arguments on all sshlogins
+  --nonall        Run the given command with no arguments on all sshlogins
   
-  --pipe         Split stdin (standard input) to multiple jobs.
+  --pipe          Split stdin (standard input) to multiple jobs.
-  --recend str   Record end separator for --pipe.
+  --recend str    Record end separator for --pipe.
-  --recstart str Record start separator for --pipe.
+  --recstart str  Record start separator for --pipe.
   
   See 'man parallel' for details
   
-  When using GNU Parallel for a publication please cite:
+  Academic tradition requires you to cite works you base your article on.
+  When using programs that use GNU Parallel to process data for publication
+  please cite:
   
-  O. Tange (2011): GNU Parallel - The Command-Line Power Tool,
+    O. Tange (2011): GNU Parallel - The Command-Line Power Tool,
-  ;login: The USENIX Magazine, February 2011:42-47.
+    ;login: The USENIX Magazine, February 2011:42-47.
   
-    When asking for help, always report the full output of this:
+  This helps funding further development; AND IT WON'T COST YOU A CENT.
+  If you pay 10000 EUR you should feel free to use GNU Parallel without citing.
+
+When asking for help, always report the full output of this:
 
   parallel --version
 
 Output:
 
-  GNU parallel 20130822
+  GNU parallel 20160323
-  Copyright (C) 2007,2008,2009,2010,2011,2012,2013 Ole Tange and Free Software Foundation, Inc.
+  Copyright (C) 2007,2008,2009,2010,2011,2012,2013,2014,2015,2016
+  Ole Tange and Free Software Foundation, Inc.
   License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
   This is free software: you are free to change and redistribute it.
   GNU parallel comes with no warranty.
   
   Web site: http://www.gnu.org/software/parallel
   
-  When using GNU Parallel for a publication please cite:
+  When using programs that use GNU Parallel to process data for publication
-  
+  please cite as described in 'parallel --bibtex'.
-  O. Tange (2011): GNU Parallel - The Command-Line Power Tool,
-  ;login: The USENIX Magazine, February 2011:42-47.
 
 In scripts B<--minversion> can be used to ensure the user has at least
 this version:
@@ -2579,7 +2612,7 @@ this version:
 
 Output:
 
-  20130722
+  20160322
   Your version is at least 20130722.
 
 If using GNU B<parallel> for research the BibTeX citation can be
@@ -2589,19 +2622,32 @@ generated using B<--bibtex>:
 
 Output:
 
+  Academic tradition requires you to cite works you base your article on.
+  When using programs that use GNU Parallel to process data for publication
+  please cite:
+  
   @article{Tange2011a,
-   title = {GNU Parallel - The Command-Line Power Tool},
+    title = {GNU Parallel - The Command-Line Power Tool},
-   author = {O. Tange},
+    author = {O. Tange},
-   address = {Frederiksberg, Denmark},
+    address = {Frederiksberg, Denmark},
-   journal = {;login: The USENIX Magazine},
+    journal = {;login: The USENIX Magazine},
-   month = {Feb},
+    month = {Feb},
-   number = {1},
+    number = {1},
-   volume = {36},
+    volume = {36},
-   url = {http://www.gnu.org/s/parallel},
+    url = {http://www.gnu.org/s/parallel},
-   year = {2011},
+    year = {2011},
-   pages = {42-47}
+    pages = {42-47},
+    doi = {10.5281/zenodo.16303}
   }
   
+  (Feel free to use \nocite{Tange2011a})
+  
+  This helps funding further development; AND IT WON'T COST YOU A CENT.
+  If you pay 10000 EUR you should feel free to use GNU Parallel without citing.
+  
+  If you send a copy of your published article to tange@gnu.org, it will be
+  mentioned in the release notes of next version of GNU Parallel.
+  
 With B<--max-line-length-allowed> GNU B<parallel> will report the maximal
 size of the command line:
 
@@ -2648,9 +2694,9 @@ Profiles can be combined:
 
 Output:
 
-  \nice -n17 /bin/bash -c echo\ A
+  echo A
-  \nice -n17 /bin/bash -c echo\ B
+  echo B
-  \nice -n17 /bin/bash -c echo\ C
+  echo C
 
 
 =head1 Spread the word

testsuite/tests-to-run/parallel-local-0.3s.sh

@@ -271,6 +271,11 @@ echo 'bug #47086: [PATCH] Initialize total_completed from joblog'
 
 echo '**'
 
+echo 'bug #47290: xargs: Warning: a NUL character occurred in the input'
+
+  perl -e 'print "foo\0not printed"' | parallel echo
+
+echo '**'
 
 EOF
 echo '### 1 .par file from --files expected'

testsuite/tests-to-run/parallel-local-ssh4.sh

@@ -128,16 +128,7 @@ func_echo() {
   echo ${assocarr[a]}
   echo Funky-"$funky"-funky
 }
-
+. `which env_parallel.bash`
-env_parallel() {
-  export PARALLEL_ENV="$(echo "shopt -s expand_aliases 2>/dev/null"; alias;typeset -p |
-    grep -vFf <(readonly) |
-    grep -v 'declare .. (GROUPS|FUNCNAME|DIRSTACK|_|PIPESTATUS|USERNAME|BASH_[A-Z_]+) ';
-    typeset -f)";
-  `which parallel` "$@";
-  unset PARALLEL_ENV;
-}
-
 env_parallel alias_echo ::: alias_works
 env_parallel func_echo ::: function_works
 env_parallel -S lo alias_echo ::: alias_works_over_ssh
@@ -165,16 +156,8 @@ func_echo() {
   echo Funky-"$funky"-funky
 }
 
-env_parallel() {
+# alias does not work:
-  export PARALLEL_ENV="$(alias | perl -pe 's/^/alias /';typeset -p |
+#   http://unix.stackexchange.com/questions/223534/defining-an-alias-and-immediately-use-it
-    grep -aFvf <(typeset -pr)|egrep -iav 'ZSH_EVAL_CONTEXT|LINENO=| _=|aliases|^typeset [a-z_]+$'|
-    egrep -av '^(typeset IFS=|..$)|cyan';
-    typeset -f)";
-  parallel "$@";
-  unset PARALLEL_ENV;
-}
-
-# alias does not work: http://unix.stackexchange.com/questions/223534/defining-an-alias-and-immediately-use-it
 env_parallel alias_echo ::: alias_does_not_work
 env_parallel func_echo ::: function_works
 env_parallel -S zsh@lo alias_echo ::: alias_does_not_work_over_ssh
@@ -202,11 +185,6 @@ func_echo() {
   echo Funky-"$funky"-funky
 }
 
-env_parallel() {
-  export PARALLEL_ENV="$(alias | perl -pe 's/^/alias /';typeset -p|egrep -v 'typeset( -i)? -r|PIPESTATUS';typeset -f)";
-  `which parallel` "$@";
-  unset PARALLEL_ENV;
-}
 env_parallel alias_echo ::: alias_works
 env_parallel func_echo ::: function_works
 env_parallel -S ksh@lo alias_echo ::: alias_works_over_ssh
@@ -218,9 +196,6 @@ EOS
 echo
 echo "### Fish environment"
 stdout ssh -q fish@lo <<'EOS' | egrep -v 'Welcome to |packages can be updated|security updates'
-# All variables cannot reliably be exported
-# perl -e 'print map { "setenv///$_///$ENV{$_}\n"} grep !/^(PWD|SHLVL|PATH)$/, keys %ENV'| sh -c 'parallel --shellquote' | perl -pe 's:///: :g' |fish
-
 set myvar "myvar  works"
 setenv myenvvar "myenvvar  works"
 
@@ -238,8 +213,7 @@ function func_echo
   echo $argv;
   echo "$myvar"
   echo "$myenvvar"
-# Arrays are not exported
+  echo $myarray[2]
-#  echo $myarray[2]
 # Assoc arrays do not exist in fish
 #  echo ${assocarr[a]}
   echo
@@ -252,12 +226,6 @@ function func_echo
   echo
 end
 
-function env_parallel
-  setenv PARALLEL_ENV (begin; functions -n | perl -pe 's/,/\n/g' | while read d; functions $d; end; perl -e 'print map { "$_///$ENV{$_}\n"} grep !/^(PWD|SHLVL|PATH)$/, keys %ENV'| sh -c 'parallel --shellquote' | perl -pe 's:^([^/]+)///:setenv $1 :'; end |perl -pe 's/\001/\\cb/g;s/\n/\001/')
-  parallel $argv;
-  set -e PARALLEL_ENV
-end
-
 env_parallel alias_echo ::: alias_works
 env_parallel func_echo ::: function_works
 env_parallel -S fish@lo alias_echo ::: alias_works_over_ssh
@@ -322,7 +290,7 @@ alias alias_echo_var 'echo $argv; echo $myvar; echo ${myarray[2]}; echo Funky-"$
 #   s/^/\001alias /;
 #   Quoted: s/\^/\\001alias\ /\;
 
-alias env_parallel 'setenv PARALLEL_ENV "`alias | perl -pe s/\\047/\\047\\042\\047\\042\\047/g\;s/\^\(\\S+\)\(\\s+\)\\\(\(.\*\)\\\)/\\1\\2\\3/\;s/\^\(\\S+\)\(\\s+\)\(.\*\)/\\1\\2\\047\\3\\047/\;s/\^/\\001alias\ /\;s/\\\!/\\\\\\\!/g;`";parallel \!*; setenv PARALLEL_ENV'
+#!# alias env_parallel 'setenv PARALLEL_ENV "`alias | perl -pe s/\\047/\\047\\042\\047\\042\\047/g\;s/\^\(\\S+\)\(\\s+\)\\\(\(.\*\)\\\)/\\1\\2\\3/\;s/\^\(\\S+\)\(\\s+\)\(.\*\)/\\1\\2\\047\\3\\047/\;s/\^/\\001alias\ /\;s/\\\!/\\\\\\\!/g;`";parallel \!*; setenv PARALLEL_ENV'
 
 
 ##  set tmpfile=`tempfile`

testsuite/wanted-results/parallel-install

@@ -175,7 +175,7 @@ pod2pdf --output-file ./parallel_design.pdf ./parallel_design.pod --title "GNU P
 Warning: pod2pdf not found. Using old parallel_design.pdf
 make[0]: Entering directory `/tmp/parallel-00000000/src'
  /bin/mkdir -p '/usr/local/bin'
- /usr/bin/install -c parallel sql niceload env_parallel env_parallel.bash env_parallel.zsh env_parallel.fish env_parallel.ksh '/usr/local/bin'
+ /usr/bin/install -c parallel sql niceload env_parallel env_parallel.bash env_parallel.zsh env_parallel.fish env_parallel.ksh env_parallel.pdksh env_parallel.csh '/usr/local/bin'
 make  install-exec-hook
 make[0]: Entering directory `/tmp/parallel-00000000/src'
 rm /usr/local/bin/sem || true

testsuite/wanted-results/parallel-local-0.3s

@@ -588,5 +588,14 @@ parallel: This job failed:
 echo 3;exit 1
 echo '**'
 **
+echo 'bug #47290: xargs: Warning: a NUL character occurred in the input'
+bug #47290: xargs: Warning: a NUL character occurred in the input
+  perl -e 'print "foo\0not printed"' | parallel echo
+foo
+parallel: Warning: a NUL character occurred in the input.
+parallel: Warning: It cannot be passed through in the argument list.
+parallel: Warning: Did you mean to use the --null option?
+echo '**'
+**
 ### 1 .par file from --files expected
 1

testsuite/wanted-results/parallel-local-ssh4

@@ -143,7 +143,7 @@ Funky-
  * Documentation:  http://www.linuxmint.com
 
 
-zsh:129: command not found: alias_echo
+zsh:130: command not found: alias_echo
 function_works
 myvar  works
 array_val2
@@ -151,7 +151,7 @@ assoc_val_a
 Funky-
 	
 
 !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~€亗儎厗噲墛媽崕彁憭摂晼棙櫄洔潪煚、￥ウЖ┆<D096><E29486><EFBFBD>辈炒刀犯购患骄坷谅媚牌侨墒颂臀闲岩釉罩棕仝圮蒉哙徕沅彐玷殛腱眍镳耱篝貊鼬<E8B28A><E9BCAC><EFBFBD><EFBFBD>-funky
-zsh:129: command not found: alias_echo
+zsh:130: command not found: alias_echo
 function_works_over_ssh
 myvar  works
 array_val2
@@ -191,26 +191,36 @@ Funky-

  * Documentation:  http://www.linuxmint.com
 
 
+env_parallel: Warning: ASCII value 1 in variables is not supported
+env_parallel: Warning: ASCII value 1 in variables is not supported
 3 arg alias_works
+env_parallel: Warning: ASCII value 1 in variables is not supported
+env_parallel: Warning: ASCII value 1 in variables is not supported
 function_works
-
+myvar  works
 myenvvar  works
+3
 
 
 
-Funky--funky
+Funky-	 
  !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~€亗儎厗噲墛媽崕彁憭摂晼棙櫄洔潪煚、￥ウЖ┆<D096><E29486><EFBFBD>辈炒刀犯购患骄坷谅媚牌侨墒颂臀闲岩釉罩棕仝圮蒉哙徕沅彐玷殛腱眍镳耱篝貊鼬<E8B28A><E9BCAC><EFBFBD><EFBFBD>-funky
 Funky-	 
  !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~€亗儎厗噲墛媽崕彁憭摂晼棙櫄洔潪煚、￥ウЖ┆<D096><E29486><EFBFBD>辈炒刀犯购患骄坷谅媚牌侨墒颂臀闲岩釉罩棕仝圮蒉哙徕沅彐玷殛腱眍镳耱篝貊鼬<E8B28A><E9BCAC><EFBFBD><EFBFBD>-funky
 
 
 
+env_parallel: Warning: ASCII value 1 in variables is not supported
+env_parallel: Warning: ASCII value 1 in variables is not supported
 3 arg alias_works_over_ssh
+env_parallel: Warning: ASCII value 1 in variables is not supported
+env_parallel: Warning: ASCII value 1 in variables is not supported
 function_works_over_ssh
-
+myvar  works
 myenvvar  works
+3
 
 
 
-Funky--funky
+Funky-	 
  !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~€亗儎厗噲墛媽崕彁憭摂晼棙櫄洔潪煚、￥ウЖ┆<D096><E29486><EFBFBD>辈炒刀犯购患骄坷谅媚牌侨墒颂臀闲岩釉罩棕仝圮蒉哙徕沅彐玷殛腱眍镳耱篝貊鼬<E8B28A><E9BCAC><EFBFBD><EFBFBD>-funky
 Funky-	 
  !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~€亗儎厗噲墛媽崕彁憭摂晼棙櫄洔潪煚、￥ウЖ┆<D096><E29486><EFBFBD>辈炒刀犯购患骄坷谅媚牌侨墒颂臀闲岩釉罩棕仝圮蒉哙徕沅彐玷殛腱眍镳耱篝貊鼬<E8B28A><E9BCAC><EFBFBD><EFBFBD>-funky
 
 

testsuite/wanted-results/parallel-tutorial

@@ -704,6 +704,10 @@ bar
   parallel --nonall --tag -S $SERVER1,$SERVER2 echo foo bar
 csh@lo	foo bar
 parallel@lo	foo bar
+  source `which env_parallel.bash`
+  alias myecho=echo
+  myvar="Joe's var is"
+  env_parallel -S $SERVER1 'myecho $myvar' ::: green
   MYVAR='foo bar'
   export MYVAR
   parallel --env MYVAR -S $SERVER1 echo '$MYVAR' ::: baz
@@ -815,9 +819,9 @@ _
   parallel --env _ -S $SERVER1 'echo $VAR; my_func2' ::: bar
 
 /bin/bash: my_func2: command not found
-  parallel -vv -S $SERVER1 echo ::: bar
+  parallel -vv --pipepart wc :::: num30000
-ssh -l parallel lo -- exec perl -e \''@GNU_Parallel=("use","IPC::Open3;","use","MIME::Base64");eval"@GNU_Parallel";my$eval=decode_base64(join"",@ARGV);eval$eval;'\' BASE64;
+<num30000 perl -e 'while(@ARGV) { sysseek(STDIN,shift,0) || die; $left = shift; while($read = sysread(STDIN,$buf, ($left > 131072 ? 131072 : $left))){ $left -= $read; syswrite(STDOUT,$buf); } }'  0 0 0 168894 | (wc)
-bar
+  30000   30000  168894
   my_func3() {
     echo in my_func $1 > $1.out
   }
@@ -854,6 +858,24 @@ Seq,Host,Starttime,JobRuntime,Send,Receive,Exitval,_Signal,Command,V1,V2,Stdout,
 13
 14
 ",
+  parallel --results outdir --sqlandworker csv:////%2Ftmp%2Flog2.csv seq ::: 10 ::: 12 13 14
+  cat /tmp/log2.csv
+10
+11
+12
+10
+11
+12
+13
+10
+11
+12
+13
+14
+Seq,Host,Starttime,JobRuntime,Send,Receive,Exitval,_Signal,Command,V1,V2,Stdout,Stderr
+1,:,000000000.000,0.000,0,9,0,0,"seq 10 12",10,12,outdir/1/10/2/12/stdout,outdir/1/10/2/12/stderr
+2,:,000000000.000,0.000,0,12,0,0,"seq 10 13",10,13,outdir/1/10/2/13/stdout,outdir/1/10/2/13/stderr
+3,:,000000000.000,0.000,0,15,0,0,"seq 10 14",10,14,outdir/1/10/2/14/stdout,outdir/1/10/2/14/stderr
   vendor://[[user][:password]@][host][:port]/[database[/table]
 /bin/bash: vendor://[[user][:password]@][host][:port]/[database[/table]: No such file or directory
   sqlite3:///%2Ftmp%2Fmydatabase/mytable
@@ -1205,6 +1227,18 @@ please cite:
 This helps funding further development; AND IT WON'T COST YOU A CENT.
 If you pay 10000 EUR you should feel free to use GNU Parallel without citing.
 
+  parallel --version
+GNU parallel VERSION
+Copyright (C) 2007,2008,2009,2010,2011,2012,2013,2014,2015,2016
+Ole Tange and Free Software Foundation, Inc.
+License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
+This is free software: you are free to change and redistribute it.
+GNU parallel comes with no warranty.
+
+Web site: http://www.gnu.org/software/parallel
+
+When using programs that use GNU Parallel to process data for publication
+please cite as described in 'parallel --bibtex'.
   parallel --minversion VERSION && echo Your version is at least VERSION.
 VERSION
 Your version is at least VERSION.