Fixed bug #46058: --shellquote broken in fish.

doc/release_new_version

@@ -212,9 +212,9 @@ cc:Tim Cuthbertson <tim3d.junk@gmail.com>,
    Ryoichiro Suzuki <ryoichiro.suzuki@gmail.com>,
    Jesse Alama <jesse.alama@gmail.com>
 
-Subject: GNU Parallel 20151022 ('<<>>') released <<[stable]>>
+Subject: GNU Parallel 20151022 ('Liquid Water') released <<[stable]>>
 
-GNU Parallel 20151022 ('<<>>') <<[stable]>> has been released. It is available for download at: http://ftp.gnu.org/gnu/parallel/
+GNU Parallel 20151022 ('Liquid Water') <<[stable]>> has been released. It is available for download at: http://ftp.gnu.org/gnu/parallel/
 
 <<No new functionality was introduced so this is a good candidate for a stable release.>>
 
@@ -240,6 +240,8 @@ http://www.researchgate.net/profile/Christoph_Junghans/publication/276178326_TAD
 
 * GNU Parallel is used in TraitAR: https://testpypi.python.org/pypi/traitar/0.1.4
 
+* GNU Parallel is used in youtube-dl-parallel: https://github.com/dlh/youtube-dl-parallel
+
 * Bug fixes and man page updates.
 
 GNU Parallel - For people who live life in the parallel lane.

src/niceload

@@ -600,9 +600,9 @@ sub verbose {
 
 sub sleep_for_recheck {
     my $self = shift;
-    if($self->{'recheck'} < 0.5) {
+    if($self->{'recheck'} < 0.01) {
-	# Never sleep less than 0.5 sec
+	# Never sleep less than 0.01 sec
-	$self->{'recheck'} = 0.5;
+	$self->{'recheck'} = 0.01;
     }
     if($self->verbose()) {
 	$self->{'recheck'} = int($self->{'recheck'}*100)/100;

src/parallel

@@ -1097,7 +1097,7 @@ sub parse_options {
 
 sub init_globals {
     # Defaults:
-    $Global::version = 20150923;
+    $Global::version = 20151001;
     $Global::progname = 'parallel';
     $Global::infinity = 2**31;
     $Global::debug = 0;
@@ -6455,7 +6455,7 @@ sub wrapped {
 	if($opt::shellquote) {
 	    # Prepend \echo (echo no-\ is wrong in csh)
 	    # and quote twice
-	    $command = "\\echo " .
+	    $command = "/bin/echo " .
 		::shell_quote_scalar(::shell_quote_scalar($command));
 	}
 	if($opt::cat) {

src/parallel.pod

@@ -3013,6 +3013,73 @@ output.
 If you have a lot of hosts use '-j0' to access more hosts in parallel.
 
 
+=head1 EXAMPLE: Using remote computers behind NAT wall
+
+If the workers are behind a NAT wall, you need some trickery to get to
+them.
+
+If you can B<ssh> to a jump host, and reach the workers from there,
+then the obvious solution would be this, but it B<does not work>:
+
+  parallel --ssh 'ssh jumphost ssh' -S host1 echo ::: DOES NOT WORK
+
+It does not work because the command is dequoted by B<ssh> twice where
+as GNU B<parallel> only expects it to be dequoted once.
+
+So instead put this in B<~/.ssh/config>:
+
+  Host host1 host2 host3
+    ProxyCommand ssh jumphost.domain nc -w 1 %h 22
+
+It requires B<nc(netcat)> to be installed on jumphost. With this you
+can simply:
+
+  parallel -S host1,host2,host3 echo ::: This does work
+
+=head2 No jumphost, but port forwards
+
+If there is no jumphost but each server has port 22 forwarded from the
+firewall (e.g. the firewall's port 22001 = port 22 on host1, 22002 = host2,
+22003 = host3) then you can use B<~/.ssh/config>:
+
+  Host host1.v
+    Port 22001
+  Host host2.v
+    Port 22002
+  Host host3.v
+    Port 22003
+  Host *.v
+    Hostname firewall
+
+And then use host{1..3}.v as normal hosts:
+
+  parallel -S host1.v,host2.v,host3.v echo ::: a b c
+
+=head2 No jumphost, no port forwards
+
+If ports cannot be forwarded, you need some sort of VPN to traverse
+the NAT-wall. TOR is one options for that, as it is very easy to get
+working.
+
+You need to install TOR and setup a hidden service. In B<torrc> put:
+
+  HiddenServiceDir /var/lib/tor/hidden_service/
+  HiddenServicePort 22 127.0.0.1:22
+
+Then start TOR: B</etc/init.d/tor restart>
+
+The TOR hostname is now in B</var/lib/tor/hidden_service/hostname> and
+is something similar to B<izjafdceobowklhz.onion>. Now you simply
+prepend B<torsocks> to B<ssh>:
+
+  parallel --ssh 'torsocks ssh' -S izjafdceobowklhz.onion \
+    -S zfcdaeiojoklbwhz.onion,auclucjzobowklhi.onion echo ::: a b c
+
+If not all hosts are accessible through TOR:
+
+  parallel -S 'torsocks ssh izjafdceobowklhz.onion,host2,host3' echo ::: a b c
+
+
 =head1 EXAMPLE: Parallelizing rsync
 
 B<rsync> is a great tool, but sometimes it will not fill up the
@@ -4166,6 +4233,39 @@ B<11> xapply -f '[ -f %1 ] && echo %1' List | ...
 B<11> parallel '[ -f {} ] && echo {}' < List | ...
 
 
+=head2 DIFFERENCES BETWEEN AIX apply AND GNU Parallel
+
+B<apply> can build command lines based on a template and arguments -
+very much like GNU B<parallel>. B<apply> does not run jobs in
+parallel. B<apply> does not use an argument separator (like B<:::>);
+instead the template must be the first argument.
+
+Here are the examples from
+https://www-01.ibm.com/support/knowledgecenter/ssw_aix_71/com.ibm.aix.cmds1/apply.htm
+
+1. To obtain results similar to those of the B<ls> command, enter:
+
+  apply echo *
+  parallel echo ::: *
+
+2. To compare the file named B<a1> to the file named B<b1>, and the
+file named B<a2> to the file named B<b2>, enter:
+
+  apply -2 cmp a1 b1 a2 b2
+  parallel -N2 cmp ::: a1 b1 a2 b2
+
+3. To run the B<who> command five times, enter:
+
+  apply -0 who 1 2 3 4 5
+  parallel -N0 who ::: 1 2 3 4 5
+
+4. To link all files in the current directory to the directory
+B</usr/joe>, enter:
+
+  apply 'ln %1 /usr/joe' *
+  parallel ln {} /usr/joe ::: *
+
+
 =head2 DIFFERENCES BETWEEN paexec AND GNU Parallel
 
 B<paexec> can run jobs in parallel on both the local and remote computers.
@@ -4669,4 +4769,3 @@ B<make>(1), B<pexec>(1), B<ppss>(1), B<xjobs>(1), B<prll>(1),
 B<dxargs>(1), B<mdm>(1)
 
 =cut
-

testsuite/tests-to-run/parallel-local-ssh4.sh

@@ -143,6 +143,8 @@ env_parallel alias_echo ::: alias_works
 env_parallel func_echo ::: function_works
 env_parallel -S lo alias_echo ::: alias_works_over_ssh
 env_parallel -S lo func_echo ::: function_works_over_ssh
+echo
+echo "$funky" | parallel --shellquote
 #EOS
 
 
@@ -178,6 +180,8 @@ env_parallel alias_echo ::: alias_does_not_work
 env_parallel func_echo ::: function_works
 env_parallel -S zsh@lo alias_echo ::: alias_does_not_work_over_ssh
 env_parallel -S zsh@lo func_echo ::: function_works_over_ssh
+echo
+echo "$funky" | parallel --shellquote
 EOS
 
 echo
@@ -208,6 +212,8 @@ env_parallel alias_echo ::: alias_works
 env_parallel func_echo ::: function_works
 env_parallel -S ksh@lo alias_echo ::: alias_works_over_ssh
 env_parallel -S ksh@lo func_echo ::: function_works_over_ssh
+echo
+echo "$funky" | parallel --shellquote
 EOS
 
 echo
@@ -257,6 +263,8 @@ env_parallel alias_echo ::: alias_works
 env_parallel func_echo ::: function_works
 env_parallel -S fish@lo alias_echo ::: alias_works_over_ssh
 env_parallel -S fish@lo func_echo ::: function_works_over_ssh
+echo 
+echo "$funky" | parallel --shellquote
 EOS
 
 echo 
@@ -265,6 +273,9 @@ echo "### csh environment"
 # makealias:
 #   alias quote     "/bin/sed -e 's/\\!/\\\\\!/g' -e 's/'\\\''/'\\\'\\\\\\\'\\\''/g' -e 's/^/'\''/' -e 's/"\$"/'\''/'"
 #   alias makealias "quote | /bin/sed 's/^/alias \!:1 /' \!:2*"
+#
+#   makealias_with_newline
+#   perl -e '$/=undef;$_=<>;s/\n/\\\n/g;s/\047/\047\042\047\042\047/g;print'
 
 stdout ssh -q csh@lo <<'EOS' | egrep -v 'Welcome to |packages can be updated.'
 set myvar = "myvar  works"
@@ -315,11 +326,51 @@ alias alias_echo_var 'echo $argv; echo $myvar; echo ${myarray[2]}; echo Funky-"$
 alias env_parallel 'setenv PARALLEL_ENV "`alias | perl -pe s/\\047/\\047\\042\\047\\042\\047/g\;s/\^\(\\S+\)\(\\s+\)\\\(\(.\*\)\\\)/\\1\\2\\3/\;s/\^\(\\S+\)\(\\s+\)\(.\*\)/\\1\\2\\047\\3\\047/\;s/\^/\\001alias\ /\;s/\\\!/\\\\\\\!/g;`";parallel \!*; setenv PARALLEL_ENV'
 
 
+##  set tmpfile=`tempfile`
+##  foreach v (`set | awk -e '{print $1}' |grep -v prompt2`);
+##  eval if'($?'$v' && ${#'$v'} <= 1) echo scalar'$v'="$'$v'"' >> $tmpfile;
+##  eval if'($?'$v' && ${#'$v'} > 1) echo array'$v'="$'$v'"' >> $tmpfile;
+##  end
+##  cat $tmpfile | parallel --shellquote | perl -pe 's/^scalar(\S+).=/set $1=/ or s/^array(\S+).=(.*)/set $1=($2)/ && s/\\ / /g;'; rm $tmpfile
+##  
+##  set tmpfile=`tempfile`
+##  foreach _vARnAmE (`set | awk -e '{print $1}' |grep -v prompt2`);
+##  eval if'($?'$_vARnAmE' && ${#'$_vARnAmE'} <= 1) echo scalar'$_vARnAmE'="$'$_vARnAmE'"' >> $tmpfile; eval if'($?'$_vARnAmE' && ${#'$_vARnAmE'} > 1) echo array'$_vARnAmE'="$'$_vARnAmE'"' >> $tmpfile;
+##  end
+##  cat $tmpfile | parallel --shellquote | perl -pe 's/^scalar(\S+).=/set $1=/ or s/^array(\S+).=(.*)/set $1=($2)/ && s/\\ / /g;'; rm $tmpfile; unset tmpfile
+##  
+##  #!/bin/csh
+##  
+##  set _tmpfile=`tempfile`;
+##  foreach _vARnAmE (`set | awk -e '{print $1}' |grep -Ev 'prompt2|_tmpfile'`);
+##  eval if'($?'$_vARnAmE' && ${#'$_vARnAmE'} <= 1) echo scalar'$_vARnAmE'="$'$_vARnAmE'"' >> $_tmpfile;
+##  eval if'($?'$_vARnAmE' && ${#'$_vARnAmE'} > 1) echo array'$_vARnAmE'="$'$_vARnAmE'"' >> $_tmpfile;
+##  end 
+##  setenv PARALLEL_ENV `cat $_tmpfile | parallel --shellquote | perl -pe 's/^scalar(\S+).=/set $1=/ or s/^array(\S+).=(.*)/set $1=($2)/ && s/\\ / /g; s/$/\001/';`
+##  rm $_tmpfile;
+##  unset _tmpfile
+##  
+##  setenv PARALLEL_ENV "$PARALLEL_ENV`alias | perl -pe s/\\047/\\047\\042\\047\\042\\047/g\;s/\^\(\\S+\)\(\\s+\)\\\(\(.\*\)\\\)/\\1\\2\\3/\;s/\^\(\\S+\)\(\\s+\)\(.\*\)/\\1\\2\\047\\3\\047/\;s/\^/\\001alias\ /\;s/\\\!/\\\\\\\!/g;`"
+##  parallel \!*
+##  setenv PARALLEL_ENV
+##  
+##  
+##  perl -e '$/=undef;$_=<>;s/\n/\\\\\n/g;s/\047/\047\042\047\042\047/g;print "eval \047$_\047"'
+##  
+##  foreach g (h i j)
+##  echo $g
+##  end
+##  
+##  
+
+
 env_parallel alias_echo ::: alias_works
 env_parallel alias_echo_var ::: alias_var_does_not_work
 env_parallel func_echo ::: function_does_not_work
 env_parallel -S csh@lo alias_echo ::: alias_works_over_ssh
 env_parallel -S csh@lo alias_echo_var ::: alias_var_does_not_work
 env_parallel -S csh@lo func_echo ::: function_does_not_work_over_ssh
+echo
+echo "$funky" | parallel --shellquote
 EOS
 

testsuite/wanted-results/parallel-local-ssh4

@@ -133,6 +133,9 @@ Funky-
 
 !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~
 €亗儎厗噲墛媽崕彁憭摂晼棙櫄洔潪煚、￥ウЖ┆<EFBFBD><EFBFBD><EFBFBD>辈炒刀犯购患骄坷谅媚牌侨墒颂臀闲岩釉罩棕仝圮蒉哙徕沅彐玷殛腱眍镳耱篝貊鼬<EFBFBD><EFBFBD><EFBFBD><EFBFBD>-funky
 
+
\\\\\\\\	
+\\\
\\\\\\\\\\\\\\ \!\"\#\$%\&\'\(\)\*+,-./0123456789:\;\<\=\>\?@ABCDEFGHIJKLMNOPQRSTUVWXYZ\[\\\]\^_\`abcdefghijklmnopqrstuvwxyz\{\|\}\~€―<C280>ソЫ<E382BD><D0AB>Ⅸ<EFBFBD>噂浬欺圭構蚕十申曾箪貼能表暴予禄兔喀媾彌拿杤歃<E69DA4>\｡\｢\｣\､\･\ｦ\ｧ\ｨ\ｩ\ｪ\ｫ\ｬ\ｭ\ｮ\ｯ\ｰ\ｱ\ｲ\ｳ\ｴ\ｵ\ｶ\ｷ\ｸ\ｹ\ｺ\ｻ\ｼ\ｽ\ｾ\ｿ\ﾀ\ﾁ\ﾂ\ﾃ\ﾄ\ﾅ\ﾆ\ﾇ\ﾈ\ﾉ\ﾊ\ﾋ\ﾌ\ﾍ\ﾎ\ﾏ\ﾐ\ﾑ\ﾒ\ﾓ\ﾔ\ﾕ\ﾖ\ﾗ\ﾘ\ﾙ\ﾚ\ﾛ\ﾜ\ﾝ\ﾞ\ﾟ\濬畚秉綵臀藹觸軆鐔饅鷭<E9A585><E9B7AD>偆砡<E58186><E7A0A1><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>纊犾<E7BA8A><E78ABE>\<5C>\<5C>
+
 ### Zsh environment
 
  * Documentation:  http://www.linuxmint.com
@@ -156,6 +159,9 @@ Funky-
 	
 
 !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~€亗儎厗噲墛媽崕彁憭摂晼棙櫄洔潪煚、￥ウЖ┆<D096><E29486><EFBFBD>辈炒刀犯购患骄坷谅媚牌侨墒颂臀闲岩釉罩棕仝圮蒉哙徕沅彐玷殛腱眍镳耱篝貊鼬<E8B28A><E9BCAC><EFBFBD><EFBFBD>-funky
 
+
\\\\\\\\	
+\\\
\\\\\\\\\\\\\\ \!\"\#\$%\&\'\(\)\*+,-./0123456789:\;\<\=\>\?@ABCDEFGHIJKLMNOPQRSTUVWXYZ\[\\\]\^_\`abcdefghijklmnopqrstuvwxyz\{\|\}\~€―<C280>ソЫ<E382BD><D0AB>Ⅸ<EFBFBD>噂浬欺圭構蚕十申曾箪貼能表暴予禄兔喀媾彌拿杤歃<E69DA4>\｡\｢\｣\､\･\ｦ\ｧ\ｨ\ｩ\ｪ\ｫ\ｬ\ｭ\ｮ\ｯ\ｰ\ｱ\ｲ\ｳ\ｴ\ｵ\ｶ\ｷ\ｸ\ｹ\ｺ\ｻ\ｼ\ｽ\ｾ\ｿ\ﾀ\ﾁ\ﾂ\ﾃ\ﾄ\ﾅ\ﾆ\ﾇ\ﾈ\ﾉ\ﾊ\ﾋ\ﾌ\ﾍ\ﾎ\ﾏ\ﾐ\ﾑ\ﾒ\ﾓ\ﾔ\ﾕ\ﾖ\ﾗ\ﾘ\ﾙ\ﾚ\ﾛ\ﾜ\ﾝ\ﾞ\ﾟ\濬畚秉綵臀藹觸軆鐔饅鷭<E9A585><E9B7AD>偆砡<E58186><E7A0A1><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>纊犾<E7BA8A><E78ABE>\<5C>\<5C>
+
 ### Ksh environment
 
  * Documentation:  http://www.linuxmint.com
@@ -177,6 +183,9 @@ assoc_val_a
 Funky-
	
 
 !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~€亗儎厗噲墛媽崕彁憭摂晼棙櫄洔潪煚、￥ウЖ┆<D096><E29486><EFBFBD>辈炒刀犯购患骄坷谅媚牌侨墒颂臀闲岩釉罩棕仝圮蒉哙徕沅彐玷殛腱眍镳耱篝貊鼬<E8B28A><E9BCAC><EFBFBD><EFBFBD>-funky
 
+
\\\\\\\\	
+\\\
\\\\\\\\\\\\\\ \!\"\#\$%\&\'\(\)\*+,-./0123456789:\;\<\=\>\?@ABCDEFGHIJKLMNOPQRSTUVWXYZ\[\\\]\^_\`abcdefghijklmnopqrstuvwxyz\{\|\}\~€―<C280>ソЫ<E382BD><D0AB>Ⅸ<EFBFBD>噂浬欺圭構蚕十申曾箪貼能表暴予禄兔喀媾彌拿杤歃<E69DA4>\｡\｢\｣\､\･\ｦ\ｧ\ｨ\ｩ\ｪ\ｫ\ｬ\ｭ\ｮ\ｯ\ｰ\ｱ\ｲ\ｳ\ｴ\ｵ\ｶ\ｷ\ｸ\ｹ\ｺ\ｻ\ｼ\ｽ\ｾ\ｿ\ﾀ\ﾁ\ﾂ\ﾃ\ﾄ\ﾅ\ﾆ\ﾇ\ﾈ\ﾉ\ﾊ\ﾋ\ﾌ\ﾍ\ﾎ\ﾏ\ﾐ\ﾑ\ﾒ\ﾓ\ﾔ\ﾕ\ﾖ\ﾗ\ﾘ\ﾙ\ﾚ\ﾛ\ﾜ\ﾝ\ﾞ\ﾟ\濬畚秉綵臀藹觸軆鐔饅鷭<E9A585><E9B7AD>偆砡<E58186><E7A0A1><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>纊犾<E7BA8A><E78ABE>\<5C>\<5C>
+
 ### Fish environment
 
  * Documentation:  http://www.linuxmint.com
@@ -208,6 +217,8 @@ Funky-	 
  !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJ
 
 
 
+
\\\\\\\\	\ \\\
\\\\\\\\\\\\\\ \ \!\"\#\$%\&\'\(\)\*+,-./0123456789:\;\<\=\>\?@ABCDEFGHIJKLMNOPQRSTUVWXYZ\[\\\]\^_\`abcdefghijklmnopqrstuvwxyz\{\|\}\~€―<C280>ソЫ<E382BD><D0AB>Ⅸ<EFBFBD>噂浬欺圭構蚕十申曾箪貼能表暴予禄兔喀媾彌拿杤歃<E69DA4>\｡\｢\｣\､\･\ｦ\ｧ\ｨ\ｩ\ｪ\ｫ\ｬ\ｭ\ｮ\ｯ\ｰ\ｱ\ｲ\ｳ\ｴ\ｵ\ｶ\ｷ\ｸ\ｹ\ｺ\ｻ\ｼ\ｽ\ｾ\ｿ\ﾀ\ﾁ\ﾂ\ﾃ\ﾄ\ﾅ\ﾆ\ﾇ\ﾈ\ﾉ\ﾊ\ﾋ\ﾌ\ﾍ\ﾎ\ﾏ\ﾐ\ﾑ\ﾒ\ﾓ\ﾔ\ﾕ\ﾖ\ﾗ\ﾘ\ﾙ\ﾚ\ﾛ\ﾜ\ﾝ\ﾞ\ﾟ\濬畚秉綵臀藹觸軆鐔饅鷭<E9A585><E9B7AD>偆砡<E58186><E7A0A1><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>纊犾<E7BA8A><E78ABE>\<5C>\<5C>
+
 ### csh environment
 
  * Documentation:  http://www.linuxmint.com
@@ -223,3 +234,4 @@ func_echo: Command not found.
 3 arg alias_works_over_ssh
 myvar: Undefined variable.
 func_echo: Command not found.
+
\\\\\\\\	\ \\\
\\\\\\\\\\\\\\ \!\"\#\$%\&\'\(\)\*+,-./0123456789:\;\<\=\>\?@ABCDEFGHIJKLMNOPQRSTUVWXYZ\[\\\]\^_\`abcdefghijklmnopqrstuvwxyz\{\|\}\~€―<C280>ソЫ<E382BD><D0AB>Ⅸ<EFBFBD>噂浬欺圭構蚕十申曾箪貼能表暴予禄兔喀媾彌拿杤歃<E69DA4>\｡\｢\｣\､\･\ｦ\ｧ\ｨ\ｩ\ｪ\ｫ\ｬ\ｭ\ｮ\ｯ\ｰ\ｱ\ｲ\ｳ\ｴ\ｵ\ｶ\ｷ\ｸ\ｹ\ｺ\ｻ\ｼ\ｽ\ｾ\ｿ\ﾀ\ﾁ\ﾂ\ﾃ\ﾄ\ﾅ\ﾆ\ﾇ\ﾈ\ﾉ\ﾊ\ﾋ\ﾌ\ﾍ\ﾎ\ﾏ\ﾐ\ﾑ\ﾒ\ﾓ\ﾔ\ﾕ\ﾖ\ﾗ\ﾘ\ﾙ\ﾚ\ﾛ\ﾜ\ﾝ\ﾞ\ﾟ\濬畚秉綵臀藹觸軆鐔饅鷭<E9A585><E9B7AD>偆砡<E58186><E7A0A1><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>纊犾<E7BA8A><E78ABE>\<5C>\<5C>