diff --git a/doc/release_new_version b/doc/release_new_version index 0f92c822..63028448 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -212,9 +212,9 @@ cc:Tim Cuthbertson , Ryoichiro Suzuki , Jesse Alama -Subject: GNU Parallel 20151022 ('<<>>') released <<[stable]>> +Subject: GNU Parallel 20151022 ('Liquid Water') released <<[stable]>> -GNU Parallel 20151022 ('<<>>') <<[stable]>> has been released. It is available for download at: http://ftp.gnu.org/gnu/parallel/ +GNU Parallel 20151022 ('Liquid Water') <<[stable]>> has been released. It is available for download at: http://ftp.gnu.org/gnu/parallel/ <> @@ -240,6 +240,8 @@ http://www.researchgate.net/profile/Christoph_Junghans/publication/276178326_TAD * GNU Parallel is used in TraitAR: https://testpypi.python.org/pypi/traitar/0.1.4 +* GNU Parallel is used in youtube-dl-parallel: https://github.com/dlh/youtube-dl-parallel + * Bug fixes and man page updates. GNU Parallel - For people who live life in the parallel lane. diff --git a/src/niceload b/src/niceload index b786ab89..fafac702 100755 --- a/src/niceload +++ b/src/niceload @@ -600,9 +600,9 @@ sub verbose { sub sleep_for_recheck { my $self = shift; - if($self->{'recheck'} < 0.5) { - # Never sleep less than 0.5 sec - $self->{'recheck'} = 0.5; + if($self->{'recheck'} < 0.01) { + # Never sleep less than 0.01 sec + $self->{'recheck'} = 0.01; } if($self->verbose()) { $self->{'recheck'} = int($self->{'recheck'}*100)/100; diff --git a/src/parallel b/src/parallel index fc40be93..79c21c65 100755 --- a/src/parallel +++ b/src/parallel @@ -1097,7 +1097,7 @@ sub parse_options { sub init_globals { # Defaults: - $Global::version = 20150923; + $Global::version = 20151001; $Global::progname = 'parallel'; $Global::infinity = 2**31; $Global::debug = 0; @@ -6455,7 +6455,7 @@ sub wrapped { if($opt::shellquote) { # Prepend \echo (echo no-\ is wrong in csh) # and quote twice - $command = "\\echo " . + $command = "/bin/echo " . ::shell_quote_scalar(::shell_quote_scalar($command)); } if($opt::cat) { diff --git a/src/parallel.pod b/src/parallel.pod index a4b09fcc..2655d2e5 100644 --- a/src/parallel.pod +++ b/src/parallel.pod @@ -3013,6 +3013,73 @@ output. If you have a lot of hosts use '-j0' to access more hosts in parallel. +=head1 EXAMPLE: Using remote computers behind NAT wall + +If the workers are behind a NAT wall, you need some trickery to get to +them. + +If you can B to a jump host, and reach the workers from there, +then the obvious solution would be this, but it B: + + parallel --ssh 'ssh jumphost ssh' -S host1 echo ::: DOES NOT WORK + +It does not work because the command is dequoted by B twice where +as GNU B only expects it to be dequoted once. + +So instead put this in B<~/.ssh/config>: + + Host host1 host2 host3 + ProxyCommand ssh jumphost.domain nc -w 1 %h 22 + +It requires B to be installed on jumphost. With this you +can simply: + + parallel -S host1,host2,host3 echo ::: This does work + +=head2 No jumphost, but port forwards + +If there is no jumphost but each server has port 22 forwarded from the +firewall (e.g. the firewall's port 22001 = port 22 on host1, 22002 = host2, +22003 = host3) then you can use B<~/.ssh/config>: + + Host host1.v + Port 22001 + Host host2.v + Port 22002 + Host host3.v + Port 22003 + Host *.v + Hostname firewall + +And then use host{1..3}.v as normal hosts: + + parallel -S host1.v,host2.v,host3.v echo ::: a b c + +=head2 No jumphost, no port forwards + +If ports cannot be forwarded, you need some sort of VPN to traverse +the NAT-wall. TOR is one options for that, as it is very easy to get +working. + +You need to install TOR and setup a hidden service. In B put: + + HiddenServiceDir /var/lib/tor/hidden_service/ + HiddenServicePort 22 127.0.0.1:22 + +Then start TOR: B + +The TOR hostname is now in B and +is something similar to B. Now you simply +prepend B to B: + + parallel --ssh 'torsocks ssh' -S izjafdceobowklhz.onion \ + -S zfcdaeiojoklbwhz.onion,auclucjzobowklhi.onion echo ::: a b c + +If not all hosts are accessible through TOR: + + parallel -S 'torsocks ssh izjafdceobowklhz.onion,host2,host3' echo ::: a b c + + =head1 EXAMPLE: Parallelizing rsync B is a great tool, but sometimes it will not fill up the @@ -4166,6 +4233,39 @@ B<11> xapply -f '[ -f %1 ] && echo %1' List | ... B<11> parallel '[ -f {} ] && echo {}' < List | ... +=head2 DIFFERENCES BETWEEN AIX apply AND GNU Parallel + +B can build command lines based on a template and arguments - +very much like GNU B. B does not run jobs in +parallel. B does not use an argument separator (like B<:::>); +instead the template must be the first argument. + +Here are the examples from +https://www-01.ibm.com/support/knowledgecenter/ssw_aix_71/com.ibm.aix.cmds1/apply.htm + +1. To obtain results similar to those of the B command, enter: + + apply echo * + parallel echo ::: * + +2. To compare the file named B to the file named B, and the +file named B to the file named B, enter: + + apply -2 cmp a1 b1 a2 b2 + parallel -N2 cmp ::: a1 b1 a2 b2 + +3. To run the B command five times, enter: + + apply -0 who 1 2 3 4 5 + parallel -N0 who ::: 1 2 3 4 5 + +4. To link all files in the current directory to the directory +B, enter: + + apply 'ln %1 /usr/joe' * + parallel ln {} /usr/joe ::: * + + =head2 DIFFERENCES BETWEEN paexec AND GNU Parallel B can run jobs in parallel on both the local and remote computers. @@ -4669,4 +4769,3 @@ B(1), B(1), B(1), B(1), B(1), B(1), B(1) =cut - diff --git a/testsuite/tests-to-run/parallel-local-ssh4.sh b/testsuite/tests-to-run/parallel-local-ssh4.sh index 11f37bac..c4cdf0c5 100644 --- a/testsuite/tests-to-run/parallel-local-ssh4.sh +++ b/testsuite/tests-to-run/parallel-local-ssh4.sh @@ -143,6 +143,8 @@ env_parallel alias_echo ::: alias_works env_parallel func_echo ::: function_works env_parallel -S lo alias_echo ::: alias_works_over_ssh env_parallel -S lo func_echo ::: function_works_over_ssh +echo +echo "$funky" | parallel --shellquote #EOS @@ -178,6 +180,8 @@ env_parallel alias_echo ::: alias_does_not_work env_parallel func_echo ::: function_works env_parallel -S zsh@lo alias_echo ::: alias_does_not_work_over_ssh env_parallel -S zsh@lo func_echo ::: function_works_over_ssh +echo +echo "$funky" | parallel --shellquote EOS echo @@ -208,6 +212,8 @@ env_parallel alias_echo ::: alias_works env_parallel func_echo ::: function_works env_parallel -S ksh@lo alias_echo ::: alias_works_over_ssh env_parallel -S ksh@lo func_echo ::: function_works_over_ssh +echo +echo "$funky" | parallel --shellquote EOS echo @@ -257,6 +263,8 @@ env_parallel alias_echo ::: alias_works env_parallel func_echo ::: function_works env_parallel -S fish@lo alias_echo ::: alias_works_over_ssh env_parallel -S fish@lo func_echo ::: function_works_over_ssh +echo +echo "$funky" | parallel --shellquote EOS echo @@ -265,6 +273,9 @@ echo "### csh environment" # makealias: # alias quote "/bin/sed -e 's/\\!/\\\\\!/g' -e 's/'\\\''/'\\\'\\\\\\\'\\\''/g' -e 's/^/'\''/' -e 's/"\$"/'\''/'" # alias makealias "quote | /bin/sed 's/^/alias \!:1 /' \!:2*" +# +# makealias_with_newline +# perl -e '$/=undef;$_=<>;s/\n/\\\n/g;s/\047/\047\042\047\042\047/g;print' stdout ssh -q csh@lo <<'EOS' | egrep -v 'Welcome to |packages can be updated.' set myvar = "myvar works" @@ -315,11 +326,51 @@ alias alias_echo_var 'echo $argv; echo $myvar; echo ${myarray[2]}; echo Funky-"$ alias env_parallel 'setenv PARALLEL_ENV "`alias | perl -pe s/\\047/\\047\\042\\047\\042\\047/g\;s/\^\(\\S+\)\(\\s+\)\\\(\(.\*\)\\\)/\\1\\2\\3/\;s/\^\(\\S+\)\(\\s+\)\(.\*\)/\\1\\2\\047\\3\\047/\;s/\^/\\001alias\ /\;s/\\\!/\\\\\\\!/g;`";parallel \!*; setenv PARALLEL_ENV' +## set tmpfile=`tempfile` +## foreach v (`set | awk -e '{print $1}' |grep -v prompt2`); +## eval if'($?'$v' && ${#'$v'} <= 1) echo scalar'$v'="$'$v'"' >> $tmpfile; +## eval if'($?'$v' && ${#'$v'} > 1) echo array'$v'="$'$v'"' >> $tmpfile; +## end +## cat $tmpfile | parallel --shellquote | perl -pe 's/^scalar(\S+).=/set $1=/ or s/^array(\S+).=(.*)/set $1=($2)/ && s/\\ / /g;'; rm $tmpfile +## +## set tmpfile=`tempfile` +## foreach _vARnAmE (`set | awk -e '{print $1}' |grep -v prompt2`); +## eval if'($?'$_vARnAmE' && ${#'$_vARnAmE'} <= 1) echo scalar'$_vARnAmE'="$'$_vARnAmE'"' >> $tmpfile; eval if'($?'$_vARnAmE' && ${#'$_vARnAmE'} > 1) echo array'$_vARnAmE'="$'$_vARnAmE'"' >> $tmpfile; +## end +## cat $tmpfile | parallel --shellquote | perl -pe 's/^scalar(\S+).=/set $1=/ or s/^array(\S+).=(.*)/set $1=($2)/ && s/\\ / /g;'; rm $tmpfile; unset tmpfile +## +## #!/bin/csh +## +## set _tmpfile=`tempfile`; +## foreach _vARnAmE (`set | awk -e '{print $1}' |grep -Ev 'prompt2|_tmpfile'`); +## eval if'($?'$_vARnAmE' && ${#'$_vARnAmE'} <= 1) echo scalar'$_vARnAmE'="$'$_vARnAmE'"' >> $_tmpfile; +## eval if'($?'$_vARnAmE' && ${#'$_vARnAmE'} > 1) echo array'$_vARnAmE'="$'$_vARnAmE'"' >> $_tmpfile; +## end +## setenv PARALLEL_ENV `cat $_tmpfile | parallel --shellquote | perl -pe 's/^scalar(\S+).=/set $1=/ or s/^array(\S+).=(.*)/set $1=($2)/ && s/\\ / /g; s/$/\001/';` +## rm $_tmpfile; +## unset _tmpfile +## +## setenv PARALLEL_ENV "$PARALLEL_ENV`alias | perl -pe s/\\047/\\047\\042\\047\\042\\047/g\;s/\^\(\\S+\)\(\\s+\)\\\(\(.\*\)\\\)/\\1\\2\\3/\;s/\^\(\\S+\)\(\\s+\)\(.\*\)/\\1\\2\\047\\3\\047/\;s/\^/\\001alias\ /\;s/\\\!/\\\\\\\!/g;`" +## parallel \!* +## setenv PARALLEL_ENV +## +## +## perl -e '$/=undef;$_=<>;s/\n/\\\\\n/g;s/\047/\047\042\047\042\047/g;print "eval \047$_\047"' +## +## foreach g (h i j) +## echo $g +## end +## +## + + env_parallel alias_echo ::: alias_works env_parallel alias_echo_var ::: alias_var_does_not_work env_parallel func_echo ::: function_does_not_work env_parallel -S csh@lo alias_echo ::: alias_works_over_ssh env_parallel -S csh@lo alias_echo_var ::: alias_var_does_not_work env_parallel -S csh@lo func_echo ::: function_does_not_work_over_ssh +echo +echo "$funky" | parallel --shellquote EOS diff --git a/testsuite/wanted-results/parallel-local-ssh4 b/testsuite/wanted-results/parallel-local-ssh4 index 2570d1f9..891ae249 100644 --- a/testsuite/wanted-results/parallel-local-ssh4 +++ b/testsuite/wanted-results/parallel-local-ssh4 @@ -133,6 +133,9 @@ Funky-  !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~ -funky +\\\\\\\\ +\ \ \ \\\\\\\\\\\\\\ \!\"\#\$%\&\'\(\)\*+,-./0123456789:\;\<\=\>\?@ABCDEFGHIJKLMNOPQRSTUVWXYZ\[\\\]\^_\`abcdefghijklmnopqrstuvwxyz\{\|\}\~\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\ + ### Zsh environment * Documentation: http://www.linuxmint.com @@ -156,6 +159,9 @@ Funky-   !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~-funky +\\\\\\\\ +\ \ \ \\\\\\\\\\\\\\ \!\"\#\$%\&\'\(\)\*+,-./0123456789:\;\<\=\>\?@ABCDEFGHIJKLMNOPQRSTUVWXYZ\[\\\]\^_\`abcdefghijklmnopqrstuvwxyz\{\|\}\~\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\ + ### Ksh environment * Documentation: http://www.linuxmint.com @@ -177,6 +183,9 @@ assoc_val_a Funky-  !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~-funky +\\\\\\\\ +\ \ \ \\\\\\\\\\\\\\ \!\"\#\$%\&\'\(\)\*+,-./0123456789:\;\<\=\>\?@ABCDEFGHIJKLMNOPQRSTUVWXYZ\[\\\]\^_\`abcdefghijklmnopqrstuvwxyz\{\|\}\~\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\ + ### Fish environment * Documentation: http://www.linuxmint.com @@ -208,6 +217,8 @@ Funky-   !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJ +\\\\\\\\ \ \ \ \ \\\\\\\\\\\\\\ \ \!\"\#\$%\&\'\(\)\*+,-./0123456789:\;\<\=\>\?@ABCDEFGHIJKLMNOPQRSTUVWXYZ\[\\\]\^_\`abcdefghijklmnopqrstuvwxyz\{\|\}\~\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\ + ### csh environment * Documentation: http://www.linuxmint.com @@ -223,3 +234,4 @@ func_echo: Command not found. 3 arg alias_works_over_ssh myvar: Undefined variable. func_echo: Command not found. +\\\\\\\\ \ \ \ \ \\\\\\\\\\\\\\ \!\"\#\$%\&\'\(\)\*+,-./0123456789:\;\<\=\>\?@ABCDEFGHIJKLMNOPQRSTUVWXYZ\[\\\]\^_\`abcdefghijklmnopqrstuvwxyz\{\|\}\~\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\