From f38d7fe29f4d79c0ff22aa9af61f90604e85e203 Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Fri, 22 Jul 2016 20:39:12 +0200 Subject: [PATCH] testsuite: env_parallel expanded. --- doc/release_new_version | 43 +- src/env_parallel.bash | 12 +- src/env_parallel.csh | 10 +- src/env_parallel.fish | 9 +- src/env_parallel.ksh | 12 +- src/env_parallel.pdksh | 10 +- src/env_parallel.pod | 536 ++++++++++--------- src/env_parallel.tcsh | 10 +- src/env_parallel.zsh | 16 +- src/parallel | 2 +- src/parallel.pod | 76 ++- testsuite/tests-to-run/parallel-local-30s.sh | 2 +- testsuite/wanted-results/parallel-local-ssh7 | 16 +- 13 files changed, 431 insertions(+), 323 deletions(-) diff --git a/doc/release_new_version b/doc/release_new_version index 3fa2332d..047a230f 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -219,29 +219,26 @@ cc:Tim Cuthbertson , Ryoichiro Suzuki , Jesse Alama -Subject: GNU Parallel 20160622 ('Orlando') released <<[stable]>> +Subject: GNU Parallel 20160722 ('Brexit') released <<[stable]>> -GNU Parallel 20160622 ('Orlando') <<[stable]>> has been released. It is available for download at: http://ftp.gnu.org/gnu/parallel/ +GNU Parallel 20160722 ('Brexit') <<[stable]>> has been released. It is available for download at: http://ftp.gnu.org/gnu/parallel/ <> Haiku of the month: - Does path on remote - not include GNU Parallel? - Try --env PATH. + + <<>> -- Ole Tange New in this release: -* $PATH can now be exported using --env PATH. Useful if GNU Parallel is not in your path on remote machines. +* env_parallel is now ready for wider testing. It is still beta quality. -* If --block is left out, --pipepart will use a block size that will result in 10 jobs per jobslot. +* env_parallel is heavily modified for all shells and testing has been increased. -* The cookie from 2016-01-04 was won by Morgan Rodgers on the 2016-06-06 after 5 months. - -* Vote for GNU Parallel's community ad on https://meta.askubuntu.com/questions/14925/community-promotion-ads-2016/15046#15046 +* Selectively choosing what to export using --env now works for env_parallel (bash, csh, fish, ksh, pdksh, tcsh, zsh). * <> @@ -269,21 +266,31 @@ for Big Data Applications https://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumb * <> -* GNU Parallel was cited in: Genotypic Resistance Tests Sequences Reveal the Role of Marginalized Populations in HIV-1 Transmission in Switzerland http://www.nature.com/articles/srep27580 +* <> -* GNU Parallel was cited in: A Deep-Learning Approach for Operation of an Automated Realtime Flare Forecast https://arxiv.org/pdf/1606.01587.pdf +* <> -* Unix parallel: Populating all the USB sticks http://www.markhneedham.com/blog/2016/06/01/unix-parallel-populating-all-the-usb-sticks/ +* GNU Parallel was cited in: HybPiper: Extracting Coding Sequence and Introns for Phylogenetics from High-Throughput Sequencing Reads Using Target Enrichment http://www.bioone.org/doi/full/10.3732/apps.1600016 -* GNU Parallel is a fantastic utility https://www.dray.be/parallel/ +* GNU Parallel was cited in: StrAuto - Automation and Parallelization of STRUCTURE Analysis http://www.crypticlineage.net/download/strauto/strauto_doc.pdf -* Combining a large number of VCF files https://gtamazian.com/2016/06/16/combining-a-large-number-of-vcf-files/ +* GNU Parallel was cited in: Tools and techniques for computational reproducibility http://gigascience.biomedcentral.com/articles/10.1186/s13742-016-0135-4 -* Useful Scraping Techniques http://blog.apps.npr.org/2016/06/17/scraping-tips.html +* GNU Parallel was cited in: FlashPCA: fast sparse canonical correlation analysis of genomic data http://biorxiv.org/content/biorxiv/suppl/2016/04/06/047217.DC1/047217-1.pdf -* Tools of the trade http://onox.com.br/2015/05/tools-of-the-trade/#more-198 +* GNU Parallel was cited in: Computational Design of DNA-Binding Proteins http://link.springer.com/protocol/10.1007/978-1-4939-3569-7_16 -* Armslist scraper https://github.com/nprapps/armslist-scraper/ +* GNU Parallel was cited in: Math Indexer and Searcher under the Hood: Fine-tuning Query Expansion and Unification Strategies http://research.nii.ac.jp/ntcir/workshop/OnlineProceedings12/pdf/ntcir/MathIR/05-NTCIR12-MathIR-RuzickaM.pdf + +* GNU Parallel was cited in: The Evolution and Fate of Super-Chandrasekhar Mass White Dwarf Merger Remnants http://arxiv.org/pdf/1606.02300.pdf + +* GNU Parallel was cited in: Evaluation of Coastal Scatterometer Products https://mdc.coaps.fsu.edu/scatterometry/meeting/docs/2016/Thu_AM/coastal-poster.pdf + +* GNU Parallel was used in: https://github.com/splitice/bulkdnsblcheck + +* The iconv slurp misfeature http://www.openfusion.net/linux/iconv_slurp_misfeature + +* แบบว่า CPU เหลือ https://veer66.wordpress.com/2016/06/15/gnu-parallel/ * Bug fixes and man page updates. diff --git a/src/env_parallel.bash b/src/env_parallel.bash index 5e251674..aab5c236 100755 --- a/src/env_parallel.bash +++ b/src/env_parallel.bash @@ -52,12 +52,12 @@ env_parallel() { } if(grep { /^_$/ } @envvar) { if(not open(IN, "<", "$ENV{HOME}/.parallel/ignored_vars")) { - print STDERR "parallel: Error: ", - "Run \"parallel --record-env\" in a clean environment first.\n"; + print STDERR "parallel: Error: ", + "Run \"parallel --record-env\" in a clean environment first.\n"; } else { - chomp(@ignored_vars = ); - $vars = join "|",map { quotemeta $_ } @ignored_vars; - print $vars ? "($vars)" : "(,,nO,,VaRs,,)"; + chomp(@ignored_vars = ); + $vars = join "|",map { quotemeta $_ } "env_parallel", @ignored_vars; + print $vars ? "($vars)" : "(,,nO,,VaRs,,)"; } } ' -- "$@" @@ -87,7 +87,7 @@ env_parallel() { local _variable_NAMES="$(compgen -A variable | grep -E "^$_grep_REGEXP"\$ | grep -vE "^$_ignore_UNDERSCORE"\$ | grep -vFf <(readonly) | - egrep -v '^(BASHOPTS|BASHPID|EUID|GROUPS|FUNCNAME|DIRSTACK|_|PIPESTATUS|PPID|SHELLOPTS|UID|USERNAME|BASH_[A-Z_]+)$')" + grep -Ev '^(BASHOPTS|BASHPID|EUID|GROUPS|FUNCNAME|DIRSTACK|_|PIPESTATUS|PPID|SHELLOPTS|UID|USERNAME|BASH_[A-Z_]+)$')" local _list_variable_VALUES="typeset -p $_variable_NAMES" if [[ "$_variable_NAMES" = "" ]] ; then # no variables selected diff --git a/src/env_parallel.csh b/src/env_parallel.csh index d8fd3bda..9236d220 100755 --- a/src/env_parallel.csh +++ b/src/env_parallel.csh @@ -57,12 +57,12 @@ else } if(grep { /^_$/ } @envvar) { if(not open(IN, "<", "$ENV{HOME}/.parallel/ignored_vars")) { - print STDERR "parallel: Error: ", - "Run 'parallel --record-env' in a clean environment first.\n"; + print STDERR "parallel: Error: ", + "Run \"parallel --record-env\" in a clean environment first.\n"; } else { - chomp(@ignored_vars = ); - $vars = join "|",map { quotemeta $_ } @ignored_vars; - print $vars ? "($vars)" : "(,,nO,,VaRs,,)"; + chomp(@ignored_vars = ); + $vars = join "|",map { quotemeta $_ } "env_parallel", @ignored_vars; + print $vars ? "($vars)" : "(,,nO,,VaRs,,)"; } } 'EOF' diff --git a/src/env_parallel.fish b/src/env_parallel.fish index 078501d8..55db6d1f 100755 --- a/src/env_parallel.fish +++ b/src/env_parallel.fish @@ -46,7 +46,8 @@ function env_parallel begin; set _grep_REGEXP ( begin; - perl -e 'for(@ARGV){ + perl -e ' + for(@ARGV){ /^_$/ and $next_is_env = 0; $next_is_env and push @envvar, split/,/, $_; $next_is_env = /^--env$/; @@ -66,12 +67,12 @@ function env_parallel } if(grep { /^_$/ } @envvar) { if(not open(IN, "<", "$ENV{HOME}/.parallel/ignored_vars")) { - print STDERR "parallel: Error: ", + print STDERR "parallel: Error: ", "Run \"parallel --record-env\" in a clean environment first.\n"; } else { chomp(@ignored_vars = ); - $vars = join "|",map { quotemeta $_ } @ignored_vars; - print $vars ? "($vars)" : "(nO,VaRs)"; + $vars = join "|",map { quotemeta $_ } "env_parallel", @ignored_vars; + print $vars ? "($vars)" : "(,,nO,,VaRs,,)"; } } ' -- $argv; diff --git a/src/env_parallel.ksh b/src/env_parallel.ksh index 3d78dbf2..ea3b4ca3 100755 --- a/src/env_parallel.ksh +++ b/src/env_parallel.ksh @@ -52,12 +52,12 @@ env_parallel() { } if(grep { /^_$/ } @envvar) { if(not open(IN, "<", "$ENV{HOME}/.parallel/ignored_vars")) { - print STDERR "parallel: Error: ", - "Run \"parallel --record-env\" in a clean environment first.\n"; + print STDERR "parallel: Error: ", + "Run \"parallel --record-env\" in a clean environment first.\n"; } else { - chomp(@ignored_vars = ); - $vars = join "|",map { quotemeta $_ } @ignored_vars; - print $vars ? "($vars)" : "(,,nO,,VaRs,,)"; + chomp(@ignored_vars = ); + $vars = join "|",map { quotemeta $_ } "env_parallel", @ignored_vars; + print $vars ? "($vars)" : "(,,nO,,VaRs,,)"; } } ' -- "$@" @@ -86,7 +86,7 @@ env_parallel() { # Grep variable names _variable_NAMES="$(typeset +p | perl -pe 's/^typeset .. //' | grep -E "^$_grep_REGEXP"\$ | grep -vE "^$_ignore_UNDERSCORE"\$ | - egrep -v '^(PIPESTATUS)$')" + grep -Ev '^(PIPESTATUS)$')" _list_variable_VALUES="typeset -p $_variable_NAMES" if [[ "$_variable_NAMES" = "" ]] ; then # no variables selected diff --git a/src/env_parallel.pdksh b/src/env_parallel.pdksh index 4a92ef8e..1bd31373 100755 --- a/src/env_parallel.pdksh +++ b/src/env_parallel.pdksh @@ -52,12 +52,12 @@ env_parallel() { } if(grep { /^_$/ } @envvar) { if(not open(IN, "<", "$ENV{HOME}/.parallel/ignored_vars")) { - print STDERR "parallel: Error: ", - "Run \"parallel --record-env\" in a clean environment first.\n"; + print STDERR "parallel: Error: ", + "Run \"parallel --record-env\" in a clean environment first.\n"; } else { - chomp(@ignored_vars = ); - $vars = join "|",map { quotemeta $_ } @ignored_vars; - print $vars ? "($vars)" : "(,,nO,,VaRs,,)"; + chomp(@ignored_vars = ); + $vars = join "|",map { quotemeta $_ } "env_parallel", @ignored_vars; + print $vars ? "($vars)" : "(,,nO,,VaRs,,)"; } } ' -- "$@" diff --git a/src/env_parallel.pod b/src/env_parallel.pod index 3985ef95..e4126fbd 100644 --- a/src/env_parallel.pod +++ b/src/env_parallel.pod @@ -15,25 +15,36 @@ B [options for GNU Parallel] =head1 DESCRIPTION B is a shell function that exports the current -environment to GNU Parallel. +environment to GNU B. If the shell function is not loaded, a dummy script will be run instead that explains how to install the function. -B is alpha quality and not production ready. +B is beta quality and not production ready. + +B is 0.1 sec slower at startup than pure GNU +B, and takes up to 15 ms to start a job. =head2 Environment space -B only works if the size of the current environment is -smaller than the maximal length of a command and smaller than half of -the max if running remotely. E.g. The max size of Bash's command is -128 KB, so B will fail if B is bigger than -128 KB. Technically the limit is in execve(1) which IPC::open3 uses. +By default B will export all environment variables, +arrays, aliases, functions and shell options (see details for the +individual shells below). + +But this only works if the size of the current environment is smaller +than the maximal length of a command and smaller than half of the max +if running remotely. E.g. The max size of Bash's command is 128 KB, so +B will fail if B is bigger than 128 +KB. Technically the limit is in execve(1) which IPC::open3 uses. Bash completion functions are well-known for taking up well over 128 KB of environment space and the primary reason for causing B to fail. +Instead you can use B<--env> to specify which variables, arrays, +aliases and functions to export as this will only export those with +the given name. + =head1 OPTIONS @@ -93,6 +104,268 @@ E.g. by doing: =back + +=head2 csh + +B<--env> is supported to export only the variable, alias, function, or +array with the given name. Multiple B<--env>s can be given. + +B for B breaks B<$PARALLEL>, so do not use +B<$PARALLEL>. + +Installation + +Put this in $HOME/.cshrc: + + source `which env_parallel.csh` + +E.g. by doing: + + echo 'source `which env_parallel.csh`' >> $HOME/.cshrc + +=over 8 + +=item aliases + + alias myecho echo + env_parallel myecho ::: test + env_parallel -S server myecho ::: test + env_parallel --env myecho myecho ::: work + env_parallel --env myecho -S server myecho ::: work + +=item functions + +Not supported by B. + +=item variables + + set myvar=test + env_parallel echo "\$myvar" ::: test + env_parallel -S csh@server echo "\$myvar" ::: test + env_parallel --env myvar echo '$myvar' ::: work + env_parallel --env myvar -S server echo '$myvar' ::: work + + +=item arrays with no special chars + + set myarray=(foo bar baz) + env_parallel echo "\${myarray\[\{\}\]}" ::: 1 2 3 + env_parallel -S csh@server echo "\${myarray\[\{\}\]}" ::: 1 2 3 + env_parallel -k --env myarray echo \$'{myarray[{}]}' ::: 1 2 3 + env_parallel -k --env myarray -S server echo \$'{myarray[{}]}' ::: 1 2 3 + +=back + + +=head2 fish + +B<--env> is supported to export only the variable, alias, function, or +array with the given name. Multiple B<--env>s can be given. + +Installation + +Put this in $HOME/.config/fish/config.fish: + + source (which env_parallel.fish) + +E.g. by doing: + + echo 'source (which env_parallel.fish)' >> $HOME/.config/fish/config.fish + +=over 8 + +=item aliases + + alias myecho 'echo aliases' + env_parallel myecho ::: work + env_parallel -S server myecho ::: work + env_parallel --env myecho myecho ::: work + env_parallel --env myecho -S server myecho ::: work + +=item functions + + function myfunc + echo functions $argv + end + env_parallel myfunc ::: work + env_parallel -S server myfunc ::: work + env_parallel --env myfunc myfunc ::: work + env_parallel --env myfunc -S server myfunc ::: work + +=item variables + + set myvar variables + env_parallel echo '$myvar' ::: work + env_parallel -S server echo '$myvar' ::: work + env_parallel --env myvar echo '$myvar' ::: work + env_parallel --env myvar -S server echo '$myvar' ::: work + +=item arrays + + set myarray arrays work, too + env_parallel -k echo '$myarray[{}]' ::: 1 2 3 + env_parallel -k -S server echo '$myarray[{}]' ::: 1 2 3 + env_parallel -k --env myarray echo '$myarray[{}]' ::: 1 2 3 + env_parallel -k --env myarray -S server echo '$myarray[{}]' ::: 1 2 3 + +=back + + +=head2 ksh + +B<--env> is supported to export only the variable, alias, function, or +array with the given name. Multiple B<--env>s can be given. + +Installation + +Put this in $HOME/.kshrc: + + source `which env_parallel.ksh` + +E.g. by doing: + + echo 'source `which env_parallel.ksh`' >> $HOME/.kshrc + +=over 8 + +=item aliases + + alias myecho='echo aliases' + env_parallel myecho ::: work + env_parallel -S server myecho ::: work + env_parallel --env myecho myecho ::: work + env_parallel --env myecho -S server myecho ::: work + +=item functions + + myfunc() { echo functions $*; } + env_parallel myfunc ::: work + env_parallel -S server myfunc ::: work + env_parallel --env myfunc myfunc ::: work + env_parallel --env myfunc -S server myfunc ::: work + +=item variables + + myvar=variables + env_parallel echo '$myvar' ::: work + env_parallel -S server echo '$myvar' ::: work + env_parallel --env myvar echo '$myvar' ::: work + env_parallel --env myvar -S server echo '$myvar' ::: work + +=item arrays + + myarray=(arrays work, too) + env_parallel -k echo '${myarray[{}]}' ::: 0 1 2 + env_parallel -k -S server echo '${myarray[{}]}' ::: 0 1 2 + env_parallel -k --env myarray echo '${myarray[{}]}' ::: 0 1 2 + env_parallel -k --env myarray -S server echo '${myarray[{}]}' ::: 0 1 2 + +=back + + +=head2 pdksh + +B<--env> is supported to export only the variable, alias, function, or +array with the given name. Multiple B<--env>s can be given. + +Installation + +Put this in $HOME/.profile: + + source `which env_parallel.pdksh` + +E.g. by doing: + + echo 'source `which env_parallel.pdksh`' >> $HOME/.profile + +=over 8 + +=item aliases + + alias myecho="echo aliases"; + env_parallel myecho ::: work; + env_parallel -S server myecho ::: work; + env_parallel --env myecho myecho ::: work; + env_parallel --env myecho -S server myecho ::: work + +=item functions + + myfunc() { echo functions $*; }; + env_parallel myfunc ::: work; + env_parallel -S server myfunc ::: work; + env_parallel --env myfunc myfunc ::: work; + env_parallel --env myfunc -S server myfunc ::: work + +=item variables + + myvar=variables; + env_parallel echo "\$myvar" ::: work; + env_parallel -S server echo "\$myvar" ::: work; + env_parallel --env myvar echo "\$myvar" ::: work; + env_parallel --env myvar -S server echo "\$myvar" ::: work + +=item arrays + + myarray=(arrays work, too); + env_parallel -k echo "\${myarray[{}]}" ::: 0 1 2; + env_parallel -k -S server echo "\${myarray[{}]}" ::: 0 1 2; + env_parallel -k --env myarray echo "\${myarray[{}]}" ::: 0 1 2; + env_parallel -k --env myarray -S server echo "\${myarray[{}]}" ::: 0 1 2 + +=back + + +=head2 tcsh + +B<--env> is supported to export only the variable, alias, function, or +array with the given name. Multiple B<--env>s can be given. + +B for B breaks B<$PARALLEL>, so do not use +B<$PARALLEL>. + +Installation + +Put this in $HOME/.tcshrc: + + source `which env_parallel.tcsh` + +E.g. by doing: + + echo 'source `which env_parallel.tcsh`' >> $HOME/.tcshrc + +=over 8 + +=item aliases + + alias myecho 'echo aliases' + env_parallel myecho ::: work + env_parallel -S server myecho ::: work + env_parallel --env myecho myecho ::: work + env_parallel --env myecho -S server myecho ::: work + +=item functions + +Not supported by B. + +=item variables + + set myvar=variables + env_parallel echo '$myvar' ::: work + env_parallel -S server echo '$myvar' ::: work + env_parallel --env myvar echo '$myvar' ::: work + env_parallel --env myvar -S server echo '$myvar' ::: work + +=item arrays with no special chars + + set myarray=(arrays work, too) + env_parallel -k echo \$'{myarray[{}]}' ::: 1 2 3 + env_parallel -k -S server echo \$'{myarray[{}]}' ::: 1 2 3 + env_parallel -k --env myarray echo \$'{myarray[{}]}' ::: 1 2 3 + env_parallel -k --env myarray -S server echo \$'{myarray[{}]}' ::: 1 2 3 + +=back + + =head2 Zsh B<--env> is supported to export only the variable, alias, function, or @@ -144,248 +417,6 @@ E.g. by doing: =back -=head2 fish - -Installation - -Put this in $HOME/.config/fish/config.fish: - - source (which env_parallel.fish) - -E.g. by doing: - - echo 'source (which env_parallel.fish)' >> $HOME/.config/fish/config.fish - -=over 8 - -=item aliases - - alias myecho 'echo aliases' - env_parallel myecho ::: work - env_parallel -S server myecho ::: work - env_parallel --env myecho myecho ::: work - env_parallel --env myecho -S server myecho ::: work - -=item functions - - function myfunc - echo functions $argv - end - env_parallel myfunc ::: work - env_parallel -S server myfunc ::: work - env_parallel --env myfunc myfunc ::: work - env_parallel --env myfunc -S server myfunc ::: work - -=item variables - - set myvar variables - env_parallel echo '$myvar' ::: work - env_parallel -S server echo '$myvar' ::: work - env_parallel --env myvar echo '$myvar' ::: work - env_parallel --env myvar -S server echo '$myvar' ::: work - -=item arrays - - set myarray arrays work, too - env_parallel -k echo '$myarray[{}]' ::: 1 2 3 - env_parallel -k -S server echo '$myarray[{}]' ::: 1 2 3 - env_parallel -k --env myarray echo '$myarray[{}]' ::: 1 2 3 - env_parallel -k --env myarray -S server echo '$myarray[{}]' ::: 1 2 3 - -=back - -=head2 ksh - -Installation - -Put this in $HOME/.kshrc: - - source `which env_parallel.ksh` - -E.g. by doing: - - echo 'source `which env_parallel.ksh`' >> $HOME/.kshrc - -=over 8 - -=item aliases - - alias myecho='echo aliases' - env_parallel myecho ::: work - env_parallel -S server myecho ::: work - env_parallel --env myecho myecho ::: work - env_parallel --env myecho -S server myecho ::: work - -=item functions - - myfunc() { echo functions $*; } - env_parallel myfunc ::: work - env_parallel -S server myfunc ::: work - env_parallel --env myfunc myfunc ::: work - env_parallel --env myfunc -S server myfunc ::: work - -=item variables - - myvar=variables - env_parallel echo '$myvar' ::: work - env_parallel -S server echo '$myvar' ::: work - env_parallel --env myvar echo '$myvar' ::: work - env_parallel --env myvar -S server echo '$myvar' ::: work - -=item arrays - - myarray=(arrays work, too) - env_parallel -k echo '${myarray[{}]}' ::: 0 1 2 - env_parallel -k -S server echo '${myarray[{}]}' ::: 0 1 2 - env_parallel -k --env myarray echo '${myarray[{}]}' ::: 0 1 2 - env_parallel -k --env myarray -S server echo '${myarray[{}]}' ::: 0 1 2 - -=back - -=head2 pdksh - -Installation - -Put this in $HOME/.profile: - - source `which env_parallel.pdksh` - -E.g. by doing: - - echo 'source `which env_parallel.pdksh`' >> $HOME/.profile - -=over 8 - -=item aliases - - alias myecho="echo aliases"; - env_parallel myecho ::: work; - env_parallel -S server myecho ::: work; - env_parallel --env myecho myecho ::: work; - env_parallel --env myecho -S server myecho ::: work - -=item functions - - myfunc() { echo functions $*; }; - env_parallel myfunc ::: work; - env_parallel -S server myfunc ::: work; - env_parallel --env myfunc myfunc ::: work; - env_parallel --env myfunc -S server myfunc ::: work - -=item variables - - myvar=variables; - env_parallel echo "\$myvar" ::: work; - env_parallel -S server echo "\$myvar" ::: work; - env_parallel --env myvar echo "\$myvar" ::: work; - env_parallel --env myvar -S server echo "\$myvar" ::: work - -=item arrays - - myarray=(arrays work, too); - env_parallel -k echo "\${myarray[{}]}" ::: 0 1 2; - env_parallel -k -S server echo "\${myarray[{}]}" ::: 0 1 2; - env_parallel -k --env myarray echo "\${myarray[{}]}" ::: 0 1 2; - env_parallel -k --env myarray -S server echo "\${myarray[{}]}" ::: 0 1 2 - -=back - -=head2 csh - -B for B breaks B<$PARALLEL>, so do not use -B<$PARALLEL>. - -Installation - -Put this in $HOME/.cshrc: - - source `which env_parallel.csh` - -E.g. by doing: - - echo 'source `which env_parallel.csh`' >> $HOME/.cshrc - -=over 8 - -=item aliases - - alias myecho echo - env_parallel myecho ::: test - env_parallel -S server myecho ::: test - env_parallel --env myecho myecho ::: work - env_parallel --env myecho -S server myecho ::: work - -=item functions - -Not supported by B. - -=item variables - - set myvar=test - env_parallel echo "\$myvar" ::: test - env_parallel -S csh@server echo "\$myvar" ::: test - env_parallel --env myvar echo '$myvar' ::: work - env_parallel --env myvar -S server echo '$myvar' ::: work - - -=item arrays with no special chars - - set myarray=(foo bar baz) - env_parallel echo "\${myarray\[\{\}\]}" ::: 1 2 3 - env_parallel -S csh@server echo "\${myarray\[\{\}\]}" ::: 1 2 3 - env_parallel -k --env myarray echo \$'{myarray[{}]}' ::: 1 2 3 - env_parallel -k --env myarray -S server echo \$'{myarray[{}]}' ::: 1 2 3 - -=back - -=head2 tcsh - -B for B breaks B<$PARALLEL>, so do not use -B<$PARALLEL>. - -Installation - -Put this in $HOME/.tcshrc: - - source `which env_parallel.tcsh` - -E.g. by doing: - - echo 'source `which env_parallel.tcsh`' >> $HOME/.tcshrc - -=over 8 - -=item aliases - - alias myecho 'echo aliases' - env_parallel myecho ::: work - env_parallel -S server myecho ::: work - env_parallel --env myecho myecho ::: work - env_parallel --env myecho -S server myecho ::: work - -=item functions - -Not supported by B. - -=item variables - - set myvar=variables - env_parallel echo '$myvar' ::: work - env_parallel -S server echo '$myvar' ::: work - env_parallel --env myvar echo '$myvar' ::: work - env_parallel --env myvar -S server echo '$myvar' ::: work - -=item arrays with no special chars - - set myarray=(arrays work, too) - env_parallel -k echo \$'{myarray[{}]}' ::: 1 2 3 - env_parallel -k -S server echo \$'{myarray[{}]}' ::: 1 2 3 - env_parallel -k --env myarray echo \$'{myarray[{}]}' ::: 1 2 3 - env_parallel -k --env myarray -S server echo \$'{myarray[{}]}' ::: 1 2 3 - -=back - - =head1 EXIT STATUS @@ -530,7 +561,10 @@ B uses GNU B. =head1 SEE ALSO -B(1), B(1), B(1), B(1), B(1), -B(1) +B(1), + +B(1), B(1), B(1), B(1), B(1) B(1), +B(1). + =cut diff --git a/src/env_parallel.tcsh b/src/env_parallel.tcsh index e19a79af..3adf862d 100755 --- a/src/env_parallel.tcsh +++ b/src/env_parallel.tcsh @@ -57,12 +57,12 @@ else } if(grep { /^_$/ } @envvar) { if(not open(IN, "<", "$ENV{HOME}/.parallel/ignored_vars")) { - print STDERR "parallel: Error: ", - "Run 'parallel --record-env' in a clean environment first.\n"; + print STDERR "parallel: Error: ", + "Run \"parallel --record-env\" in a clean environment first.\n"; } else { - chomp(@ignored_vars = ); - $vars = join "|",map { quotemeta $_ } @ignored_vars; - print $vars ? "($vars)" : "(,,nO,,VaRs,,)"; + chomp(@ignored_vars = ); + $vars = join "|",map { quotemeta $_ } "env_parallel", @ignored_vars; + print $vars ? "($vars)" : "(,,nO,,VaRs,,)"; } } 'EOF' diff --git a/src/env_parallel.zsh b/src/env_parallel.zsh index aa8f0087..154fb7b5 100755 --- a/src/env_parallel.zsh +++ b/src/env_parallel.zsh @@ -52,12 +52,12 @@ env_parallel() { } if(grep { /^_$/ } @envvar) { if(not open(IN, "<", "$ENV{HOME}/.parallel/ignored_vars")) { - print STDERR "parallel: Error: ", - "Run \"parallel --record-env\" in a clean environment first.\n"; + print STDERR "parallel: Error: ", + "Run \"parallel --record-env\" in a clean environment first.\n"; } else { - chomp(@ignored_vars = ); - $vars = join "|",map { quotemeta $_ } @ignored_vars; - print $vars ? "($vars)" : "(,,nO,,VaRs,,)"; + chomp(@ignored_vars = ); + $vars = join "|",map { quotemeta $_ } "env_parallel", @ignored_vars; + print $vars ? "($vars)" : "(,,nO,,VaRs,,)"; } } ' -- "$@" @@ -86,11 +86,11 @@ env_parallel() { unset _function_NAMES # Grep variable names - # The egrep -v is crap and should be better + # The grep -Ev is crap and should be better _variable_NAMES="$(print -l ${(k)parameters} | grep -E "^$_grep_REGEXP"\$ | grep -vE "^$_ignore_UNDERSCORE"\$ | - egrep -v '^([-?#!$*@_0]|zsh_eval_context|ZSH_EVAL_CONTEXT|LINENO|IFS|commands|functions|options|aliases|EUID|EGID|UID|GID)$' | - egrep -v 'terminfo|funcstack|galiases|keymaps|parameters|jobdirs|dirstack|functrace|funcsourcetrace|zsh_scheduled_events|dis_aliases|dis_reswords|dis_saliases|modules|reswords|saliases|widgets|userdirs|historywords|nameddirs|termcap|dis_builtins|dis_functions|jobtexts|funcfiletrace|dis_galiases|builtins|history|jobstates' + grep -Ev '^([-?#!$*@_0]|zsh_eval_context|ZSH_EVAL_CONTEXT|LINENO|IFS|commands|functions|options|aliases|EUID|EGID|UID|GID)$' | + grep -Ev 'terminfo|funcstack|galiases|keymaps|parameters|jobdirs|dirstack|functrace|funcsourcetrace|zsh_scheduled_events|dis_aliases|dis_reswords|dis_saliases|modules|reswords|saliases|widgets|userdirs|historywords|nameddirs|termcap|dis_builtins|dis_functions|jobtexts|funcfiletrace|dis_galiases|builtins|history|jobstates' )" _list_variable_VALUES="typeset -p "$(echo $_variable_NAMES|xargs)" | grep -aFvf <(typeset -pr) diff --git a/src/parallel b/src/parallel index a0f80a54..c6c9ebb2 100755 --- a/src/parallel +++ b/src/parallel @@ -3734,7 +3734,7 @@ sub citation { " url = {http://www.gnu.org/s/parallel},", " year = {2011},", " pages = {42-47},", - " doi = {10.5281/zenodo.16303}", + " doi = {http://dx.doi.org/10.5281/zenodo.16303}", "}", "", "(Feel free to use \\nocite{Tange2011a})", diff --git a/src/parallel.pod b/src/parallel.pod index 2eaa8565..94c99881 100644 --- a/src/parallel.pod +++ b/src/parallel.pod @@ -266,6 +266,14 @@ shell quote a string number of jobs in total +=item Z<> B<$job->slot()> + +slot number of job + +=item Z<> B<$job->seq()> + +sequence number of job + =back Example: @@ -1476,7 +1484,7 @@ sequence numbers in B<--joblog> then the input, the command, and B<--joblog> all have to remain unchanged; otherwise GNU B may run wrong commands. -See also B<--joblog>, B<--results>, B<--resume-failed>. +See also B<--joblog>, B<--results>, B<--resume-failed>, B<--retries>. =item B<--resume-failed> @@ -1489,7 +1497,7 @@ numbers in B<--joblog> then the input, the command, and B<--joblog> all have to remain unchanged; otherwise GNU B may run wrong commands. -See also B<--joblog>, B<--resume>. +See also B<--joblog>, B<--resume>, B<--retry-failed>, B<--retries>. =item B<--retry-failed> @@ -1497,10 +1505,66 @@ See also B<--joblog>, B<--resume>. Retry all failed jobs in joblog. By reading B<--joblog> GNU B will figure out the failed jobs and run those again. -B<--retry-failed> ignore the command and arguments: It only looks at -the joblog. +B<--retry-failed> ignores the command and arguments on the command +line: It only looks at the joblog. -See also B<--joblog>, B<--resume>, B<--resume-failed>. + B + +In this example B will cause every other job to fail. + + timeout -k 1 4 parallel --joblog log -j10 'sleep {}; exit {= $_%=2 =}' ::: {10..1} + +4 jobs completed. 2 failed: + + Seq [...] Exitval Signal Command + 10 [...] 1 0 sleep 1; exit 1 + 9 [...] 0 0 sleep 2; exit 0 + 8 [...] 1 0 sleep 3; exit 1 + 7 [...] 0 0 sleep 4; exit 0 + +B<--resume> does not care about the Exitval, but only looks at Seq. If +the Seq is run, it will not be run again. So if needed, you can change +the command for the seqs not run yet: + + parallel --resume --joblog log -j10 'sleep .{}; exit {= $_%=2 =}' ::: {10..1} + + Seq [...] Exitval Signal Command + [... as above ...] + 1 [...] 0 0 sleep .10; exit 0 + 6 [...] 1 0 sleep .5; exit 1 + 5 [...] 0 0 sleep .6; exit 0 + 4 [...] 1 0 sleep .7; exit 1 + 3 [...] 0 0 sleep .8; exit 0 + 2 [...] 1 0 sleep .9; exit 1 + +B<--resume-failed> cares about the Exitval, but also only looks at Seq +to figure out which commands to run. Again this means you can change +the command, but not the arguments. It will run the failed seqs and +the seqs not yet run: + + parallel --resume-failed --joblog log -j10 'echo {};sleep .{}; exit {= $_%=3 =}' ::: {10..1} + + Seq [...] Exitval Signal Command + [... as above ...] + 10 [...] 1 0 echo 1;sleep .1; exit 1 + 8 [...] 0 0 echo 3;sleep .3; exit 0 + 6 [...] 2 0 echo 5;sleep .5; exit 2 + 4 [...] 1 0 echo 7;sleep .7; exit 1 + 2 [...] 0 0 echo 9;sleep .9; exit 0 + +B<--retry-failed> cares about the Exitval, but takes the command from +the joblog. It ignores any arguments or commands given on the command +line: + + parallel --retry-failed --joblog log -j10 this part is ignored + + Seq [...] Exitval Signal Command + [... as above ...] + 10 [...] 1 0 echo 1;sleep .1; exit 1 + 6 [...] 2 0 echo 5;sleep .5; exit 2 + 4 [...] 1 0 echo 7;sleep .7; exit 1 + +See also B<--joblog>, B<--resume>, B<--resume-failed>, B<--retries>. =item B<--retries> I @@ -1609,6 +1673,8 @@ Here are a few examples: --rpl '{0#} $f = "%0".int(1+log(total_jobs())/log(10))."d"; $_=sprintf($f,$job->seq())' Job sequence counting from 0 --rpl '{#0} $_ = $job->seq() - 1' + Job slot counting from 2 + --rpl '{%1} $_ = $job->slot() + 1' See also: B<{= perl expression =}> B<--parens> diff --git a/testsuite/tests-to-run/parallel-local-30s.sh b/testsuite/tests-to-run/parallel-local-30s.sh index 747d7142..5f299c63 100644 --- a/testsuite/tests-to-run/parallel-local-30s.sh +++ b/testsuite/tests-to-run/parallel-local-30s.sh @@ -15,7 +15,7 @@ par_tmp_full() { # Assume /tmp/shm is easy to fill up export SHM=/tmp/shm/parallel mkdir -p $SHM - sudo umount -l $SHM + sudo umount -l $SHM 2>/dev/null sudo mount -t tmpfs -o size=10% none $SHM echo "### Test --tmpdir running full. bug #40733 was caused by this" diff --git a/testsuite/wanted-results/parallel-local-ssh7 b/testsuite/wanted-results/parallel-local-ssh7 index b00442e6..8e4e448f 100644 --- a/testsuite/wanted-results/parallel-local-ssh7 +++ b/testsuite/wanted-results/parallel-local-ssh7 @@ -62,13 +62,13 @@ par_bash_underscore aliases in and arrays in functions work par_bash_underscore aliases in and arrays in functions work par_bash_underscore aliases in functions work par_bash_underscore aliases in functions work -par_bash_underscore /bin/bash: line 139: myecho: command not found +par_bash_underscore /bin/bash: line 70: myecho: command not found par_bash_underscore OK if no myecho ^^^^^^^^^^^^^^^^^^^^^^^^^ -par_bash_underscore /bin/bash: line 139: myecho: command not found +par_bash_underscore /bin/bash: line 70: myecho: command not found par_bash_underscore OK if no myecho ^^^^^^^^^^^^^^^^^^^^^^^^^ -par_bash_underscore /bin/bash: line 136: myfunc: command not found +par_bash_underscore /bin/bash: line 68: myfunc: command not found par_bash_underscore OK if no myfunc ^^^^^^^^^^^^^^^^^^^^^^^^^ -par_bash_underscore /bin/bash: line 136: myfunc: command not found +par_bash_underscore /bin/bash: line 68: myfunc: command not found par_bash_underscore OK if no myfunc ^^^^^^^^^^^^^^^^^^^^^^^^^ par_csh_funky 3 arg alias_works par_csh_funky myvar works @@ -274,13 +274,13 @@ par_ksh_underscore aliases in and arrays in functions work par_ksh_underscore aliases in and arrays in functions work par_ksh_underscore aliases in functions work par_ksh_underscore aliases in functions work -par_ksh_underscore /usr/bin/ksh[193]: myecho: not found [No such file or directory] +par_ksh_underscore /usr/bin/ksh[113]: myecho: not found [No such file or directory] par_ksh_underscore OK if no myecho ^^^^^^^^^^^^^^^^^ -par_ksh_underscore /usr/bin/ksh[193]: myecho: not found [No such file or directory] +par_ksh_underscore /usr/bin/ksh[113]: myecho: not found [No such file or directory] par_ksh_underscore OK if no myecho ^^^^^^^^^^^^^^^^^ -par_ksh_underscore /usr/bin/ksh: line 193: myfunc: not found +par_ksh_underscore /usr/bin/ksh: line 113: myfunc: not found par_ksh_underscore OK if no myfunc ^^^^^^^^^^^^^^^^^ -par_ksh_underscore /usr/bin/ksh: line 193: myfunc: not found +par_ksh_underscore /usr/bin/ksh: line 113: myfunc: not found par_ksh_underscore OK if no myfunc ^^^^^^^^^^^^^^^^^ par_pdksh_funky 3 arg alias_works par_pdksh_funky function_works