From 87815c2a91d94229bff2d54de23a16bba7fa83a1 Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Sun, 12 Feb 2017 21:48:15 +0100 Subject: [PATCH] env_parallel.{ash/dash/sh}: Added. env_parallel.{bash/zsh/ksh}: Make them more similar to sh. --- src/Makefile.am | 8 +- src/Makefile.in | 6 +- src/env_parallel.ash | 174 +++ src/env_parallel.bash | 133 +- src/env_parallel.dash | 174 +++ src/env_parallel.ksh | 127 +- src/env_parallel.pdksh | 127 +- src/env_parallel.pod | 132 ++ src/env_parallel.sh | 174 +++ src/env_parallel.zsh | 117 +- src/parallel | 2 +- src/parallel_tutorial.1 | 2801 --------------------------------------- testsuite/REQUIREMENTS | 2 +- 13 files changed, 1024 insertions(+), 2953 deletions(-) create mode 100644 src/env_parallel.ash create mode 100755 src/env_parallel.dash create mode 100644 src/env_parallel.sh delete mode 100644 src/parallel_tutorial.1 diff --git a/src/Makefile.am b/src/Makefile.am index 6f47b5da..ff37b262 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -1,6 +1,8 @@ -bin_SCRIPTS = parallel sql niceload parcat \ - env_parallel env_parallel.bash env_parallel.zsh env_parallel.fish \ - env_parallel.ksh env_parallel.pdksh env_parallel.csh env_parallel.tcsh +bin_SCRIPTS = parallel sql niceload parcat env_parallel \ + env_parallel.ash env_parallel.bash env_parallel.csh \ + env_parallel.dash env_parallel.fish env_parallel.ksh \ + env_parallel.pdksh env_parallel.sh env_parallel.tcsh \ + env_parallel.zsh install-exec-hook: rm $(DESTDIR)$(bindir)/sem || true diff --git a/src/Makefile.in b/src/Makefile.in index 2fa5e1c7..99a0271e 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -230,8 +230,10 @@ top_build_prefix = @top_build_prefix@ top_builddir = @top_builddir@ top_srcdir = @top_srcdir@ bin_SCRIPTS = parallel sql niceload parcat \ - env_parallel env_parallel.bash env_parallel.zsh env_parallel.fish \ - env_parallel.ksh env_parallel.pdksh env_parallel.csh env_parallel.tcsh + env_parallel env_parallel.bash env_parallel.csh \ + env_parallel.dash env_parallel.fish env_parallel.ksh \ + env_parallel.pdksh env_parallel.sh env_parallel.tcsh \ + env_parallel.zsh @DOCUMENTATION_TRUE@man_MANS = parallel.1 env_parallel.1 sem.1 sql.1 niceload.1 \ @DOCUMENTATION_TRUE@ parallel_tutorial.7 parallel_design.7 parallel_alternatives.7 \ diff --git a/src/env_parallel.ash b/src/env_parallel.ash new file mode 100644 index 00000000..e1f714b7 --- /dev/null +++ b/src/env_parallel.ash @@ -0,0 +1,174 @@ +#!/usr/bin/env ash + +# This file must be sourced in ash: +# +# . `which env_parallel.ash` +# +# after which 'env_parallel' works +# +# +# Copyright (C) 2017 +# Ole Tange and Free Software Foundation, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, see +# or write to the Free Software Foundation, Inc., 51 Franklin St, +# Fifth Floor, Boston, MA 02110-1301 USA + +env_parallel() { + # based on env_parallel.sh + + _names_of_ALIASES() { + alias | perl -pe 's/=.*//' + } + _bodies_of_ALIASES() { + alias "$@" | perl -pe 's/^/alias /' + } + _names_of_maybe_FUNCTIONS() { + set | perl -ne '/^(\S+)\(\)\{$/ and print "$1\n"' + } + _names_of_FUNCTIONS() { + # myfunc is a function + type `_names_of_maybe_FUNCTIONS` | + perl -ne '/^(\S+) is a function$/ and not $seen{$1}++ and print "$1\n"' + } + _bodies_of_FUNCTIONS() { + type "$@" | perl -ne '/^(\S+) is a function$/ or print' + } + _names_of_VARIABLES() { + # This may screw up if variables contain \n and = + set | perl -ne 's/^(\S+)=.*/$1/ and print;' + } + _bodies_of_VARIABLES() { + # Crappy typeset -p + for _i in "$@" + do + perl -e 'print @ARGV' "$_i=" + eval echo \"\$$_i\" | perl -e '$/=undef; $a=<>; chop($a); print $a' | + perl -pe 's/[\002-\011\013-\032\\\#\?\`\(\)\{\}\[\]\^\*\<\=\>\~\|\; \"\!\$\&\202-\377]/\\$&/go;'"s/'/\\\'/g; s/[\n]/'\\n'/go;"; + echo + done + } + _remove_bad_NAMES() { + # Do not transfer vars and funcs from env_parallel + grep -Ev '^(_names_of_ALIASES|_bodies_of_ALIASES|_names_of_maybe_FUNCTIONS|_names_of_FUNCTIONS|_bodies_of_FUNCTIONS|_names_of_VARIABLES|_bodies_of_VARIABLES|_remove_bad_NAMES|_prefix_PARALLEL_ENV|_get_ignored_VARS|_make_grep_REGEXP|_ignore_UNDERSCORE|_alias_NAMES|_list_alias_BODIES|_function_NAMES|_list_function_BODIES|_variable_NAMES|_list_variable_VALUES|_prefix_PARALLEL_ENV)$' | + # Filter names matching --env + grep -E "^$_grep_REGEXP"\$ | grep -vE "^$_ignore_UNDERSCORE"\$ | + # Vars set by /bin/sh + grep -Ev '^(_)$' + } + + _get_ignored_VARS() { + perl -e ' + for(@ARGV){ + $next_is_env and push @envvar, split/,/, $_; + $next_is_env=/^--env$/; + } + if(grep { /^_$/ } @envvar) { + if(not open(IN, "<", "$ENV{HOME}/.parallel/ignored_vars")) { + print STDERR "parallel: Error: ", + "Run \"parallel --record-env\" in a clean environment first.\n"; + } else { + chomp(@ignored_vars = ); + $vars = join "|",map { quotemeta $_ } "env_parallel", @ignored_vars; + print $vars ? "($vars)" : "(,,nO,,VaRs,,)"; + } + } + ' -- "$@" + } + + # Get the --env variables if set + # --env _ should be ignored + # and convert a b c to (a|b|c) + # If --env not set: Match everything (.*) + _make_grep_REGEXP() { + perl -e ' + for(@ARGV){ + /^_$/ and $next_is_env = 0; + $next_is_env and push @envvar, split/,/, $_; + $next_is_env = /^--env$/; + } + $vars = join "|",map { quotemeta $_ } @envvar; + print $vars ? "($vars)" : "(.*)"; + ' -- "$@" + } + + if which parallel | grep 'no parallel in' >/dev/null; then + echo 'env_parallel: Error: parallel must be in $PATH.' >&2 + return 1 + fi + if which parallel >/dev/null; then + true which on linux + else + echo 'env_parallel: Error: parallel must be in $PATH.' >&2 + return 1 + fi + + # Grep regexp for vars given by --env + _grep_REGEXP="`_make_grep_REGEXP \"$@\"`" + + # Deal with --env _ + _ignore_UNDERSCORE="`_get_ignored_VARS \"$@\"`" + + # --record-env + if perl -e 'exit grep { /^--record-env$/ } @ARGV' -- "$@"; then + true skip + else + (_names_of_ALIASES; + _names_of_FUNCTIONS; + _names_of_VARIABLES) | + cat > $HOME/.parallel/ignored_vars + return 0 + fi + + # Grep alias names + _alias_NAMES="`_names_of_ALIASES | _remove_bad_NAMES`" + _list_alias_BODIES="_bodies_of_ALIASES $_alias_NAMES" + if [ "$_alias_NAMES" = "" ] ; then + # no aliases selected + _list_alias_BODIES="true" + fi + unset _alias_NAMES + + # Grep function names + _function_NAMES="`_names_of_FUNCTIONS | _remove_bad_NAMES`" + _list_function_BODIES="_bodies_of_FUNCTIONS $_function_NAMES" + if [ "$_function_NAMES" = "" ] ; then + # no functions selected + _list_function_BODIES="true" + fi + unset _function_NAMES + + # Grep variable names + _variable_NAMES="`_names_of_VARIABLES | _remove_bad_NAMES`" + _list_variable_VALUES="_bodies_of_VARIABLES $_variable_NAMES" + if [ "$_variable_NAMES" = "" ] ; then + # no variables selected + _list_variable_VALUES="true" + fi + unset _variable_NAMES + + PARALLEL_ENV="` + $_list_alias_BODIES; + $_list_function_BODIES; + $_list_variable_VALUES; + `" + export PARALLEL_ENV + unset _list_alias_BODIES + unset _list_variable_VALUES + unset _list_function_BODIES + `which parallel` "$@"; + _parallel_exit_CODE=$? + unset PARALLEL_ENV; + return $_parallel_exit_CODE +} diff --git a/src/env_parallel.bash b/src/env_parallel.bash index a9f519be..d69becdd 100755 --- a/src/env_parallel.bash +++ b/src/env_parallel.bash @@ -28,23 +28,40 @@ env_parallel() { # env_parallel.bash - # Get the --env variables if set - # --env _ should be ignored - # and convert a b c to (a|b|c) - # If --env not set: Match everything (.*) - local _grep_REGEXP="$( - perl -e ' - for(@ARGV){ - /^_$/ and $next_is_env = 0; - $next_is_env and push @envvar, split/,/, $_; - $next_is_env = /^--env$/; - } - $vars = join "|",map { quotemeta $_ } @envvar; - print $vars ? "($vars)" : "(.*)"; - ' -- "$@" - )" - # Deal with --env _ - local _ignore_UNDERSCORE="$( + _names_of_ALIASES() { + compgen -a + } + _bodies_of_ALIASES() { + alias "$@" + } + _names_of_FUNCTIONS() { + compgen -A function + } + _bodies_of_FUNCTIONS() { + typeset -f "$@" + } + _names_of_VARIABLES() { + compgen -A variable + } + _bodies_of_VARIABLES() { + typeset -p "$@" + } + _remove_bad_NAMES() { + # Do not transfer vars and funcs from env_parallel + grep -Ev '^(_names_of_ALIASES|_bodies_of_ALIASES|_names_of_maybe_FUNCTIONS|_names_of_FUNCTIONS|_bodies_of_FUNCTIONS|_names_of_VARIABLES|_bodies_of_VARIABLES|_remove_bad_NAMES|_prefix_PARALLEL_ENV|_get_ignored_VARS|_make_grep_REGEXP|_ignore_UNDERSCORE|_alias_NAMES|_list_alias_BODIES|_function_NAMES|_list_function_BODIES|_variable_NAMES|_list_variable_VALUES|_prefix_PARALLEL_ENV)$' | + grep -E "^$_grep_REGEXP"\$ | grep -vE "^$_ignore_UNDERSCORE"\$ | + grep -vFf <(readonly) | + grep -Ev '^(BASHOPTS|BASHPID|EUID|GROUPS|FUNCNAME|DIRSTACK|_|PIPESTATUS|PPID|SHELLOPTS|UID|USERNAME|BASH_[A-Z_]+)$' + } + _prefix_PARALLEL_ENV() { + shopt 2>/dev/null | + perl -pe 's:\s+off:;: and s/^/shopt -u /; + s:\s+on:;: and s/^/shopt -s /; + s:;$:&>/dev/null;:'; + echo 'shopt -s expand_aliases &>/dev/null'; + } + + _get_ignored_VARS() { perl -e ' for(@ARGV){ $next_is_env and push @envvar, split/,/, $_; @@ -61,44 +78,74 @@ env_parallel() { } } ' -- "$@" - )" + } + + # Get the --env variables if set + # --env _ should be ignored + # and convert a b c to (a|b|c) + # If --env not set: Match everything (.*) + _make_grep_REGEXP() { + perl -e ' + for(@ARGV){ + /^_$/ and $next_is_env = 0; + $next_is_env and push @envvar, split/,/, $_; + $next_is_env = /^--env$/; + } + $vars = join "|",map { quotemeta $_ } @envvar; + print $vars ? "($vars)" : "(.*)"; + ' -- "$@" + } + + if which parallel | grep 'no parallel in' >/dev/null; then + echo 'env_parallel: Error: parallel must be in $PATH.' >&2 + return 1 + fi + if which parallel >/dev/null; then + true which on linux + else + echo 'env_parallel: Error: parallel must be in $PATH.' >&2 + return 1 + fi + + # Grep regexp for vars given by --env + _grep_REGEXP="`_make_grep_REGEXP \"$@\"`" + + # Deal with --env _ + _ignore_UNDERSCORE="`_get_ignored_VARS \"$@\"`" # --record-env - if ! perl -e 'exit grep { /^--record-env$/ } @ARGV' -- "$@"; then - (compgen -a; - compgen -A function; - compgen -A variable) | + if perl -e 'exit grep { /^--record-env$/ } @ARGV' -- "$@"; then + true skip + else + (_names_of_ALIASES; + _names_of_FUNCTIONS; + _names_of_VARIABLES) | cat > $HOME/.parallel/ignored_vars return 0 fi - + # Grep alias names - local _alias_NAMES="$(compgen -a | - grep -E "^$_grep_REGEXP"\$ | grep -vE "^$_ignore_UNDERSCORE"\$ )" - local _list_alias_BODIES="alias $_alias_NAMES" - if [[ "$_alias_NAMES" = "" ]] ; then + _alias_NAMES="`_names_of_ALIASES | _remove_bad_NAMES`" + _list_alias_BODIES="_bodies_of_ALIASES $_alias_NAMES" + if [ "$_alias_NAMES" = "" ] ; then # no aliases selected _list_alias_BODIES="true" fi unset _alias_NAMES # Grep function names - local _function_NAMES="$(compgen -A function | - grep -E "^$_grep_REGEXP"\$ | grep -vE "^$_ignore_UNDERSCORE"\$ )" - local _list_function_BODIES="typeset -f $_function_NAMES" - if [[ "$_function_NAMES" = "" ]] ; then + _function_NAMES="`_names_of_FUNCTIONS | _remove_bad_NAMES`" + _list_function_BODIES="_bodies_of_FUNCTIONS $_function_NAMES" + if [ "$_function_NAMES" = "" ] ; then # no functions selected _list_function_BODIES="true" fi unset _function_NAMES # Grep variable names - local _variable_NAMES="$(compgen -A variable | - grep -E "^$_grep_REGEXP"\$ | grep -vE "^$_ignore_UNDERSCORE"\$ | - grep -vFf <(readonly) | - grep -Ev '^(BASHOPTS|BASHPID|EUID|GROUPS|FUNCNAME|DIRSTACK|_|PIPESTATUS|PPID|SHELLOPTS|UID|USERNAME|BASH_[A-Z_]+)$')" - local _list_variable_VALUES="typeset -p $_variable_NAMES" - if [[ "$_variable_NAMES" = "" ]] ; then + _variable_NAMES="`_names_of_VARIABLES | _remove_bad_NAMES`" + _list_variable_VALUES="_bodies_of_VARIABLES $_variable_NAMES" + if [ "$_variable_NAMES" = "" ] ; then # no variables selected _list_variable_VALUES="true" fi @@ -106,15 +153,13 @@ env_parallel() { # Copy shopt (so e.g. extended globbing works) # But force expand_aliases as aliases otherwise do not work - export PARALLEL_ENV="$( - shopt 2>/dev/null | - perl -pe 's:\s+off:;: and s/^/shopt -u /; - s:\s+on:;: and s/^/shopt -s /; - s:;$:&>/dev/null;:'; - echo 'shopt -s expand_aliases &>/dev/null'; + PARALLEL_ENV="` + _prefix_PARALLEL_ENV $_list_alias_BODIES; + $_list_function_BODIES; $_list_variable_VALUES; - $_list_function_BODIES)"; + `" + export PARALLEL_ENV unset _list_alias_BODIES unset _list_variable_VALUES unset _list_function_BODIES diff --git a/src/env_parallel.dash b/src/env_parallel.dash new file mode 100755 index 00000000..a8b249d5 --- /dev/null +++ b/src/env_parallel.dash @@ -0,0 +1,174 @@ +#!/usr/bin/env dash + +# This file must be sourced in dash: +# +# . `which env_parallel.dash` +# +# after which 'env_parallel' works +# +# +# Copyright (C) 2017 +# Ole Tange and Free Software Foundation, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, see +# or write to the Free Software Foundation, Inc., 51 Franklin St, +# Fifth Floor, Boston, MA 02110-1301 USA + +env_parallel() { + # based on env_parallel.sh + + _names_of_ALIASES() { + alias | perl -pe 's/=.*//' + } + _bodies_of_ALIASES() { + alias "$@" | perl -pe 's/^/alias /' + } + _names_of_maybe_FUNCTIONS() { + set | perl -ne '/^(\S+)\(\)\{$/ and print "$1\n"' + } + _names_of_FUNCTIONS() { + # myfunc is a function + type `_names_of_maybe_FUNCTIONS` | + perl -ne '/^(\S+) is a function$/ and not $seen{$1}++ and print "$1\n"' + } + _bodies_of_FUNCTIONS() { + type "$@" | perl -ne '/^(\S+) is a function$/ or print' + } + _names_of_VARIABLES() { + # This may screw up if variables contain \n and = + set | perl -ne 's/^(\S+)=.*/$1/ and print;' + } + _bodies_of_VARIABLES() { + # Crappy typeset -p + for _i in "$@" + do + perl -e 'print @ARGV' "$_i=" + eval echo \"\$$_i\" | perl -e '$/=undef; $a=<>; chop($a); print $a' | + perl -pe 's/[\002-\011\013-\032\\\#\?\`\(\)\{\}\[\]\^\*\<\=\>\~\|\; \"\!\$\&\202-\377]/\\$&/go;'"s/'/\\\'/g; s/[\n]/'\\n'/go;"; + echo + done + } + _remove_bad_NAMES() { + # Do not transfer vars and funcs from env_parallel + grep -Ev '^(_names_of_ALIASES|_bodies_of_ALIASES|_names_of_maybe_FUNCTIONS|_names_of_FUNCTIONS|_bodies_of_FUNCTIONS|_names_of_VARIABLES|_bodies_of_VARIABLES|_remove_bad_NAMES|_prefix_PARALLEL_ENV|_get_ignored_VARS|_make_grep_REGEXP|_ignore_UNDERSCORE|_alias_NAMES|_list_alias_BODIES|_function_NAMES|_list_function_BODIES|_variable_NAMES|_list_variable_VALUES|_prefix_PARALLEL_ENV)$' | + # Filter names matching --env + grep -E "^$_grep_REGEXP"\$ | grep -vE "^$_ignore_UNDERSCORE"\$ | + # Vars set by /bin/sh + grep -Ev '^(_)$' + } + + _get_ignored_VARS() { + perl -e ' + for(@ARGV){ + $next_is_env and push @envvar, split/,/, $_; + $next_is_env=/^--env$/; + } + if(grep { /^_$/ } @envvar) { + if(not open(IN, "<", "$ENV{HOME}/.parallel/ignored_vars")) { + print STDERR "parallel: Error: ", + "Run \"parallel --record-env\" in a clean environment first.\n"; + } else { + chomp(@ignored_vars = ); + $vars = join "|",map { quotemeta $_ } "env_parallel", @ignored_vars; + print $vars ? "($vars)" : "(,,nO,,VaRs,,)"; + } + } + ' -- "$@" + } + + # Get the --env variables if set + # --env _ should be ignored + # and convert a b c to (a|b|c) + # If --env not set: Match everything (.*) + _make_grep_REGEXP() { + perl -e ' + for(@ARGV){ + /^_$/ and $next_is_env = 0; + $next_is_env and push @envvar, split/,/, $_; + $next_is_env = /^--env$/; + } + $vars = join "|",map { quotemeta $_ } @envvar; + print $vars ? "($vars)" : "(.*)"; + ' -- "$@" + } + + if which parallel | grep 'no parallel in' >/dev/null; then + echo 'env_parallel: Error: parallel must be in $PATH.' >&2 + return 1 + fi + if which parallel >/dev/null; then + true which on linux + else + echo 'env_parallel: Error: parallel must be in $PATH.' >&2 + return 1 + fi + + # Grep regexp for vars given by --env + _grep_REGEXP="`_make_grep_REGEXP \"$@\"`" + + # Deal with --env _ + _ignore_UNDERSCORE="`_get_ignored_VARS \"$@\"`" + + # --record-env + if perl -e 'exit grep { /^--record-env$/ } @ARGV' -- "$@"; then + true skip + else + (_names_of_ALIASES; + _names_of_FUNCTIONS; + _names_of_VARIABLES) | + cat > $HOME/.parallel/ignored_vars + return 0 + fi + + # Grep alias names + _alias_NAMES="`_names_of_ALIASES | _remove_bad_NAMES`" + _list_alias_BODIES="_bodies_of_ALIASES $_alias_NAMES" + if [ "$_alias_NAMES" = "" ] ; then + # no aliases selected + _list_alias_BODIES="true" + fi + unset _alias_NAMES + + # Grep function names + _function_NAMES="`_names_of_FUNCTIONS | _remove_bad_NAMES`" + _list_function_BODIES="_bodies_of_FUNCTIONS $_function_NAMES" + if [ "$_function_NAMES" = "" ] ; then + # no functions selected + _list_function_BODIES="true" + fi + unset _function_NAMES + + # Grep variable names + _variable_NAMES="`_names_of_VARIABLES | _remove_bad_NAMES`" + _list_variable_VALUES="_bodies_of_VARIABLES $_variable_NAMES" + if [ "$_variable_NAMES" = "" ] ; then + # no variables selected + _list_variable_VALUES="true" + fi + unset _variable_NAMES + + PARALLEL_ENV="` + $_list_alias_BODIES; + $_list_function_BODIES; + $_list_variable_VALUES; + `" + export PARALLEL_ENV + unset _list_alias_BODIES + unset _list_variable_VALUES + unset _list_function_BODIES + `which parallel` "$@"; + _parallel_exit_CODE=$? + unset PARALLEL_ENV; + return $_parallel_exit_CODE +} diff --git a/src/env_parallel.ksh b/src/env_parallel.ksh index cc43ecc3..9cac59aa 100755 --- a/src/env_parallel.ksh +++ b/src/env_parallel.ksh @@ -28,23 +28,37 @@ env_parallel() { # env_parallel.ksh - # Get the --env variables if set - # --env _ should be ignored - # and convert a b c to (a|b|c) - # If --env not set: Match everything (.*) - _grep_REGEXP="$( - perl -e ' - for(@ARGV){ - /^_$/ and $next_is_env = 0; - $next_is_env and push @envvar, split/,/, $_; - $next_is_env = /^--env$/; - } - $vars = join "|",map { quotemeta $_ } @envvar; - print $vars ? "($vars)" : "(.*)"; - ' -- "$@" - )" - # Deal with --env _ - _ignore_UNDERSCORE="$( + _names_of_ALIASES() { + alias | perl -pe 's/=.*//' + } + _bodies_of_ALIASES() { + alias "$@" | perl -pe 's/^/alias /' + } + _names_of_maybe_FUNCTIONS() { + true not used + } + _names_of_FUNCTIONS() { + typeset +p -f | perl -pe 's/\(\).*//' + } + _bodies_of_FUNCTIONS() { + typeset -f "$@" + } + _names_of_VARIABLES() { + typeset +p | perl -pe 's/^typeset .. //' + } + _bodies_of_VARIABLES() { + typeset -p "$@" + } + _remove_bad_NAMES() { + # Do not transfer vars and funcs from env_parallel + grep -Ev '^(_names_of_ALIASES|_bodies_of_ALIASES|_names_of_maybe_FUNCTIONS|_names_of_FUNCTIONS|_bodies_of_FUNCTIONS|_names_of_VARIABLES|_bodies_of_VARIABLES|_remove_bad_NAMES|_prefix_PARALLEL_ENV|_get_ignored_VARS|_make_grep_REGEXP|_ignore_UNDERSCORE|_alias_NAMES|_list_alias_BODIES|_function_NAMES|_list_function_BODIES|_variable_NAMES|_list_variable_VALUES|_prefix_PARALLEL_ENV)$' | + # Filter names matching --env + grep -E "^$_grep_REGEXP"\$ | grep -vE "^$_ignore_UNDERSCORE"\$ | + # Vars set by /bin/sh + grep -Ev '^(_)$' + } + + _get_ignored_VARS() { perl -e ' for(@ARGV){ $next_is_env and push @envvar, split/,/, $_; @@ -61,56 +75,89 @@ env_parallel() { } } ' -- "$@" - )" + } + + # Get the --env variables if set + # --env _ should be ignored + # and convert a b c to (a|b|c) + # If --env not set: Match everything (.*) + _make_grep_REGEXP() { + perl -e ' + for(@ARGV){ + /^_$/ and $next_is_env = 0; + $next_is_env and push @envvar, split/,/, $_; + $next_is_env = /^--env$/; + } + $vars = join "|",map { quotemeta $_ } @envvar; + print $vars ? "($vars)" : "(.*)"; + ' -- "$@" + } + + if which parallel | grep 'no parallel in' >/dev/null; then + echo 'env_parallel: Error: parallel must be in $PATH.' >&2 + return 1 + fi + if which parallel >/dev/null; then + true which on linux + else + echo 'env_parallel: Error: parallel must be in $PATH.' >&2 + return 1 + fi + + # Grep regexp for vars given by --env + _grep_REGEXP="`_make_grep_REGEXP \"$@\"`" + + # Deal with --env _ + _ignore_UNDERSCORE="`_get_ignored_VARS \"$@\"`" # --record-env - if ! perl -e 'exit grep { /^--record-env$/ } @ARGV' -- "$@"; then - (alias | perl -pe 's/=.*//'; - typeset +p -f | perl -pe 's/\(\).*//'; - typeset +p | perl -pe 's/^typeset .. //') | + if perl -e 'exit grep { /^--record-env$/ } @ARGV' -- "$@"; then + true skip + else + (_names_of_ALIASES; + _names_of_FUNCTIONS; + _names_of_VARIABLES) | cat > $HOME/.parallel/ignored_vars return 0 fi - + # Grep alias names - _alias_NAMES="$(alias | perl -pe 's/=.*//' | - grep -E "^$_grep_REGEXP"\$ | grep -vE "^$_ignore_UNDERSCORE"\$ )" - _list_alias_BODIES="alias $_alias_NAMES | perl -pe 's/^/alias /'" - if [[ "$_alias_NAMES" = "" ]] ; then + _alias_NAMES="`_names_of_ALIASES | _remove_bad_NAMES`" + _list_alias_BODIES="_bodies_of_ALIASES $_alias_NAMES" + if [ "$_alias_NAMES" = "" ] ; then # no aliases selected _list_alias_BODIES="true" fi unset _alias_NAMES # Grep function names - _function_NAMES="$(typeset +p -f | perl -pe 's/\(\).*//' | - grep -E "^$_grep_REGEXP"\$ | grep -vE "^$_ignore_UNDERSCORE"\$ )" - _list_function_BODIES="typeset -f $_function_NAMES" - if [[ "$_function_NAMES" = "" ]] ; then + _function_NAMES="`_names_of_FUNCTIONS | _remove_bad_NAMES`" + _list_function_BODIES="_bodies_of_FUNCTIONS $_function_NAMES" + if [ "$_function_NAMES" = "" ] ; then # no functions selected _list_function_BODIES="true" fi unset _function_NAMES # Grep variable names - _variable_NAMES="$(typeset +p | perl -pe 's/^typeset .. //' | - grep -E "^$_grep_REGEXP"\$ | grep -vE "^$_ignore_UNDERSCORE"\$ | - grep -Ev '^(PIPESTATUS)$')" - _list_variable_VALUES="typeset -p $_variable_NAMES" - if [[ "$_variable_NAMES" = "" ]] ; then + _variable_NAMES="`_names_of_VARIABLES | _remove_bad_NAMES`" + _list_variable_VALUES="_bodies_of_VARIABLES $_variable_NAMES" + if [ "$_variable_NAMES" = "" ] ; then # no variables selected _list_variable_VALUES="true" fi unset _variable_NAMES - # eval is needed for aliases - cannot explain why - export PARALLEL_ENV="$( - eval $_list_alias_BODIES; + PARALLEL_ENV="` + $_list_alias_BODIES; + $_list_function_BODIES; $_list_variable_VALUES; - $_list_function_BODIES)"; + `" + export PARALLEL_ENV unset _list_alias_BODIES unset _list_variable_VALUES unset _list_function_BODIES + `which parallel` "$@"; _parallel_exit_CODE=$? unset PARALLEL_ENV; diff --git a/src/env_parallel.pdksh b/src/env_parallel.pdksh index a03eb380..76b2a768 100755 --- a/src/env_parallel.pdksh +++ b/src/env_parallel.pdksh @@ -28,6 +28,93 @@ env_parallel() { # env_parallel.pdksh + _names_of_ALIASES() { + compgen -a + } + _bodies_of_ALIASES() { + alias "$@" | perl -pe 's/^/alias /' + } + _names_of_FUNCTIONS() { + compgen -A function + } + _bodies_of_FUNCTIONS() { + typeset -f "$@" + } + _names_of_VARIABLES() { + compgen -A variable + } + _bodies_of_VARIABLES() { + typeset -p "$@" + } + _remove_bad_NAMES() { + _tmp_READONLY="$(mktemp)" + readonly > "$_tmp_READONLY" + grep -E "^$_grep_REGEXP"\$ | grep -vE "^$_ignore_UNDERSCORE"\$ | + grep -vFf $_tmp_READONLY | + grep -Ev '^(PIPESTATUS)' + rm $_tmp_READONLY + unset _tmp_READONLY + } + _prefix_PARALLEL_ENV() { + shopt 2>/dev/null | + perl -pe 's:\s+off:;: and s/^/shopt -u /; + s:\s+on:;: and s/^/shopt -s /; + s:;$:&>/dev/null;:'; + echo 'shopt -s expand_aliases &>/dev/null'; + } + + _get_ignored_VARS() { + perl -e ' + for(@ARGV){ + $next_is_env and push @envvar, split/,/, $_; + $next_is_env=/^--env$/; + } + if(grep { /^_$/ } @envvar) { + if(not open(IN, "<", "$ENV{HOME}/.parallel/ignored_vars")) { + print STDERR "parallel: Error: ", + "Run \"parallel --record-env\" in a clean environment first.\n"; + } else { + chomp(@ignored_vars = ); + $vars = join "|",map { quotemeta $_ } "env_parallel", @ignored_vars; + print $vars ? "($vars)" : "(,,nO,,VaRs,,)"; + } + } + ' -- "$@" + } + + # Get the --env variables if set + # --env _ should be ignored + # and convert a b c to (a|b|c) + # If --env not set: Match everything (.*) + _make_grep_REGEXP() { + perl -e ' + for(@ARGV){ + /^_$/ and $next_is_env = 0; + $next_is_env and push @envvar, split/,/, $_; + $next_is_env = /^--env$/; + } + $vars = join "|",map { quotemeta $_ } @envvar; + print $vars ? "($vars)" : "(.*)"; + ' -- "$@" + } + + if which parallel | grep 'no parallel in' >/dev/null; then + echo 'env_parallel: Error: parallel must be in $PATH.' >&2 + return 1 + fi + if which parallel >/dev/null; then + true which on linux + else + echo 'env_parallel: Error: parallel must be in $PATH.' >&2 + return 1 + fi + + # Grep regexp for vars given by --env + _grep_REGEXP="`_make_grep_REGEXP \"$@\"`" + + # Deal with --env _ + _ignore_UNDERSCORE="`_get_ignored_VARS \"$@\"`" + # Get the --env variables if set # --env _ should be ignored # and convert a b c to (a|b|c) @@ -64,53 +151,49 @@ env_parallel() { )" # --record-env - if ! perl -e 'exit grep { /^--record-env$/ } @ARGV' -- "$@"; then - (compgen -a; compgen -A function; compgen -A variable) | + if perl -e 'exit grep { /^--record-env$/ } @ARGV' -- "$@"; then + true skip + else + (_names_of_ALIASES; + _names_of_FUNCTIONS; + _names_of_VARIABLES) | cat > $HOME/.parallel/ignored_vars return 0 fi # Grep alias names - _alias_NAMES="$(alias | perl -pe 's/=.*//' | - grep -E "^$_grep_REGEXP"\$ | grep -vE "^$_ignore_UNDERSCORE"\$ )" - _list_alias_BODIES="alias $_alias_NAMES | perl -pe 's/^/alias /'" - if [[ "$_alias_NAMES" = "" ]] ; then + _alias_NAMES="`_names_of_ALIASES | _remove_bad_NAMES`" + _list_alias_BODIES="_bodies_of_ALIASES $_alias_NAMES" + if [ "$_alias_NAMES" = "" ] ; then # no aliases selected _list_alias_BODIES="true" fi unset _alias_NAMES # Grep function names - _function_NAMES=$(typeset +p -f | perl -pe 's/\(\).*//' | - grep -E "^$_grep_REGEXP"\$ | grep -vE "^$_ignore_UNDERSCORE"\$ ) - _list_function_BODIES="typeset -f $_function_NAMES" - if [[ "$_function_NAMES" = "" ]] ; then + _function_NAMES="`_names_of_FUNCTIONS | _remove_bad_NAMES`" + _list_function_BODIES="_bodies_of_FUNCTIONS $_function_NAMES" + if [ "$_function_NAMES" = "" ] ; then # no functions selected _list_function_BODIES="true" fi unset _function_NAMES # Grep variable names - _tmp_READONLY="$(mktemp)" - readonly > "$_tmp_READONLY" - _variable_NAMES="$(typeset | perl -pe 's/^(type)?set( -.)* //' | - grep -E "^$_grep_REGEXP"\$ | grep -vE "^$_ignore_UNDERSCORE"\$ | - grep -vFf $_tmp_READONLY | - grep -Ev '^(PIPESTATUS)')" - rm $_tmp_READONLY - unset _tmp_READONLY - _list_variable_VALUES="typeset -p $_variable_NAMES" - if [[ "$_variable_NAMES" = "" ]] ; then + _variable_NAMES="`_names_of_VARIABLES | _remove_bad_NAMES`" + _list_variable_VALUES="_bodies_of_VARIABLES $_variable_NAMES" + if [ "$_variable_NAMES" = "" ] ; then # no variables selected _list_variable_VALUES="true" fi unset _variable_NAMES # eval is needed for aliases - cannot explain why - export PARALLEL_ENV="$( + export PARALLEL_ENV="` eval $_list_alias_BODIES; + $_list_function_BODIES $_list_variable_VALUES; - $_list_function_BODIES)"; + `"; unset _list_alias_BODIES unset _list_variable_VALUES unset _list_function_BODIES diff --git a/src/env_parallel.pod b/src/env_parallel.pod index 21a52694..87c2da5a 100644 --- a/src/env_parallel.pod +++ b/src/env_parallel.pod @@ -81,6 +81,50 @@ Same as GNU B. =head1 SUPPORTED SHELLS +=head2 ash + +B<--env> is supported to export only the variable, or alias with the +given name. Multiple B<--env>s can be given. + +Installation + +Put this in $HOME/.profile: + + . `which env_parallel.ash` + +E.g. by doing: + + echo '. `which env_parallel.ash`' >> $HOME/.profile + +=over 8 + +=item aliases + + alias myecho='echo aliases' + env_parallel myecho ::: work + env_parallel -S server myecho ::: work + env_parallel --env myecho myecho ::: work + env_parallel --env myecho -S server myecho ::: work + +=item functions + + ash cannot list defined functions - thus is not supported. + +=item variables + + myvar=variables + env_parallel echo '$myvar' ::: work + env_parallel -S server echo '$myvar' ::: work + env_parallel --env myvar echo '$myvar' ::: work + env_parallel --env myvar -S server echo '$myvar' ::: work + +=item arrays + + ash does not support arrays. + +=back + + =head2 Bash B<--env> is supported to export only the variable, alias, function, or @@ -184,6 +228,50 @@ Not supported by B. =back +=head2 dash + +B<--env> is supported to export only the variable, or alias with the +given name. Multiple B<--env>s can be given. + +Installation + +Put this in $HOME/.profile: + + . `which env_parallel.dash` + +E.g. by doing: + + echo '. `which env_parallel.dash`' >> $HOME/.profile + +=over 8 + +=item aliases + + alias myecho='echo aliases' + env_parallel myecho ::: work + env_parallel -S server myecho ::: work + env_parallel --env myecho myecho ::: work + env_parallel --env myecho -S server myecho ::: work + +=item functions + + dash cannot list defined functions - thus is not supported. + +=item variables + + myvar=variables + env_parallel echo '$myvar' ::: work + env_parallel -S server echo '$myvar' ::: work + env_parallel --env myvar echo '$myvar' ::: work + env_parallel --env myvar -S server echo '$myvar' ::: work + +=item arrays + + dash does not support arrays. + +=back + + =head2 fish B<--env> is supported to export only the variable, alias, function, or @@ -342,6 +430,50 @@ E.g. by doing: =back +=head2 sh + +B<--env> is supported to export only the variable, or alias with the +given name. Multiple B<--env>s can be given. + +Installation + +Put this in $HOME/.profile: + + . `which env_parallel.sh` + +E.g. by doing: + + echo '. `which env_parallel.sh`' >> $HOME/.profile + +=over 8 + +=item aliases + + sh does not support aliases. + +=item functions + + myfunc() { echo functions $*; } + env_parallel myfunc ::: work + env_parallel -S server myfunc ::: work + env_parallel --env myfunc myfunc ::: work + env_parallel --env myfunc -S server myfunc ::: work + +=item variables + + myvar=variables + env_parallel echo '$myvar' ::: work + env_parallel -S server echo '$myvar' ::: work + env_parallel --env myvar echo '$myvar' ::: work + env_parallel --env myvar -S server echo '$myvar' ::: work + +=item arrays + + sh does not support arrays. + +=back + + =head2 tcsh B<--env> is supported to export only the variable, alias, or diff --git a/src/env_parallel.sh b/src/env_parallel.sh new file mode 100644 index 00000000..c7693758 --- /dev/null +++ b/src/env_parallel.sh @@ -0,0 +1,174 @@ +#!/usr/bin/env sh + +# This file must be sourced in sh: +# +# . `which env_parallel.sh` +# +# after which 'env_parallel' works +# +# +# Copyright (C) 2017 +# Ole Tange and Free Software Foundation, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, see +# or write to the Free Software Foundation, Inc., 51 Franklin St, +# Fifth Floor, Boston, MA 02110-1301 USA + +env_parallel() { + # env_parallel.sh + + _names_of_ALIASES() { + alias | perl -pe 's/=.*//' + } + _bodies_of_ALIASES() { + alias "$@" | perl -pe 's/^/alias /' + } + _names_of_maybe_FUNCTIONS() { + set | perl -ne '/^(\S+)\(\)\{$/ and print "$1\n"' + } + _names_of_FUNCTIONS() { + # myfunc is a function + type `_names_of_maybe_FUNCTIONS` | + perl -ne '/^(\S+) is a function$/ and not $seen{$1}++ and print "$1\n"' + } + _bodies_of_FUNCTIONS() { + type "$@" | perl -ne '/^(\S+) is a function$/ or print' + } + _names_of_VARIABLES() { + # This may screw up if variables contain \n and = + set | perl -ne 's/^(\S+)=.*/$1/ and print;' + } + _bodies_of_VARIABLES() { + # Crappy typeset -p + for _i in "$@" + do + perl -e 'print @ARGV' "$_i=" + eval echo \"\$$_i\" | perl -e '$/=undef; $a=<>; chop($a); print $a' | + perl -pe 's/[\002-\011\013-\032\\\#\?\`\(\)\{\}\[\]\^\*\<\=\>\~\|\; \"\!\$\&\202-\377]/\\$&/go;'"s/'/\\\'/g; s/[\n]/'\\n'/go;"; + echo + done + } + _remove_bad_NAMES() { + # Do not transfer vars and funcs from env_parallel + grep -Ev '^(_names_of_ALIASES|_bodies_of_ALIASES|_names_of_maybe_FUNCTIONS|_names_of_FUNCTIONS|_bodies_of_FUNCTIONS|_names_of_VARIABLES|_bodies_of_VARIABLES|_remove_bad_NAMES|_prefix_PARALLEL_ENV|_get_ignored_VARS|_make_grep_REGEXP|_ignore_UNDERSCORE|_alias_NAMES|_list_alias_BODIES|_function_NAMES|_list_function_BODIES|_variable_NAMES|_list_variable_VALUES|_prefix_PARALLEL_ENV)$' | + # Filter names matching --env + grep -E "^$_grep_REGEXP"\$ | grep -vE "^$_ignore_UNDERSCORE"\$ | + # Vars set by /bin/sh + grep -Ev '^(_)$' + } + + _get_ignored_VARS() { + perl -e ' + for(@ARGV){ + $next_is_env and push @envvar, split/,/, $_; + $next_is_env=/^--env$/; + } + if(grep { /^_$/ } @envvar) { + if(not open(IN, "<", "$ENV{HOME}/.parallel/ignored_vars")) { + print STDERR "parallel: Error: ", + "Run \"parallel --record-env\" in a clean environment first.\n"; + } else { + chomp(@ignored_vars = ); + $vars = join "|",map { quotemeta $_ } "env_parallel", @ignored_vars; + print $vars ? "($vars)" : "(,,nO,,VaRs,,)"; + } + } + ' -- "$@" + } + + # Get the --env variables if set + # --env _ should be ignored + # and convert a b c to (a|b|c) + # If --env not set: Match everything (.*) + _make_grep_REGEXP() { + perl -e ' + for(@ARGV){ + /^_$/ and $next_is_env = 0; + $next_is_env and push @envvar, split/,/, $_; + $next_is_env = /^--env$/; + } + $vars = join "|",map { quotemeta $_ } @envvar; + print $vars ? "($vars)" : "(.*)"; + ' -- "$@" + } + + if which parallel | grep 'no parallel in' >/dev/null; then + echo 'env_parallel: Error: parallel must be in $PATH.' >&2 + return 1 + fi + if which parallel >/dev/null; then + true which on linux + else + echo 'env_parallel: Error: parallel must be in $PATH.' >&2 + return 1 + fi + + # Grep regexp for vars given by --env + _grep_REGEXP="`_make_grep_REGEXP \"$@\"`" + + # Deal with --env _ + _ignore_UNDERSCORE="`_get_ignored_VARS \"$@\"`" + + # --record-env + if perl -e 'exit grep { /^--record-env$/ } @ARGV' -- "$@"; then + true skip + else + (_names_of_ALIASES; + _names_of_FUNCTIONS; + _names_of_VARIABLES) | + cat > $HOME/.parallel/ignored_vars + return 0 + fi + + # Grep alias names + _alias_NAMES="`_names_of_ALIASES | _remove_bad_NAMES`" + _list_alias_BODIES="_bodies_of_ALIASES $_alias_NAMES" + if [ "$_alias_NAMES" = "" ] ; then + # no aliases selected + _list_alias_BODIES="true" + fi + unset _alias_NAMES + + # Grep function names + _function_NAMES="`_names_of_FUNCTIONS | _remove_bad_NAMES`" + _list_function_BODIES="_bodies_of_FUNCTIONS $_function_NAMES" + if [ "$_function_NAMES" = "" ] ; then + # no functions selected + _list_function_BODIES="true" + fi + unset _function_NAMES + + # Grep variable names + _variable_NAMES="`_names_of_VARIABLES | _remove_bad_NAMES`" + _list_variable_VALUES="_bodies_of_VARIABLES $_variable_NAMES" + if [ "$_variable_NAMES" = "" ] ; then + # no variables selected + _list_variable_VALUES="true" + fi + unset _variable_NAMES + + PARALLEL_ENV="` + $_list_alias_BODIES; + $_list_function_BODIES; + $_list_variable_VALUES; + `" + export PARALLEL_ENV + unset _list_alias_BODIES + unset _list_variable_VALUES + unset _list_function_BODIES + `which parallel` "$@"; + _parallel_exit_CODE=$? + unset PARALLEL_ENV; + return $_parallel_exit_CODE +} diff --git a/src/env_parallel.zsh b/src/env_parallel.zsh index 89602552..ff93fc9d 100755 --- a/src/env_parallel.zsh +++ b/src/env_parallel.zsh @@ -28,23 +28,34 @@ env_parallel() { # env_parallel.zsh - # Get the --env variables if set - # --env _ should be ignored - # and convert a b c to (a|b|c) - # If --env not set: Match everything (.*) - _grep_REGEXP="$( - perl -e ' - for(@ARGV){ - /^_$/ and $next_is_env = 0; - $next_is_env and push @envvar, split/,/, $_; - $next_is_env = /^--env$/; - } - $vars = join "|",map { quotemeta $_ } @envvar; - print $vars ? "($vars)" : "(.*)"; - ' -- "$@" - )" - # Deal with --env _ - local _ignore_UNDERSCORE="$( + _names_of_ALIASES() { + print -l ${(k)aliases} + } + _bodies_of_ALIASES() { + echo "alias "$(echo "$@"|xargs)" | perl -pe 's/^/alias /'" + } + _names_of_FUNCTIONS() { + print -l ${(k)functions} + } + _bodies_of_FUNCTIONS() { + echo "typeset -f "$(echo "$@"|xargs) + } + _names_of_VARIABLES() { + print -l ${(k)parameters} + } + _bodies_of_VARIABLES() { + echo typeset -p "$(echo $@|xargs)" '| grep -aFvf <(typeset -pr)' + } + _remove_bad_NAMES() { + # Do not transfer vars and funcs from env_parallel + grep -Ev '^(_names_of_ALIASES|_bodies_of_ALIASES|_names_of_maybe_FUNCTIONS|_names_of_FUNCTIONS|_bodies_of_FUNCTIONS|_names_of_VARIABLES|_bodies_of_VARIABLES|_remove_bad_NAMES|_prefix_PARALLEL_ENV|_get_ignored_VARS|_make_grep_REGEXP|_ignore_UNDERSCORE|_alias_NAMES|_list_alias_BODIES|_function_NAMES|_list_function_BODIES|_variable_NAMES|_list_variable_VALUES|_prefix_PARALLEL_ENV)$' | + grep -E "^$_grep_REGEXP"\$ | grep -vE "^$_ignore_UNDERSCORE"\$ | + grep -v '=' | + grep -Ev '^([-?#!$*@_0]|zsh_eval_context|ZSH_EVAL_CONTEXT|LINENO|IFS|commands|functions|options|aliases|EUID|EGID|UID|GID)$' | + grep -Ev '^(dis_patchars|patchars|terminfo|funcstack|galiases|keymaps|parameters|jobdirs|dirstack|functrace|funcsourcetrace|zsh_scheduled_events|dis_aliases|dis_reswords|dis_saliases|modules|reswords|saliases|widgets|userdirs|historywords|nameddirs|termcap|dis_builtins|dis_functions|jobtexts|funcfiletrace|dis_galiases|builtins|history|jobstates)$' + } + + _get_ignored_VARS() { perl -e ' for(@ARGV){ $next_is_env and push @envvar, split/,/, $_; @@ -61,59 +72,87 @@ env_parallel() { } } ' -- "$@" - )" + } + + # Get the --env variables if set + # --env _ should be ignored + # and convert a b c to (a|b|c) + # If --env not set: Match everything (.*) + _make_grep_REGEXP() { + perl -e ' + for(@ARGV){ + /^_$/ and $next_is_env = 0; + $next_is_env and push @envvar, split/,/, $_; + $next_is_env = /^--env$/; + } + $vars = join "|",map { quotemeta $_ } @envvar; + print $vars ? "($vars)" : "(.*)"; + ' -- "$@" + } + + if which parallel | grep 'no parallel in' >/dev/null; then + echo 'env_parallel: Error: parallel must be in $PATH.' >&2 + return 1 + fi + if which parallel >/dev/null; then + true which on linux + else + echo 'env_parallel: Error: parallel must be in $PATH.' >&2 + return 1 + fi + + # Grep regexp for vars given by --env + _grep_REGEXP="`_make_grep_REGEXP \"$@\"`" + + # Deal with --env _ + _ignore_UNDERSCORE="`_get_ignored_VARS \"$@\"`" # --record-env - if ! perl -e 'exit grep { /^--record-env$/ } @ARGV' -- "$@"; then - (print -l ${(k)aliases}; - print -l ${(k)functions}; - print -l ${(k)parameters}) | + if perl -e 'exit grep { /^--record-env$/ } @ARGV' -- "$@"; then + true skip + else + (_names_of_ALIASES; + _names_of_FUNCTIONS; + _names_of_VARIABLES) | cat > $HOME/.parallel/ignored_vars return 0 fi # Grep alias names - _alias_NAMES="$(print -l ${(k)aliases} | - grep -E "^$_grep_REGEXP"\$ | grep -vE "^$_ignore_UNDERSCORE"\$ )" + _alias_NAMES="`_names_of_ALIASES | _remove_bad_NAMES`" _list_alias_BODIES="alias "$(echo $_alias_NAMES|xargs)" | perl -pe 's/^/alias /'" - if [[ "$_alias_NAMES" = "" ]] ; then + if [ "$_alias_NAMES" = "" ] ; then # no aliases selected _list_alias_BODIES="true" fi unset _alias_NAMES # Grep function names - _function_NAMES="$(print -l ${(k)functions} | - grep -E "^$_grep_REGEXP\$" | grep -vE "^$_ignore_UNDERSCORE\$" | - grep -v '=' - )" + _function_NAMES="`_names_of_FUNCTIONS | _remove_bad_NAMES`" _list_function_BODIES="typeset -f "$(echo $_function_NAMES|xargs) - if [[ "$_function_NAMES" = "" ]] ; then + if [ "$_function_NAMES" = "" ] ; then # no functions selected _list_function_BODIES="true" fi unset _function_NAMES # Grep variable names - # The grep -Ev is crap and should be better - _variable_NAMES="$(print -l ${(k)parameters} | - grep -E "^$_grep_REGEXP"\$ | grep -vE "^$_ignore_UNDERSCORE"\$ | - grep -Ev '^([-?#!$*@_0]|zsh_eval_context|ZSH_EVAL_CONTEXT|LINENO|IFS|commands|functions|options|aliases|EUID|EGID|UID|GID)$' | - grep -Ev '^(dis_patchars|patchars|terminfo|funcstack|galiases|keymaps|parameters|jobdirs|dirstack|functrace|funcsourcetrace|zsh_scheduled_events|dis_aliases|dis_reswords|dis_saliases|modules|reswords|saliases|widgets|userdirs|historywords|nameddirs|termcap|dis_builtins|dis_functions|jobtexts|funcfiletrace|dis_galiases|builtins|history|jobstates)$' - )" + _variable_NAMES="`_names_of_VARIABLES | _remove_bad_NAMES`" _list_variable_VALUES="typeset -p "$(echo $_variable_NAMES|xargs)" | grep -aFvf <(typeset -pr) " - if [[ "$_variable_NAMES" = "" ]] ; then + if [ "$_variable_NAMES" = "" ] ; then # no variables selected _list_variable_VALUES="true" fi unset _variable_NAMES - export PARALLEL_ENV="$( + + PARALLEL_ENV="` eval $_list_alias_BODIES; eval $_list_function_BODIES; eval $_list_variable_VALUES; - )"; + `" + export PARALLEL_ENV unset _list_alias_BODIES unset _list_variable_VALUES unset _list_function_BODIES diff --git a/src/parallel b/src/parallel index d052174d..8daa9fe7 100755 --- a/src/parallel +++ b/src/parallel @@ -3508,7 +3508,7 @@ sub onall { (@Global::ret_files ? map { "--return ".::shell_quote_scalar($_) } @Global::ret_files : ""), (@opt::env ? map { "--env ".::shell_quote_scalar($_) } @opt::env : ""), - (@opt::v ? "-vv" : ""), + (map { "-v" } @opt::v), ); ::debug("init", "| $0 $options\n"); open(my $parallel_fh, "|-", "$0 --will-cite -j0 $options") || diff --git a/src/parallel_tutorial.1 b/src/parallel_tutorial.1 deleted file mode 100644 index 788dbcae..00000000 --- a/src/parallel_tutorial.1 +++ /dev/null @@ -1,2801 +0,0 @@ -.\" Automatically generated by Pod::Man 2.27 (Pod::Simple 3.28) -.\" -.\" Standard preamble: -.\" ======================================================================== -.de Sp \" Vertical space (when we can't use .PP) -.if t .sp .5v -.if n .sp -.. -.de Vb \" Begin verbatim text -.ft CW -.nf -.ne \\$1 -.. -.de Ve \" End verbatim text -.ft R -.fi -.. -.\" Set up some character translations and predefined strings. \*(-- will -.\" give an unbreakable dash, \*(PI will give pi, \*(L" will give a left -.\" double quote, and \*(R" will give a right double quote. \*(C+ will -.\" give a nicer C++. Capital omega is used to do unbreakable dashes and -.\" therefore won't be available. \*(C` and \*(C' expand to `' in nroff, -.\" nothing in troff, for use with C<>. -.tr \(*W- -.ds C+ C\v'-.1v'\h'-1p'\s-2+\h'-1p'+\s0\v'.1v'\h'-1p' -.ie n \{\ -. ds -- \(*W- -. ds PI pi -. if (\n(.H=4u)&(1m=24u) .ds -- \(*W\h'-12u'\(*W\h'-12u'-\" diablo 10 pitch -. if (\n(.H=4u)&(1m=20u) .ds -- \(*W\h'-12u'\(*W\h'-8u'-\" diablo 12 pitch -. ds L" "" -. ds R" "" -. ds C` "" -. ds C' "" -'br\} -.el\{\ -. ds -- \|\(em\| -. ds PI \(*p -. ds L" `` -. ds R" '' -. ds C` -. ds C' -'br\} -.\" -.\" Escape single quotes in literal strings from groff's Unicode transform. -.ie \n(.g .ds Aq \(aq -.el .ds Aq ' -.\" -.\" If the F register is turned on, we'll generate index entries on stderr for -.\" titles (.TH), headers (.SH), subsections (.SS), items (.Ip), and index -.\" entries marked with X<> in POD. Of course, you'll have to process the -.\" output yourself in some meaningful fashion. -.\" -.\" Avoid warning from groff about undefined register 'F'. -.de IX -.. -.nr rF 0 -.if \n(.g .if rF .nr rF 1 -.if (\n(rF:(\n(.g==0)) \{ -. if \nF \{ -. de IX -. tm Index:\\$1\t\\n%\t"\\$2" -.. -. if !\nF==2 \{ -. nr % 0 -. nr F 2 -. \} -. \} -.\} -.rr rF -.\" -.\" Accent mark definitions (@(#)ms.acc 1.5 88/02/08 SMI; from UCB 4.2). -.\" Fear. Run. Save yourself. No user-serviceable parts. -. \" fudge factors for nroff and troff -.if n \{\ -. ds #H 0 -. ds #V .8m -. ds #F .3m -. ds #[ \f1 -. ds #] \fP -.\} -.if t \{\ -. ds #H ((1u-(\\\\n(.fu%2u))*.13m) -. ds #V .6m -. ds #F 0 -. ds #[ \& -. ds #] \& -.\} -. \" simple accents for nroff and troff -.if n \{\ -. ds ' \& -. ds ` \& -. ds ^ \& -. ds , \& -. ds ~ ~ -. ds / -.\} -.if t \{\ -. ds ' \\k:\h'-(\\n(.wu*8/10-\*(#H)'\'\h"|\\n:u" -. ds ` \\k:\h'-(\\n(.wu*8/10-\*(#H)'\`\h'|\\n:u' -. ds ^ \\k:\h'-(\\n(.wu*10/11-\*(#H)'^\h'|\\n:u' -. ds , \\k:\h'-(\\n(.wu*8/10)',\h'|\\n:u' -. ds ~ \\k:\h'-(\\n(.wu-\*(#H-.1m)'~\h'|\\n:u' -. ds / \\k:\h'-(\\n(.wu*8/10-\*(#H)'\z\(sl\h'|\\n:u' -.\} -. \" troff and (daisy-wheel) nroff accents -.ds : \\k:\h'-(\\n(.wu*8/10-\*(#H+.1m+\*(#F)'\v'-\*(#V'\z.\h'.2m+\*(#F'.\h'|\\n:u'\v'\*(#V' -.ds 8 \h'\*(#H'\(*b\h'-\*(#H' -.ds o \\k:\h'-(\\n(.wu+\w'\(de'u-\*(#H)/2u'\v'-.3n'\*(#[\z\(de\v'.3n'\h'|\\n:u'\*(#] -.ds d- \h'\*(#H'\(pd\h'-\w'~'u'\v'-.25m'\f2\(hy\fP\v'.25m'\h'-\*(#H' -.ds D- D\\k:\h'-\w'D'u'\v'-.11m'\z\(hy\v'.11m'\h'|\\n:u' -.ds th \*(#[\v'.3m'\s+1I\s-1\v'-.3m'\h'-(\w'I'u*2/3)'\s-1o\s+1\*(#] -.ds Th \*(#[\s+2I\s-2\h'-\w'I'u*3/5'\v'-.3m'o\v'.3m'\*(#] -.ds ae a\h'-(\w'a'u*4/10)'e -.ds Ae A\h'-(\w'A'u*4/10)'E -. \" corrections for vroff -.if v .ds ~ \\k:\h'-(\\n(.wu*9/10-\*(#H)'\s-2\u~\d\s+2\h'|\\n:u' -.if v .ds ^ \\k:\h'-(\\n(.wu*10/11-\*(#H)'\v'-.4m'^\v'.4m'\h'|\\n:u' -. \" for low resolution devices (crt and lpr) -.if \n(.H>23 .if \n(.V>19 \ -\{\ -. ds : e -. ds 8 ss -. ds o a -. ds d- d\h'-1'\(ga -. ds D- D\h'-1'\(hy -. ds th \o'bp' -. ds Th \o'LP' -. ds ae ae -. ds Ae AE -.\} -.rm #[ #] #H #V #F C -.\" ======================================================================== -.\" -.IX Title "PARALLEL_TUTORIAL 1" -.TH PARALLEL_TUTORIAL 1 "2014-11-26" "20141122" "parallel" -.\" For nroff, turn off justification. Always turn off hyphenation; it makes -.\" way too many mistakes in technical documents. -.if n .ad l -.nh -.SH "GNU Parallel Tutorial" -.IX Header "GNU Parallel Tutorial" -This tutorial shows off much of \s-1GNU\s0 Parallel's functionality. The -tutorial is meant to learn the options in \s-1GNU\s0 Parallel. The tutorial -is not to show realistic examples from the real world. -.PP -Spend an hour walking through the tutorial. Your command line will -love you for it. -.SH "Prerequisites" -.IX Header "Prerequisites" -To run this tutorial you must have the following: -.IP "parallel >= version 20140622" 9 -.IX Item "parallel >= version 20140622" -Install the newest version with: -.Sp -.Vb 1 -\& (wget \-O \- pi.dk/3 || curl pi.dk/3/) | bash -.Ve -.Sp -This will also install the newest version of the tutorial: -.Sp -.Vb 1 -\& man parallel_tutorial -.Ve -.Sp -Most of the tutorial will work on older versions, too. -.IP "abc-file:" 9 -.IX Item "abc-file:" -The file can be generated by: -.Sp -.Vb 1 -\& parallel \-k echo ::: A B C > abc\-file -.Ve -.IP "def-file:" 9 -.IX Item "def-file:" -The file can be generated by: -.Sp -.Vb 1 -\& parallel \-k echo ::: D E F > def\-file -.Ve -.IP "abc0\-file:" 9 -.IX Item "abc0-file:" -The file can be generated by: -.Sp -.Vb 1 -\& perl \-e \*(Aqprintf "A\e0B\e0C\e0"\*(Aq > abc0\-file -.Ve -.IP "abc_\-file:" 9 -.IX Item "abc_-file:" -The file can be generated by: -.Sp -.Vb 1 -\& perl \-e \*(Aqprintf "A_B_C_"\*(Aq > abc_\-file -.Ve -.IP "tsv\-file.tsv" 9 -.IX Item "tsv-file.tsv" -The file can be generated by: -.Sp -.Vb 1 -\& perl \-e \*(Aqprintf "f1\etf2\enA\etB\enC\etD\en"\*(Aq > tsv\-file.tsv -.Ve -.IP "num30000" 9 -.IX Item "num30000" -The file can be generated by: -.Sp -.Vb 1 -\& perl \-e \*(Aqfor(1..30000){print "$_\en"}\*(Aq > num30000 -.Ve -.IP "num1000000" 9 -.IX Item "num1000000" -The file can be generated by: -.Sp -.Vb 1 -\& perl \-e \*(Aqfor(1..1000000){print "$_\en"}\*(Aq > num1000000 -.Ve -.IP "num_%header" 9 -.IX Item "num_%header" -The file can be generated by: -.Sp -.Vb 1 -\& (echo %head1; echo %head2; perl \-e \*(Aqfor(1..10){print "$_\en"}\*(Aq) > num_%header -.Ve -.ie n .IP "For remote running: ssh login on 2 servers with no password in $SERVER1 and $SERVER2" 9 -.el .IP "For remote running: ssh login on 2 servers with no password in \f(CW$SERVER1\fR and \f(CW$SERVER2\fR" 9 -.IX Item "For remote running: ssh login on 2 servers with no password in $SERVER1 and $SERVER2" -.Vb 2 -\& SERVER1=server.example.com -\& SERVER2=server2.example.net -.Ve -.Sp -You must be able to: -.Sp -.Vb 2 -\& ssh $SERVER1 echo works -\& ssh $SERVER2 echo works -.Ve -.Sp -It can be setup by running 'ssh\-keygen \-t dsa; ssh-copy-id \f(CW$SERVER1\fR' -and using an empty pass phrase. -.SH "Input sources" -.IX Header "Input sources" -\&\s-1GNU\s0 Parallel reads input from input sources. These can be files, the -command line, and stdin (standard input or a pipe). -.SS "A single input source" -.IX Subsection "A single input source" -Input can be read from the command line: -.PP -.Vb 1 -\& parallel echo ::: A B C -.Ve -.PP -Output (the order may be different because the jobs are run in -parallel): -.PP -.Vb 3 -\& A -\& B -\& C -.Ve -.PP -The input source can be a file: -.PP -.Vb 1 -\& parallel \-a abc\-file echo -.Ve -.PP -Output: Same as above. -.PP -\&\s-1STDIN \s0(standard input) can be the input source: -.PP -.Vb 1 -\& cat abc\-file | parallel echo -.Ve -.PP -Output: Same as above. -.SS "Multiple input sources" -.IX Subsection "Multiple input sources" -\&\s-1GNU\s0 Parallel can take multiple input sources given on the command -line. \s-1GNU\s0 Parallel then generates all combinations of the input -sources: -.PP -.Vb 1 -\& parallel echo ::: A B C ::: D E F -.Ve -.PP -Output (the order may be different): -.PP -.Vb 9 -\& A D -\& A E -\& A F -\& B D -\& B E -\& B F -\& C D -\& C E -\& C F -.Ve -.PP -The input sources can be files: -.PP -.Vb 1 -\& parallel \-a abc\-file \-a def\-file echo -.Ve -.PP -Output: Same as above. -.PP -\&\s-1STDIN \s0(standard input) can be one of the input sources using '\-': -.PP -.Vb 1 -\& cat abc\-file | parallel \-a \- \-a def\-file echo -.Ve -.PP -Output: Same as above. -.PP -Instead of \-a files can be given after '::::': -.PP -.Vb 1 -\& cat abc\-file | parallel echo :::: \- def\-file -.Ve -.PP -Output: Same as above. -.PP -::: and :::: can be mixed: -.PP -.Vb 1 -\& parallel echo ::: A B C :::: def\-file -.Ve -.PP -Output: Same as above. -.PP -\fIMatching arguments from all input sources\fR -.IX Subsection "Matching arguments from all input sources" -.PP -With \-\-xapply you can get one argument from each input source: -.PP -.Vb 1 -\& parallel \-\-xapply echo ::: A B C ::: D E F -.Ve -.PP -Output (the order may be different): -.PP -.Vb 3 -\& A D -\& B E -\& C F -.Ve -.PP -If one of the input sources is too short, its values will wrap: -.PP -.Vb 1 -\& parallel \-\-xapply echo ::: A B C D E ::: F G -.Ve -.PP -Output (the order may be different): -.PP -.Vb 5 -\& A F -\& B G -\& C F -\& D G -\& E F -.Ve -.SS "Changing the argument separator." -.IX Subsection "Changing the argument separator." -\&\s-1GNU\s0 Parallel can use other separators than ::: or ::::. This is -typically useful if ::: or :::: is used in the command to run: -.PP -.Vb 1 -\& parallel \-\-arg\-sep ,, echo ,, A B C :::: def\-file -.Ve -.PP -Output (the order may be different): -.PP -.Vb 9 -\& A D -\& A E -\& A F -\& B D -\& B E -\& B F -\& C D -\& C E -\& C F -.Ve -.PP -Changing the argument file separator: -.PP -.Vb 1 -\& parallel \-\-arg\-file\-sep // echo ::: A B C // def\-file -.Ve -.PP -Output: Same as above. -.SS "Changing the argument delimiter" -.IX Subsection "Changing the argument delimiter" -\&\s-1GNU\s0 Parallel will normally treat a full line as a single argument: It -uses \en as argument delimiter. This can be changed with \-d: -.PP -.Vb 1 -\& parallel \-d _ echo :::: abc_\-file -.Ve -.PP -Output (the order may be different): -.PP -.Vb 3 -\& A -\& B -\& C -.Ve -.PP -\&\s-1NULL\s0 can be given as \e0: -.PP -.Vb 1 -\& parallel \-d \*(Aq\e0\*(Aq echo :::: abc0\-file -.Ve -.PP -Output: Same as above. -.PP -A shorthand for \-d '\e0' is \-0 (this will often be used to read files -from find ... \-print0): -.PP -.Vb 1 -\& parallel \-0 echo :::: abc0\-file -.Ve -.PP -Output: Same as above. -.SS "End-of-file value for input source" -.IX Subsection "End-of-file value for input source" -\&\s-1GNU\s0 Parallel can stop reading when it encounters a certain value: -.PP -.Vb 1 -\& parallel \-E stop echo ::: A B stop C D -.Ve -.PP -Output: -.PP -.Vb 2 -\& A -\& B -.Ve -.SS "Skipping empty lines" -.IX Subsection "Skipping empty lines" -Using \-\-no\-run\-if\-empty \s-1GNU\s0 Parallel will skip empty lines. -.PP -.Vb 1 -\& (echo 1; echo; echo 2) | parallel \-\-no\-run\-if\-empty echo -.Ve -.PP -Output: -.PP -.Vb 2 -\& 1 -\& 2 -.Ve -.SH "Building the command line" -.IX Header "Building the command line" -.SS "No command means arguments are commands" -.IX Subsection "No command means arguments are commands" -If no command is given after parallel the arguments themselves are -treated as commands: -.PP -.Vb 1 -\& parallel ::: ls \*(Aqecho foo\*(Aq pwd -.Ve -.PP -Output (the order may be different): -.PP -.Vb 3 -\& [list of files in current dir] -\& foo -\& [/path/to/current/working/dir] -.Ve -.PP -The command can be a script, a binary or a Bash function if the function is -exported using 'export \-f': -.PP -.Vb 6 -\& # Only works in Bash and only if $SHELL=.../bash -\& my_func() { -\& echo in my_func $1 -\& } -\& export \-f my_func -\& parallel my_func ::: 1 2 3 -.Ve -.PP -Output (the order may be different): -.PP -.Vb 3 -\& in my_func 1 -\& in my_func 2 -\& in my_func 3 -.Ve -.SS "Replacement strings" -.IX Subsection "Replacement strings" -\fIThe 7 predefined replacement strings\fR -.IX Subsection "The 7 predefined replacement strings" -.PP -\&\s-1GNU\s0 Parallel has several replacement strings. If no replacement -strings are used the default is to append {}: -.PP -.Vb 1 -\& parallel echo ::: A/B.C -.Ve -.PP -Output: -.PP -.Vb 1 -\& A/B.C -.Ve -.PP -The default replacement string is {}: -.PP -.Vb 1 -\& parallel echo {} ::: A/B.C -.Ve -.PP -Output: -.PP -.Vb 1 -\& A/B.C -.Ve -.PP -The replacement string {.} removes the extension: -.PP -.Vb 1 -\& parallel echo {.} ::: A/B.C -.Ve -.PP -Output: -.PP -.Vb 1 -\& A/B -.Ve -.PP -The replacement string {/} removes the path: -.PP -.Vb 1 -\& parallel echo {/} ::: A/B.C -.Ve -.PP -Output: -.PP -.Vb 1 -\& B.C -.Ve -.PP -The replacement string {//} keeps only the path: -.PP -.Vb 1 -\& parallel echo {//} ::: A/B.C -.Ve -.PP -Output: -.PP -.Vb 1 -\& A -.Ve -.PP -The replacement string {/.} removes the path and the extension: -.PP -.Vb 1 -\& parallel echo {/.} ::: A/B.C -.Ve -.PP -Output: -.PP -.Vb 1 -\& B -.Ve -.PP -The replacement string {#} gives the job number: -.PP -.Vb 1 -\& parallel echo {#} ::: A B C -.Ve -.PP -Output (the order may be different): -.PP -.Vb 3 -\& 1 -\& 2 -\& 3 -.Ve -.PP -The replacement string {%} gives the job slot number (between 1 and -number of jobs to run in parallel): -.PP -.Vb 1 -\& parallel \-j 2 echo {%} ::: A B C -.Ve -.PP -Output (the order may be different): -.PP -.Vb 3 -\& 1 -\& 2 -\& 1 -.Ve -.PP -\fIChanging the replacement strings\fR -.IX Subsection "Changing the replacement strings" -.PP -The replacement string {} can be changed with \-I: -.PP -.Vb 1 -\& parallel \-I ,, echo ,, ::: A/B.C -.Ve -.PP -Output: -.PP -.Vb 1 -\& A/B.C -.Ve -.PP -The replacement string {.} can be changed with \-\-extensionreplace: -.PP -.Vb 1 -\& parallel \-\-extensionreplace ,, echo ,, ::: A/B.C -.Ve -.PP -Output: -.PP -.Vb 1 -\& A/B -.Ve -.PP -The replacement string {/} can be replaced with \-\-basenamereplace: -.PP -.Vb 1 -\& parallel \-\-basenamereplace ,, echo ,, ::: A/B.C -.Ve -.PP -Output: -.PP -.Vb 1 -\& B.C -.Ve -.PP -The replacement string {//} can be changed with \-\-dirnamereplace: -.PP -.Vb 1 -\& parallel \-\-dirnamereplace ,, echo ,, ::: A/B.C -.Ve -.PP -Output: -.PP -.Vb 1 -\& A -.Ve -.PP -The replacement string {/.} can be changed with \-\-basenameextensionreplace: -.PP -.Vb 1 -\& parallel \-\-basenameextensionreplace ,, echo ,, ::: A/B.C -.Ve -.PP -Output: -.PP -.Vb 1 -\& B -.Ve -.PP -The replacement string {#} can be changed with \-\-seqreplace: -.PP -.Vb 1 -\& parallel \-\-seqreplace ,, echo ,, ::: A B C -.Ve -.PP -Output (the order may be different): -.PP -.Vb 3 -\& 1 -\& 2 -\& 3 -.Ve -.PP -The replacement string {%} can be changed with \-\-slotreplace: -.PP -.Vb 1 -\& parallel \-j2 \-\-slotreplace ,, echo ,, ::: A B C -.Ve -.PP -Output (the order may be different): -.PP -.Vb 3 -\& 1 -\& 2 -\& 1 -.Ve -.PP -\fIPerl expression replacement string\fR -.IX Subsection "Perl expression replacement string" -.PP -When predefined replacement strings are not flexible enough a perl -expression can be used instead. One example is to remove two -extensions: foo.tar.gz \-> foo -.PP -.Vb 1 -\& parallel echo \*(Aq{= s:\e.[^.]+$::;s:\e.[^.]+$::; =}\*(Aq ::: foo.tar.gz -.Ve -.PP -Output: -.PP -.Vb 1 -\& foo -.Ve -.PP -If the strings \fB{=\fR and \fB=}\fR cause problems they can be replaced with \-\-parens: -.PP -.Vb 1 -\& parallel \-\-parens ,,,, echo \*(Aq,, s:\e.[^.]+$::;s:\e.[^.]+$::; ,,\*(Aq ::: foo.tar.gz -.Ve -.PP -Output: Same as above. -.PP -To define a short hand replacement string use \fB\-\-rpl\fR: -.PP -.Vb 1 -\& parallel \-\-rpl \*(Aq.. s:\e.[^.]+$::;s:\e.[^.]+$::;\*(Aq echo \*(Aq..\*(Aq ::: foo.tar.gz -.Ve -.PP -Output: Same as above. -.PP -If the short hand starts with '{' it can be used as a positional -replacement string, too: -.PP -.Vb 1 -\& parallel \-\-rpl \*(Aq{..} s:\e.[^.]+$::;s:\e.[^.]+$::;\*(Aq echo \*(Aq{..}\*(Aq ::: foo.tar.gz -.Ve -.PP -Output: Same as above. -.PP -\&\s-1GNU \s0\fBparallel\fR's 7 replacement strings are implemented as: -.PP -.Vb 7 -\& \-\-rpl \*(Aq{} \*(Aq -\& \-\-rpl \*(Aq{#} $_=$job\->seq()\*(Aq -\& \-\-rpl \*(Aq{%} $_=$job\->slot()\*(Aq -\& \-\-rpl \*(Aq{/} s:.*/::\*(Aq -\& \-\-rpl \*(Aq{//} $Global::use{"File::Basename"} ||= eval "use File::Basename; 1;"; $_ = dirname($_);\*(Aq -\& \-\-rpl \*(Aq{/.} s:.*/::; s:\e.[^/.]+$::;\*(Aq -\& \-\-rpl \*(Aq{.} s:\e.[^/.]+$::\*(Aq -.Ve -.PP -\fIPositional replacement strings\fR -.IX Subsection "Positional replacement strings" -.PP -With multiple input sources the argument from the individual input -sources can be access with {number}: -.PP -.Vb 1 -\& parallel echo {1} and {2} ::: A B ::: C D -.Ve -.PP -Output (the order may be different): -.PP -.Vb 4 -\& A and C -\& A and D -\& B and C -\& B and D -.Ve -.PP -The positional replacement strings can also be modified using / // /. and .: -.PP -.Vb 1 -\& parallel echo /={1/} //={1//} /.={1/.} .={1.} ::: A/B.C D/E.F -.Ve -.PP -Output (the order may be different): -.PP -.Vb 2 -\& /=B.C //=A /.=B .=A/B -\& /=E.F //=D /.=E .=D/E -.Ve -.PP -If a position is negative, it will refer to the input source counted -from behind: -.PP -.Vb 1 -\& parallel echo 1={1} 2={2} 3={3} \-1={\-1} \-2={\-2} \-3={\-3} ::: A B ::: C D ::: E F -.Ve -.PP -Output (the order may be different): -.PP -.Vb 8 -\& 1=A 2=C 3=E \-1=E \-2=C \-3=A -\& 1=A 2=C 3=F \-1=F \-2=C \-3=A -\& 1=A 2=D 3=E \-1=E \-2=D \-3=A -\& 1=A 2=D 3=F \-1=F \-2=D \-3=A -\& 1=B 2=C 3=E \-1=E \-2=C \-3=B -\& 1=B 2=C 3=F \-1=F \-2=C \-3=B -\& 1=B 2=D 3=E \-1=E \-2=D \-3=B -\& 1=B 2=D 3=F \-1=F \-2=D \-3=B -.Ve -.PP -\fIPositional perl expression replacement string\fR -.IX Subsection "Positional perl expression replacement string" -.PP -To use a perl expression as a positional replacement string simply -prepend the perl expression with number and space: -.PP -.Vb 1 -\& parallel echo \*(Aq{=2 s:\e.[^.]+$::;s:\e.[^.]+$::; =} {1}\*(Aq ::: bar ::: foo.tar.gz -.Ve -.PP -Output: -.PP -.Vb 1 -\& foo bar -.Ve -.PP -If a defined short hand starts with '{' it can be used as a positional -replacement string, too: -.PP -.Vb 1 -\& parallel \-\-rpl \*(Aq{..} s:\e.[^.]+$::;s:\e.[^.]+$::;\*(Aq echo \*(Aq{2..} {1}\*(Aq ::: bar ::: foo.tar.gz -.Ve -.PP -Output: Same as above. -.PP -\fIInput from columns\fR -.IX Subsection "Input from columns" -.PP -The columns in a file can be bound to positional replacement strings -using \-\-colsep. Here the columns are separated with \s-1TAB \s0(\et): -.PP -.Vb 1 -\& parallel \-\-colsep \*(Aq\et\*(Aq echo 1={1} 2={2} :::: tsv\-file.tsv -.Ve -.PP -Output (the order may be different): -.PP -.Vb 3 -\& 1=f1 2=f2 -\& 1=A 2=B -\& 1=C 2=D -.Ve -.PP -\fIHeader defined replacement strings\fR -.IX Subsection "Header defined replacement strings" -.PP -With \-\-header \s-1GNU\s0 Parallel will use the first value of the input -source as the name of the replacement string. Only the non-modified -version {} is supported: -.PP -.Vb 1 -\& parallel \-\-header : echo f1={f1} f2={f2} ::: f1 A B ::: f2 C D -.Ve -.PP -Output (the order may be different): -.PP -.Vb 4 -\& f1=A f2=C -\& f1=A f2=D -\& f1=B f2=C -\& f1=B f2=D -.Ve -.PP -It is useful with \-\-colsep for processing files with \s-1TAB\s0 separated values: -.PP -.Vb 1 -\& parallel \-\-header : \-\-colsep \*(Aq\et\*(Aq echo f1={f1} f2={f2} :::: tsv\-file.tsv -.Ve -.PP -Output (the order may be different): -.PP -.Vb 2 -\& f1=A f2=B -\& f1=C f2=D -.Ve -.SS "More than one argument" -.IX Subsection "More than one argument" -With \-\-xargs will \s-1GNU\s0 Parallel fit as many arguments as possible on a -single line: -.PP -.Vb 1 -\& cat num30000 | parallel \-\-xargs echo | wc \-l -.Ve -.PP -Output: -.PP -.Vb 1 -\& 2 -.Ve -.PP -The 30000 arguments fitted on 2 lines. -.PP -The maximal length of a single line can be set with \-s. With a maximal -line length of 10000 chars 17 commands will be run: -.PP -.Vb 1 -\& cat num30000 | parallel \-\-xargs \-s 10000 echo | wc \-l -.Ve -.PP -Output: -.PP -.Vb 1 -\& 17 -.Ve -.PP -For better parallelism \s-1GNU\s0 Parallel can distribute the arguments -between all the parallel jobs when end of file is met. -.PP -Below \s-1GNU\s0 Parallel reads the last argument when generating the second -job. When \s-1GNU\s0 Parallel reads the last argument, it spreads all the -arguments for the second job over 4 jobs instead, as 4 parallel jobs -are requested. -.PP -The first job will be the same as the \-\-xargs example above, but the -second job will be split into 4 evenly sized jobs, resulting in a -total of 5 jobs: -.PP -.Vb 1 -\& cat num30000 | parallel \-\-jobs 4 \-m echo | wc \-l -.Ve -.PP -Output: -.PP -.Vb 1 -\& 5 -.Ve -.PP -This is even more visible when running 4 jobs with 10 arguments. The -10 arguments are being spread over 4 jobs: -.PP -.Vb 1 -\& parallel \-\-jobs 4 \-m echo ::: {1..10} -.Ve -.PP -Output: -.PP -.Vb 4 -\& 1 2 3 -\& 4 5 6 -\& 7 8 9 -\& 10 -.Ve -.PP -A replacement string can be part of a word. \-m will not repeat the context: -.PP -.Vb 1 -\& parallel \-\-jobs 4 \-m echo pre\-{}\-post ::: A B C D E F G -.Ve -.PP -Output (the order may be different): -.PP -.Vb 4 -\& pre\-A B\-post -\& pre\-C D\-post -\& pre\-E F\-post -\& pre\-G\-post -.Ve -.PP -To repeat the context use \-X which otherwise works like \-m: -.PP -.Vb 1 -\& parallel \-\-jobs 4 \-X echo pre\-{}\-post ::: A B C D E F G -.Ve -.PP -Output (the order may be different): -.PP -.Vb 4 -\& pre\-A\-post pre\-B\-post -\& pre\-C\-post pre\-D\-post -\& pre\-E\-post pre\-F\-post -\& pre\-G\-post -.Ve -.PP -To limit the number of arguments use \-N: -.PP -.Vb 1 -\& parallel \-N3 echo ::: A B C D E F G H -.Ve -.PP -Output (the order may be different): -.PP -.Vb 3 -\& A B C -\& D E F -\& G H -.Ve -.PP -\&\-N also sets the positional replacement strings: -.PP -.Vb 1 -\& parallel \-N3 echo 1={1} 2={2} 3={3} ::: A B C D E F G H -.Ve -.PP -Output (the order may be different): -.PP -.Vb 3 -\& 1=A 2=B 3=C -\& 1=D 2=E 3=F -\& 1=G 2=H 3= -.Ve -.PP -\&\-N0 reads 1 argument but inserts none: -.PP -.Vb 1 -\& parallel \-N0 echo foo ::: 1 2 3 -.Ve -.PP -Output: -.PP -.Vb 3 -\& foo -\& foo -\& foo -.Ve -.SS "Quoting" -.IX Subsection "Quoting" -Command lines that contain special characters may need to be protected from the shell. -.PP -The perl program 'print \*(L"@ARGV\en\*(R"' basically works like echo. -.PP -.Vb 1 -\& perl \-e \*(Aqprint "@ARGV\en"\*(Aq A -.Ve -.PP -Output: -.PP -.Vb 1 -\& A -.Ve -.PP -To run that in parallel the command needs to be quoted: -.PP -.Vb 1 -\& parallel perl \-e \*(Aqprint "@ARGV\en"\*(Aq ::: This wont work -.Ve -.PP -Output: -.PP -.Vb 1 -\& [Nothing] -.Ve -.PP -To quote the command use \-q: -.PP -.Vb 1 -\& parallel \-q perl \-e \*(Aqprint "@ARGV\en"\*(Aq ::: This works -.Ve -.PP -Output (the order may be different): -.PP -.Vb 2 -\& This -\& works -.Ve -.PP -Or you can quote the critical part using \e': -.PP -.Vb 1 -\& parallel perl \-e \e\*(Aq\*(Aqprint "@ARGV\en"\*(Aq\e\*(Aq ::: This works, too -.Ve -.PP -Output (the order may be different): -.PP -.Vb 3 -\& This -\& works, -\& too -.Ve -.PP -\&\s-1GNU\s0 Parallel can also \e\-quote full lines. Simply run: -.PP -.Vb 4 -\& parallel \-\-shellquote -\& parallel: Warning: Input is read from the terminal. Only experts do this on purpose. Press CTRL\-D to exit. -\& perl \-e \*(Aqprint "@ARGV\en"\*(Aq -\& [CTRL\-D] -.Ve -.PP -Output: -.PP -.Vb 1 -\& perl\e \-e\e \e\*(Aqprint\e \e"@ARGV\e\en\e"\e\*(Aq -.Ve -.PP -This can then be used as the command: -.PP -.Vb 1 -\& parallel perl\e \-e\e \e\*(Aqprint\e \e"@ARGV\e\en\e"\e\*(Aq ::: This also works -.Ve -.PP -Output (the order may be different): -.PP -.Vb 3 -\& This -\& also -\& works -.Ve -.SS "Trimming space" -.IX Subsection "Trimming space" -Space can be trimmed on the arguments using \-\-trim: -.PP -.Vb 1 -\& parallel \-\-trim r echo pre\-{}\-post ::: \*(Aq A \*(Aq -.Ve -.PP -Output: -.PP -.Vb 1 -\& pre\- A\-post -.Ve -.PP -To trim on the left side: -.PP -.Vb 1 -\& parallel \-\-trim l echo pre\-{}\-post ::: \*(Aq A \*(Aq -.Ve -.PP -Output: -.PP -.Vb 1 -\& pre\-A \-post -.Ve -.PP -To trim on the both sides: -.PP -.Vb 1 -\& parallel \-\-trim lr echo pre\-{}\-post ::: \*(Aq A \*(Aq -.Ve -.PP -Output: -.PP -.Vb 1 -\& pre\-A\-post -.Ve -.SH "Controling the output" -.IX Header "Controling the output" -The output can prefixed with the argument: -.PP -.Vb 1 -\& parallel \-\-tag echo foo\-{} ::: A B C -.Ve -.PP -Output (the order may be different): -.PP -.Vb 3 -\& A foo\-A -\& B foo\-B -\& C foo\-C -.Ve -.PP -To prefix it with another string use \-\-tagstring: -.PP -.Vb 1 -\& parallel \-\-tagstring {}\-bar echo foo\-{} ::: A B C -.Ve -.PP -Output (the order may be different): -.PP -.Vb 3 -\& A\-bar foo\-A -\& B\-bar foo\-B -\& C\-bar foo\-C -.Ve -.PP -To see what commands will be run without running them: -.PP -.Vb 1 -\& parallel \-\-dryrun echo {} ::: A B C -.Ve -.PP -Output (the order may be different): -.PP -.Vb 3 -\& echo A -\& echo B -\& echo C -.Ve -.PP -To print the command before running them use \-\-verbose: -.PP -.Vb 1 -\& parallel \-\-verbose echo {} ::: A B C -.Ve -.PP -Output (the order may be different): -.PP -.Vb 6 -\& echo A -\& echo B -\& A -\& echo C -\& B -\& C -.Ve -.PP -\&\s-1GNU\s0 Parallel will postpone the output until the command completes: -.PP -.Vb 1 -\& parallel \-j2 \*(Aqprintf "%s\-start\en%s" {} {};sleep {};printf "%s\en" \-middle;echo {}\-end\*(Aq ::: 4 2 1 -.Ve -.PP -Output: -.PP -.Vb 9 -\& 2\-start -\& 2\-middle -\& 2\-end -\& 1\-start -\& 1\-middle -\& 1\-end -\& 4\-start -\& 4\-middle -\& 4\-end -.Ve -.PP -To get the output immediately use \-\-ungroup: -.PP -.Vb 1 -\& parallel \-j2 \-\-ungroup \*(Aqprintf "%s\-start\en%s" {} {};sleep {};printf "%s\en" \-middle;echo {}\-end\*(Aq ::: 4 2 1 -.Ve -.PP -Output: -.PP -.Vb 9 -\& 4\-start -\& 42\-start -\& 2\-middle -\& 2\-end -\& 1\-start -\& 1\-middle -\& 1\-end -\& \-middle -\& 4\-end -.Ve -.PP -\&\-\-ungroup is fast, but can cause half a line from one job to be mixed -with half a line of another job. That has happend in the second line, -where the line '4\-middle' is mixed with '2\-start'. -.PP -To avoid this use \-\-linebuffer: -.PP -.Vb 1 -\& parallel \-j2 \-\-linebuffer \*(Aqprintf "%s\-start\en%s" {} {};sleep {};printf "%s\en" \-middle;echo {}\-end\*(Aq ::: 4 2 1 -.Ve -.PP -Output: -.PP -.Vb 9 -\& 4\-start -\& 2\-start -\& 2\-middle -\& 2\-end -\& 1\-start -\& 1\-middle -\& 1\-end -\& 4\-middle -\& 4\-end -.Ve -.PP -To force the output in the same order as the arguments use \-\-keep\-order/\-k: -.PP -.Vb 1 -\& parallel \-j2 \-k \*(Aqprintf "%s\-start\en%s" {} {};sleep {};printf "%s\en" \-middle;echo {}\-end\*(Aq ::: 4 2 1 -.Ve -.PP -Output: -.PP -.Vb 9 -\& 4\-start -\& 4\-middle -\& 4\-end -\& 2\-start -\& 2\-middle -\& 2\-end -\& 1\-start -\& 1\-middle -\& 1\-end -.Ve -.SS "Saving output into files" -.IX Subsection "Saving output into files" -\&\s-1GNU\s0 Parallel can save the output of each job into files: -.PP -.Vb 1 -\& parallel \-\-files echo ::: A B C -.Ve -.PP -Output will be similar to: -.PP -.Vb 3 -\& /tmp/pAh6uWuQCg.par -\& /tmp/opjhZCzAX4.par -\& /tmp/W0AT_Rph2o.par -.Ve -.PP -By default \s-1GNU\s0 Parallel will cache the output in files in /tmp. This -can be changed by setting \f(CW$TMPDIR\fR or \-\-tmpdir: -.PP -.Vb 1 -\& parallel \-\-tmpdir /var/tmp \-\-files echo ::: A B C -.Ve -.PP -Output will be similar to: -.PP -.Vb 3 -\& /var/tmp/N_vk7phQRc.par -\& /var/tmp/7zA4Ccf3wZ.par -\& /var/tmp/LIuKgF_2LP.par -.Ve -.PP -Or: -.PP -.Vb 1 -\& TMPDIR=/var/tmp parallel \-\-files echo ::: A B C -.Ve -.PP -Output: Same as above. -.PP -The output files can be saved in a structured way using \-\-results: -.PP -.Vb 1 -\& parallel \-\-results outdir echo ::: A B C -.Ve -.PP -Output: -.PP -.Vb 3 -\& A -\& B -\& C -.Ve -.PP -but also these files were generated containing the standard output -(stdout) and standard error (stderr): -.PP -.Vb 6 -\& outdir/1/A/stderr -\& outdir/1/A/stdout -\& outdir/1/B/stderr -\& outdir/1/B/stdout -\& outdir/1/C/stderr -\& outdir/1/C/stdout -.Ve -.PP -This is useful if you are running multiple variables: -.PP -.Vb 1 -\& parallel \-\-header : \-\-results outdir echo ::: f1 A B ::: f2 C D -.Ve -.PP -Generated files: -.PP -.Vb 8 -\& outdir/f1/A/f2/C/stderr -\& outdir/f1/A/f2/C/stdout -\& outdir/f1/A/f2/D/stderr -\& outdir/f1/A/f2/D/stdout -\& outdir/f1/B/f2/C/stderr -\& outdir/f1/B/f2/C/stdout -\& outdir/f1/B/f2/D/stderr -\& outdir/f1/B/f2/D/stdout -.Ve -.PP -The directories are named after the variables and their values. -.SH "Control the execution" -.IX Header "Control the execution" -.SS "Number of simultaneous jobs" -.IX Subsection "Number of simultaneous jobs" -The number of concurrent jobs is given with \-\-jobs/\-j: -.PP -.Vb 1 -\& /usr/bin/time parallel \-N0 \-j64 sleep 1 ::: {1..128} -.Ve -.PP -With 64 jobs in parallel the 128 sleeps will take 2\-8 seconds to run \- -depending on how fast your machine is. -.PP -By default \-\-jobs is the same as the number of \s-1CPU\s0 cores. So this: -.PP -.Vb 1 -\& /usr/bin/time parallel \-N0 sleep 1 ::: {1..128} -.Ve -.PP -should take twice the time of running 2 jobs per \s-1CPU\s0 core: -.PP -.Vb 1 -\& /usr/bin/time parallel \-N0 \-\-jobs 200% sleep 1 ::: {1..128} -.Ve -.PP -\&\-\-jobs 0 will run as many jobs in parallel as possible: -.PP -.Vb 1 -\& /usr/bin/time parallel \-N0 \-\-jobs 0 sleep 1 ::: {1..128} -.Ve -.PP -which should take 1\-7 seconds depending on how fast your machine is. -.PP -\&\-\-jobs can read from a file which is re-read when a job finishes: -.PP -.Vb 5 -\& echo 50% > my_jobs -\& /usr/bin/time parallel \-N0 \-\-jobs my_jobs sleep 1 ::: {1..128} & -\& sleep 1 -\& echo 0 > my_jobs -\& wait -.Ve -.PP -The first second only 50% of the \s-1CPU\s0 cores will run a job. The '0' is -put into my_jobs and then the rest of the jobs will be started in -parallel. -.PP -Instead of basing the percentage on the number of \s-1CPU\s0 cores -\&\s-1GNU\s0 Parallel can base it on the number of CPUs: -.PP -.Vb 1 -\& parallel \-\-use\-cpus\-instead\-of\-cores \-N0 sleep 1 ::: {1..128} -.Ve -.SS "Interactivity" -.IX Subsection "Interactivity" -\&\s-1GNU\s0 Parallel can ask the user if a command should be run using \-\-interactive: -.PP -.Vb 1 -\& parallel \-\-interactive echo ::: 1 2 3 -.Ve -.PP -Output: -.PP -.Vb 5 -\& echo 1 ?...y -\& echo 2 ?...n -\& 1 -\& echo 3 ?...y -\& 3 -.Ve -.PP -\&\s-1GNU\s0 Parallel can be used to put arguments on the command line for an -interactive command such as emacs to edit one file at a time: -.PP -.Vb 1 -\& parallel \-\-tty emacs ::: 1 2 3 -.Ve -.PP -Or give multiple argument in one go to open multiple files: -.PP -.Vb 1 -\& parallel \-X \-\-tty vi ::: 1 2 3 -.Ve -.SS "Timing" -.IX Subsection "Timing" -Some jobs do heavy I/O when they start. To avoid a thundering herd \s-1GNU\s0 -Parallel can delay starting new jobs. \-\-delay X will make sure there is -at least X seconds between each start: -.PP -.Vb 1 -\& parallel \-\-delay 2.5 echo Starting {}\e;date ::: 1 2 3 -.Ve -.PP -Output: -.PP -.Vb 6 -\& Starting 1 -\& Thu Aug 15 16:24:33 CEST 2013 -\& Starting 2 -\& Thu Aug 15 16:24:35 CEST 2013 -\& Starting 3 -\& Thu Aug 15 16:24:38 CEST 2013 -.Ve -.PP -If jobs taking more than a certain amount of time are known to fail, -they can be stopped with \-\-timeout: -.PP -.Vb 1 -\& parallel \-\-timeout 2.1 sleep {}\e; echo {} ::: 1 2 3 4 -.Ve -.PP -Output: -.PP -.Vb 2 -\& 1 -\& 2 -.Ve -.PP -\&\s-1GNU\s0 Parallel can compute the median runtime for jobs and kill those -that take more than 200% of the median runtime: -.PP -.Vb 1 -\& parallel \-\-timeout 200% sleep {}\e; echo {} ::: 2.1 2.2 3 7 2.3 -.Ve -.PP -Output: -.PP -.Vb 4 -\& 2.1 -\& 2.2 -\& 3 -\& 2.3 -.Ve -.PP -Based on the runtime of completed jobs \s-1GNU\s0 Parallel can estimate the -total runtime: -.PP -.Vb 1 -\& parallel \-\-eta sleep ::: 1 3 2 2 1 3 3 2 1 -.Ve -.PP -Output: -.PP -.Vb 2 -\& Computers / CPU cores / Max jobs to run -\& 1:local / 2 / 2 -\& -\& Computer:jobs running/jobs completed/%of started jobs/Average seconds to complete -\& ETA: 2s 0left 1.11avg local:0/9/100%/1.1s -.Ve -.SS "Progress" -.IX Subsection "Progress" -\&\s-1GNU\s0 Parallel can give progress information with \-\-progress: -.PP -.Vb 1 -\& parallel \-\-progress sleep ::: 1 3 2 2 1 3 3 2 1 -.Ve -.PP -Output: -.PP -.Vb 2 -\& Computers / CPU cores / Max jobs to run -\& 1:local / 2 / 2 -\& -\& Computer:jobs running/jobs completed/%of started jobs/Average seconds to complete -\& local:0/9/100%/1.1s -.Ve -.PP -A logfile of the jobs completed so far can be generated with \-\-joblog: -.PP -.Vb 2 -\& parallel \-\-joblog /tmp/log exit ::: 1 2 3 0 -\& cat /tmp/log -.Ve -.PP -Output: -.PP -.Vb 5 -\& Seq Host Starttime Runtime Send Receive Exitval Signal Command -\& 1 : 1376577364.974 0.008 0 0 1 0 exit 1 -\& 2 : 1376577364.982 0.013 0 0 2 0 exit 2 -\& 3 : 1376577364.990 0.013 0 0 3 0 exit 3 -\& 4 : 1376577365.003 0.003 0 0 0 0 exit 0 -.Ve -.PP -The log contains the job sequence, which host the job was run on, the -start time and run time, how much data was transferred if the job was -run on a remote host, the exit value, the signal that killed the job, -and finally the command being run. -.PP -With a joblog \s-1GNU\s0 Parallel can be stopped and later pickup where it -left off. It it important that the input of the completed jobs is -unchanged. -.PP -.Vb 4 -\& parallel \-\-joblog /tmp/log exit ::: 1 2 3 0 -\& cat /tmp/log -\& parallel \-\-resume \-\-joblog /tmp/log exit ::: 1 2 3 0 0 0 -\& cat /tmp/log -.Ve -.PP -Output: -.PP -.Vb 5 -\& Seq Host Starttime Runtime Send Receive Exitval Signal Command -\& 1 : 1376580069.544 0.008 0 0 1 0 exit 1 -\& 2 : 1376580069.552 0.009 0 0 2 0 exit 2 -\& 3 : 1376580069.560 0.012 0 0 3 0 exit 3 -\& 4 : 1376580069.571 0.005 0 0 0 0 exit 0 -\& -\& Seq Host Starttime Runtime Send Receive Exitval Signal Command -\& 1 : 1376580069.544 0.008 0 0 1 0 exit 1 -\& 2 : 1376580069.552 0.009 0 0 2 0 exit 2 -\& 3 : 1376580069.560 0.012 0 0 3 0 exit 3 -\& 4 : 1376580069.571 0.005 0 0 0 0 exit 0 -\& 5 : 1376580070.028 0.009 0 0 0 0 exit 0 -\& 6 : 1376580070.038 0.007 0 0 0 0 exit 0 -.Ve -.PP -Note how the start time of the last 2 jobs is clearly from the second run. -.PP -With \-\-resume\-failed \s-1GNU\s0 Parallel will re-run the jobs that failed: -.PP -.Vb 2 -\& parallel \-\-resume\-failed \-\-joblog /tmp/log exit ::: 1 2 3 0 0 0 -\& cat /tmp/log -.Ve -.PP -Output: -.PP -.Vb 10 -\& Seq Host Starttime Runtime Send Receive Exitval Signal Command -\& 1 : 1376580069.544 0.008 0 0 1 0 exit 1 -\& 2 : 1376580069.552 0.009 0 0 2 0 exit 2 -\& 3 : 1376580069.560 0.012 0 0 3 0 exit 3 -\& 4 : 1376580069.571 0.005 0 0 0 0 exit 0 -\& 5 : 1376580070.028 0.009 0 0 0 0 exit 0 -\& 6 : 1376580070.038 0.007 0 0 0 0 exit 0 -\& 1 : 1376580154.433 0.010 0 0 1 0 exit 1 -\& 2 : 1376580154.444 0.022 0 0 2 0 exit 2 -\& 3 : 1376580154.466 0.005 0 0 3 0 exit 3 -.Ve -.PP -Note how seq 1 2 3 have been repeated because they had exit value != 0. -.SS "Termination" -.IX Subsection "Termination" -For certain jobs there is no need to continue if one of the jobs fails -and has an exit code != 0. \s-1GNU\s0 Parallel will stop spawning new jobs -with \-\-halt 1: -.PP -.Vb 1 -\& parallel \-j2 \-\-halt 1 echo {}\e; exit {} ::: 0 0 1 2 3 -.Ve -.PP -Output: -.PP -.Vb 8 -\& 0 -\& 0 -\& 1 -\& parallel: Starting no more jobs. Waiting for 2 jobs to finish. This job failed: -\& echo 1; exit 1 -\& 2 -\& parallel: Starting no more jobs. Waiting for 1 jobs to finish. This job failed: -\& echo 2; exit 2 -.Ve -.PP -With \-\-halt 2 the running jobs will be killed immediately: -.PP -.Vb 1 -\& parallel \-j2 \-\-halt 2 echo {}\e; exit {} ::: 0 0 1 2 3 -.Ve -.PP -Output: -.PP -.Vb 5 -\& 0 -\& 0 -\& 1 -\& parallel: This job failed: -\& echo 1; exit 1 -.Ve -.PP -If \-\-halt is given a percentage this percentage of the jobs must fail -(though minimum 3) before \s-1GNU\s0 Parallel stops spawning more jobs: -.PP -.Vb 1 -\& parallel \-j2 \-\-halt 20% echo {}\e; exit {} ::: 0 0 1 2 3 4 5 6 7 -.Ve -.PP -Output: -.PP -.Vb 11 -\& 0 -\& 0 -\& 1 -\& 2 -\& 3 -\& 4 -\& parallel: Starting no more jobs. Waiting for 2 jobs to finish. This job failed: -\& echo 4; exit 4 -\& 5 -\& parallel: Starting no more jobs. Waiting for 1 jobs to finish. This job failed: -\& echo 5; exit 5 -.Ve -.PP -\&\s-1GNU\s0 Parallel can retry the command with \-\-retries. This is useful if a -command fails for unknown reasons now and then. -.PP -.Vb 2 -\& parallel \-k \-\-retries 3 \*(Aqecho tried {} >>/tmp/runs; echo completed {}; exit {}\*(Aq ::: 1 2 0 -\& cat /tmp/runs -.Ve -.PP -Output: -.PP -.Vb 3 -\& completed 1 -\& completed 2 -\& completed 0 -\& -\& tried 1 -\& tried 2 -\& tried 1 -\& tried 2 -\& tried 1 -\& tried 2 -\& tried 0 -.Ve -.PP -Note how job 1 and 2 were tried 3 times, but 0 was not retried because it had exit code 0. -.SS "Limiting the resources" -.IX Subsection "Limiting the resources" -To avoid overloading systems \s-1GNU\s0 Parallel can look at the system load -before starting another job: -.PP -.Vb 1 -\& parallel \-\-load 100% echo load is less than {} job per cpu ::: 1 -.Ve -.PP -Output: -.PP -.Vb 2 -\& [when then load is less than the number of cpu cores] -\& load is less than 1 job per cpu -.Ve -.PP -\&\s-1GNU\s0 Parallel can also check if the system is swapping. -.PP -.Vb 1 -\& parallel \-\-noswap echo the system is not swapping ::: now -.Ve -.PP -Output: -.PP -.Vb 2 -\& [when then system is not swapping] -\& the system is not swapping now -.Ve -.PP -\&\s-1GNU\s0 Parallel can run the jobs with a nice value. This will work both -locally and remotely. -.PP -.Vb 1 -\& parallel \-\-nice 17 echo this is being run with nice \-n ::: 17 -.Ve -.PP -Output: -.PP -.Vb 1 -\& this is being run with nice \-n 17 -.Ve -.SH "Remote execution" -.IX Header "Remote execution" -\&\s-1GNU\s0 Parallel can run jobs on remote servers. It uses ssh to -communicate with the remote machines. -.SS "Sshlogin" -.IX Subsection "Sshlogin" -The most basic sshlogin is \-S host: -.PP -.Vb 1 -\& parallel \-S $SERVER1 echo running on ::: $SERVER1 -.Ve -.PP -Output: -.PP -.Vb 1 -\& running on [$SERVER1] -.Ve -.PP -To use a different username prepend the server with username@ -.PP -.Vb 1 -\& parallel \-S username@$SERVER1 echo running on ::: username@$SERVER1 -.Ve -.PP -Output: -.PP -.Vb 1 -\& running on [username@$SERVER1] -.Ve -.PP -The special sshlogin ':' is the local machine: -.PP -.Vb 1 -\& parallel \-S : echo running on ::: the_local_machine -.Ve -.PP -Output: -.PP -.Vb 1 -\& running on the_local_machine -.Ve -.PP -If ssh is not in \f(CW$PATH\fR it can be prepended to \f(CW$SERVER1:\fR -.PP -.Vb 1 -\& parallel \-S \*(Aq/usr/bin/ssh \*(Aq$SERVER1 echo custom ::: ssh -.Ve -.PP -Output: -.PP -.Vb 1 -\& custom ssh -.Ve -.PP -Several servers can be given using multiple \-S: -.PP -.Vb 1 -\& parallel \-S $SERVER1 \-S $SERVER2 echo ::: running on more hosts -.Ve -.PP -Output (the order may be different): -.PP -.Vb 4 -\& running -\& on -\& more -\& hosts -.Ve -.PP -Or they can be separated by ,: -.PP -.Vb 1 -\& parallel \-S $SERVER1,$SERVER2 echo ::: running on more hosts -.Ve -.PP -Output: Same as above. -.PP -The can also be read from a file (replace user@ with the user on \f(CW$SERVER2\fR): -.PP -.Vb 4 -\& echo $SERVER1 > nodefile -\& # Force 4 cores, special ssh\-command, username -\& echo 4//usr/bin/ssh user@$SERVER2 >> nodefile -\& parallel \-\-sshloginfile nodefile echo ::: running on more hosts -.Ve -.PP -Output: Same as above. -.PP -The special \-\-sshloginfile '..' reads from ~/.parallel/sshloginfile. -.PP -To force \s-1GNU\s0 Parallel to treat a server having a given number of \s-1CPU\s0 -cores prepend #/ to the sshlogin: -.PP -.Vb 1 -\& parallel \-S 4/$SERVER1 echo force {} cpus on server ::: 4 -.Ve -.PP -Output: -.PP -.Vb 1 -\& force 4 cpus on server -.Ve -.SS "Transferring files" -.IX Subsection "Transferring files" -\&\s-1GNU\s0 Parallel can transfer the files to be processed to the remote -host. It does that using rsync. -.PP -.Vb 2 -\& echo This is input_file > input_file -\& parallel \-S $SERVER1 \-\-transfer cat ::: input_file -.Ve -.PP -Output: -.PP -.Vb 1 -\& This is input_file -.Ve -.PP -If the files is processed into another file, the resulting file can be -transferred back: -.PP -.Vb 3 -\& echo This is input_file > input_file -\& parallel \-S $SERVER1 \-\-transfer \-\-return {}.out cat {} ">"{}.out ::: input_file -\& cat input_file.out -.Ve -.PP -Output: Same as above. -.PP -To remove the input and output file on the remote server use \-\-cleanup: -.PP -.Vb 3 -\& echo This is input_file > input_file -\& parallel \-S $SERVER1 \-\-transfer \-\-return {}.out \-\-cleanup cat {} ">"{}.out ::: input_file -\& cat input_file.out -.Ve -.PP -Output: Same as above. -.PP -There is a short hand for \-\-transfer \-\-return \-\-cleanup called \-\-trc: -.PP -.Vb 3 -\& echo This is input_file > input_file -\& parallel \-S $SERVER1 \-\-trc {}.out cat {} ">"{}.out ::: input_file -\& cat input_file.out -.Ve -.PP -Output: Same as above. -.PP -Some jobs need a common database for all jobs. \s-1GNU\s0 Parallel can -transfer that using \-\-basefile which will transfer the file before the -first job: -.PP -.Vb 2 -\& echo common data > common_file -\& parallel \-\-basefile common_file \-S $SERVER1 cat common_file\e; echo {} ::: foo -.Ve -.PP -Output: -.PP -.Vb 2 -\& common data -\& foo -.Ve -.PP -To remove it from the remote host after the last job use \-\-cleanup. -.SS "Working dir" -.IX Subsection "Working dir" -The default working dir on the remote machines is the login dir. This -can be changed with \-\-workdir \fImydir\fR. -.PP -Files transferred using \-\-transfer and \-\-return will be relative -to \fImydir\fR on remote computers, and the command will be executed in -the dir \fImydir\fR. -.PP -The special \fImydir\fR value ... will create working dirs under -~/.parallel/tmp/ on the remote computers. If \-\-cleanup is given -these dirs will be removed. -.PP -The special \fImydir\fR value . uses the current working dir. If the -current working dir is beneath your home dir, the value . is -treated as the relative path to your home dir. This means that if your -home dir is different on remote computers (e.g. if your login is -different) the relative path will still be relative to your home dir. -.PP -.Vb 3 -\& parallel \-S $SERVER1 pwd ::: "" -\& parallel \-\-workdir . \-S $SERVER1 pwd ::: "" -\& parallel \-\-workdir ... \-S $SERVER1 pwd ::: "" -.Ve -.PP -Output: -.PP -.Vb 3 -\& [the login dir on $SERVER1] -\& [current dir relative on $SERVER1] -\& [a dir in ~/.parallel/tmp/...] -.Ve -.SS "Avoid overloading sshd" -.IX Subsection "Avoid overloading sshd" -If many jobs are started on the same server, sshd can be -overloaded. \s-1GNU\s0 Parallel can insert a delay between each job run on -the same server: -.PP -.Vb 1 -\& parallel \-S $SERVER1 \-\-sshdelay 0.2 echo ::: 1 2 3 -.Ve -.PP -Output (the order may be different): -.PP -.Vb 3 -\& 1 -\& 2 -\& 3 -.Ve -.PP -Sshd will be less overloaded if using \-\-controlmaster, which will -multiplex ssh connections: -.PP -.Vb 1 -\& parallel \-\-controlmaster \-S $SERVER1 echo ::: 1 2 3 -.Ve -.PP -Output: Same as above. -.SS "Ignore hosts that are down" -.IX Subsection "Ignore hosts that are down" -In clusters with many hosts a few of the are often down. \s-1GNU\s0 Parallel -can ignore those hosts. In this case the host 173.194.32.46 is down: -.PP -.Vb 1 -\& parallel \-\-filter\-hosts \-S 173.194.32.46,$SERVER1 echo ::: bar -.Ve -.PP -Output: -.PP -.Vb 1 -\& bar -.Ve -.SS "Running the same commands on all hosts" -.IX Subsection "Running the same commands on all hosts" -\&\s-1GNU\s0 Parallel can run the same command on all the hosts: -.PP -.Vb 1 -\& parallel \-\-onall \-S $SERVER1,$SERVER2 echo ::: foo bar -.Ve -.PP -Output (the order may be different): -.PP -.Vb 4 -\& foo -\& bar -\& foo -\& bar -.Ve -.PP -Often you will just want to run a single command on all hosts with out -arguments. \-\-nonall is a no argument \-\-onall: -.PP -.Vb 1 -\& parallel \-\-nonall \-S $SERVER1,$SERVER2 echo foo bar -.Ve -.PP -Output: -.PP -.Vb 2 -\& foo bar -\& foo bar -.Ve -.PP -When \-\-tag is used with \-\-nonall and \-\-onall the \-\-tagstring is the host: -.PP -.Vb 1 -\& parallel \-\-nonall \-\-tag \-S $SERVER1,$SERVER2 echo foo bar -.Ve -.PP -Output (the order may be different): -.PP -.Vb 2 -\& $SERVER1 foo bar -\& $SERVER2 foo bar -.Ve -.PP -\&\-\-jobs sets the number of servers to log in to in parallel. -.SS "Transfer environment variables and functions" -.IX Subsection "Transfer environment variables and functions" -Using \-\-env \s-1GNU\s0 Parallel can transfer an environment variable to the -remote system. -.PP -.Vb 3 -\& MYVAR=\*(Aqfoo bar\*(Aq -\& export MYVAR -\& parallel \-\-env MYVAR \-S $SERVER1 echo \*(Aq$MYVAR\*(Aq ::: baz -.Ve -.PP -Output: -.PP -.Vb 1 -\& foo bar baz -.Ve -.PP -This works for functions too if your shell is Bash: -.PP -.Vb 6 -\& # This only works in Bash -\& my_func() { -\& echo in my_func $1 -\& } -\& export \-f my_func -\& parallel \-\-env my_func \-S $SERVER1 my_func ::: baz -.Ve -.PP -Output: -.PP -.Vb 1 -\& in my_func baz -.Ve -.PP -\&\s-1GNU\s0 Parallel can copy all defined variables and functions to the -remote system. It just needs to record which ones to ignore in -~/.parallel/ignored_vars. Do that by running this once: -.PP -.Vb 2 -\& parallel \-\-record\-env -\& cat ~/.parallel/ignored_vars -.Ve -.PP -Output: -.PP -.Vb 1 -\& [list of variables to ignore \- including $PATH and $HOME] -.Ve -.PP -Now all new variables and functions defined will be copied when using -\&\-\-env _: -.PP -.Vb 7 -\& # The function is only copied if using Bash -\& my_func2() { -\& echo in my_func2 $VAR $1 -\& } -\& export \-f my_func2 -\& VAR=foo -\& export VAR -\& -\& parallel \-\-env _ \-S $SERVER1 \*(Aqecho $VAR; my_func2\*(Aq ::: bar -.Ve -.PP -Output: -.PP -.Vb 2 -\& foo -\& in my_func2 foo bar -.Ve -.SS "Showing what is actually run" -.IX Subsection "Showing what is actually run" -\&\-\-verbose will show the command that would be run on the local -machine. When a job is run on a remote machine this is wrapped with -ssh and possibly transferring files and environment variables, setting -the workdir, and setting \-\-nice value. \-vv shows all of this. -.PP -.Vb 1 -\& parallel \-vv \-S $SERVER1 echo ::: bar -.Ve -.PP -Output: -.PP -.Vb 2 -\& ssh \-tt \-oLogLevel=quiet lo \*(Aqeval \`echo $SHELL | grep "/t\e{0,1\e}csh" > /dev/null && echo setenv PARALLEL_SEQ \*(Aq$PARALLEL_SEQ\*(Aq\e; setenv PARALLEL_PID \*(Aq$PARALLEL_PID\*(Aq || echo PARALLEL_SEQ=\*(Aq$PARALLEL_SEQ\*(Aq\e;export PARALLEL_SEQ\e; PARALLEL_PID=\*(Aq$PARALLEL_PID\*(Aq\e;export PARALLEL_PID\` ;\*(Aq tty\e \e>/dev/null\e \e&\e&\e stty\e isig\e \-onlcr\e \-echo\e;echo\e bar; -\& bar -.Ve -.PP -When the command gets more complex, the output is so hard to read, that it is only useful for debugging: -.PP -.Vb 5 -\& my_func3() { -\& echo in my_func $1 > $1.out -\& } -\& export \-f my_func3 -\& parallel \-vv \-\-workdir ... \-\-nice 17 \-\-env _ \-\-trc {}.out \-S $SERVER1 my_func3 {} ::: abc\-file -.Ve -.PP -Output will be similar to: -.PP -.Vb 5 -\& ssh server mkdir \-p .parallel/tmp/hk\-31483\-1; rsync \-rlDzR \-essh ./abc\-file server:.parallel/tmp/hk\-31483\-1;ssh \-tt \-oLogLevel=quiet server \*(Aqeval \`echo $SHELL | grep "/t\e{0,1\e}csh" > /dev/null && echo setenv PARALLEL_SEQ \*(Aq$PARALLEL_SEQ\*(Aq\e; setenv PARALLEL_PID \*(Aq$PARALLEL_PID\*(Aq || echo PARALLEL_SEQ=\*(Aq$PARALLEL_SEQ\*(Aq\e;export PARALLEL_SEQ\e; PARALLEL_PID=\*(Aq$PARALLEL_PID\*(Aq\e;export PARALLEL_PID\` ;\*(Aq tty\e \e>/dev/null\e \e&\e&\e stty\e isig\e \-onlcr\e \-echo\e;mkdir\e \-p\e .parallel/tmp/hk\-31483\-1\e;\e cd\e .parallel/tmp/hk\-31483\-1\e \e&\e&\e echo\e \e$SHELL\e \e|\e grep\e \e"/t\e\e\e{0,1\e\e\e}csh\e"\e \e>\e /dev/null\e \e&\e&\e setenv\e my_func3\e \e\e\e(\e\e\e)\e\e\e \e\e\e{\e\e\e \e\e\e echo\e\e\e in\e\e\e my_func\e\e\e \e\e\e$1\e\e\e \e\e\e>\e\e\e \e\e\e$1.out\e"\*(Aq -\& \*(Aq\e"\e\e\e}\e \e&\e&\e setenv\e VAR\e foo\e \e&\e&\e setenv\e my_func2\e \e\e\e(\e\e\e)\e\e\e \e\e\e{\e\e\e \e\e\e echo\e\e\e in\e\e\e my_func2\e\e\e \e\e\e$VAR\e\e\e \e\e\e$1\e"\*(Aq -\& \*(Aq\e"\e\e\e}\e \e|\e|\e export\e my_func3=\e\e\e(\e\e\e)\e\e\e \e\e\e{\e\e\e \e\e\e echo\e\e\e in\e\e\e my_func\e\e\e \e\e\e$1\e\e\e \e\e\e>\e\e\e \e\e\e$1.out\e"\*(Aq -\& \*(Aq\e"\e\e\e}\e \e&\e&\e export\e VAR=foo\e \e&\e&\e export\e my_func2=\e\e\e(\e\e\e)\e\e\e \e\e\e{\e\e\e \e\e\e echo\e\e\e in\e\e\e my_func2\e\e\e \e\e\e$VAR\e\e\e \e\e\e$1\e"\*(Aq -\& \*(Aq\e"\e\e\e}\e \e&\e&\e eval\e my_func3\e"\e$my_func3\e"\e \e&\e&\e eval\e my_func2\e"\e$my_func2\e"\e;\e\enice\e \-n17\e /bin/bash\e \-c\e my_func3\e\e\e abc\-file;_EXIT_status=$?; mkdir \-p .; rsync \-\-rsync\-path=cd\e .parallel/tmp/hk\-31483\-1/.\e;\e rsync \-rlDzR \-essh server:abc\-file.out .;ssh server rm\e \-f\e .parallel/tmp/hk\-31483\-1/abc\-file\e;rm\e \-f\e .parallel/tmp/hk\-31483\-1/abc\-file.out\e;rm \-rf .parallel/tmp/hk\-31483\-1\e;; exit $_EXIT_status; -.Ve -.SH "\-\-pipe" -.IX Header "--pipe" -The \-\-pipe functionality puts \s-1GNU\s0 Parallel in a different mode: -Instead of treating the data on stdin (standard input) as arguments -for a command to run, the data will be sent to stdin (standard input) -of the command. -.PP -The typical situation is: -.PP -.Vb 1 -\& command_A | command_B | command_C -.Ve -.PP -where command_B is slow, and you want to speed up command_B. -.SS "Chunk size" -.IX Subsection "Chunk size" -By default \s-1GNU\s0 Parallel will start an instance of command_B, read a -chunk of 1 \s-1MB,\s0 and pass that to the instance. Then start another -instance, read another chunk, and pass that to the second instance. -.PP -.Vb 1 -\& cat num1000000 | parallel \-\-pipe wc -.Ve -.PP -Output (the order may be different): -.PP -.Vb 7 -\& 165668 165668 1048571 -\& 149797 149797 1048579 -\& 149796 149796 1048572 -\& 149797 149797 1048579 -\& 149797 149797 1048579 -\& 149796 149796 1048572 -\& 85349 85349 597444 -.Ve -.PP -The size of the chunk is not exactly 1 \s-1MB\s0 because \s-1GNU\s0 Parallel only -passes full lines \- never half a line, thus the blocksize is only -average 1 \s-1MB.\s0 You can change the block size to 2 \s-1MB\s0 with \-\-block: -.PP -.Vb 1 -\& cat num1000000 | parallel \-\-pipe \-\-block 2M wc -.Ve -.PP -Output (the order may be different): -.PP -.Vb 4 -\& 315465 315465 2097150 -\& 299593 299593 2097151 -\& 299593 299593 2097151 -\& 85349 85349 597444 -.Ve -.PP -\&\s-1GNU\s0 Parallel treats each line as a record. If the order of record is -unimportant (e.g. you need all lines processed, but you do not care -which is processed first), then you can use \-\-round\-robin. Without -\&\-\-round\-robin \s-1GNU\s0 Parallel will start a command per block; with -\&\-\-round\-robin only the requested number of jobs will be started -(\-\-jobs). The records will then be distributed between the running -jobs: -.PP -.Vb 1 -\& cat num1000000 | parallel \-\-pipe \-j4 \-\-round\-robin wc -.Ve -.PP -Output will be similar to: -.PP -.Vb 4 -\& 149797 149797 1048579 -\& 299593 299593 2097151 -\& 315465 315465 2097150 -\& 235145 235145 1646016 -.Ve -.PP -One of the 4 instances got a single record, 2 instances got 2 full -records each, and one instance got 1 full and 1 partial record. -.SS "Records" -.IX Subsection "Records" -\&\s-1GNU\s0 Parallel sees the input as records. The default record is a single -line. -.PP -Using \-N140000 \s-1GNU\s0 Parallel will read 140000 records at a time: -.PP -.Vb 1 -\& cat num1000000 | parallel \-\-pipe \-N140000 wc -.Ve -.PP -Output (the order may be different): -.PP -.Vb 8 -\& 140000 140000 868895 -\& 140000 140000 980000 -\& 140000 140000 980000 -\& 140000 140000 980000 -\& 140000 140000 980000 -\& 140000 140000 980000 -\& 140000 140000 980000 -\& 20000 20000 140001 -.Ve -.PP -Notice that the last job could not get the full 140000 lines, but only -20000 lines. -.PP -If a record is 75 lines \-L can be used: -.PP -.Vb 1 -\& cat num1000000 | parallel \-\-pipe \-L75 wc -.Ve -.PP -Output (the order may be different): -.PP -.Vb 8 -\& 165600 165600 1048095 -\& 149850 149850 1048950 -\& 149775 149775 1048425 -\& 149775 149775 1048425 -\& 149850 149850 1048950 -\& 149775 149775 1048425 -\& 85350 85350 597450 -\& 25 25 176 -.Ve -.PP -Notice \s-1GNU\s0 Parallel still reads a block of around 1 \s-1MB\s0; but instead of -passing full lines to 'wc' it passes full 75 lines at a time. This -of course does not hold for the last job (which in this case got 25 -lines). -.SS "Record separators" -.IX Subsection "Record separators" -\&\s-1GNU\s0 Parallel uses separators to determine where two record split. -.PP -\&\-\-recstart gives the string that starts a record; \-\-recend gives the -string that ends a record. The default is \-\-recend '\en' (newline). -.PP -If both \-\-recend and \-\-recstart are given, then the record will only -split if the recend string is immediately followed by the recstart -string. -.PP -Here the \-\-recend is set to ', ': -.PP -.Vb 1 -\& echo /foo, bar/, /baz, qux/, | parallel \-kN1 \-\-recend \*(Aq, \*(Aq \-\-pipe echo JOB{#}\e;cat\e;echo END -.Ve -.PP -Output: -.PP -.Vb 9 -\& JOB1 -\& /foo, END -\& JOB2 -\& bar/, END -\& JOB3 -\& /baz, END -\& JOB4 -\& qux/, -\& END -.Ve -.PP -Here the \-\-recstart is set to '/': -.PP -.Vb 1 -\& echo /foo, bar/, /baz, qux/, | parallel \-kN1 \-\-recstart \*(Aq/\*(Aq \-\-pipe echo JOB{#}\e;cat\e;echo END -.Ve -.PP -Output: -.PP -.Vb 9 -\& JOB1 -\& /foo, barEND -\& JOB2 -\& /, END -\& JOB3 -\& /baz, quxEND -\& JOB4 -\& /, -\& END -.Ve -.PP -Here both \-\-recend and \-\-recstart are set: -.PP -.Vb 1 -\& echo /foo, bar/, /baz, qux/, | parallel \-kN1 \-\-recend \*(Aq, \*(Aq \-\-recstart \*(Aq/\*(Aq \-\-pipe echo JOB{#}\e;cat\e;echo END -.Ve -.PP -Output: -.PP -.Vb 5 -\& JOB1 -\& /foo, bar/, END -\& JOB2 -\& /baz, qux/, -\& END -.Ve -.PP -Note the difference between setting one string and setting both strings. -.PP -With \-\-regexp the \-\-recend and \-\-recstart will be treated as a regular expression: -.PP -.Vb 1 -\& echo foo,bar,_baz,_\|_qux, | parallel \-kN1 \-\-regexp \-\-recend \*(Aq,_+\*(Aq \-\-pipe echo JOB{#}\e;cat\e;echo END -.Ve -.PP -Output: -.PP -.Vb 7 -\& JOB1 -\& foo,bar,_END -\& JOB2 -\& baz,_\|_END -\& JOB3 -\& qux, -\& END -.Ve -.PP -\&\s-1GNU\s0 Parallel can remove the record separators with \-\-remove\-rec\-sep/\-\-rrs: -.PP -.Vb 1 -\& echo foo,bar,_baz,_\|_qux, | parallel \-kN1 \-\-rrs \-\-regexp \-\-recend \*(Aq,_+\*(Aq \-\-pipe echo JOB{#}\e;cat\e;echo END -.Ve -.PP -Output: -.PP -.Vb 7 -\& JOB1 -\& foo,barEND -\& JOB2 -\& bazEND -\& JOB3 -\& qux, -\& END -.Ve -.SS "Header" -.IX Subsection "Header" -If the input data has a header, the header can be repeated for each -job by matching the header with \-\-header. If headers start with \f(CW%:\fR -.PP -.Vb 1 -\& cat num_%header | parallel \-\-header \*(Aq(%.*\en)*\*(Aq \-\-pipe \-N3 echo JOB{#}\e;cat -.Ve -.PP -Output (the order may be different): -.PP -.Vb 10 -\& JOB1 -\& %head1 -\& %head2 -\& 1 -\& 2 -\& 3 -\& JOB2 -\& %head1 -\& %head2 -\& 4 -\& 5 -\& 6 -\& JOB3 -\& %head1 -\& %head2 -\& 7 -\& 8 -\& 9 -\& JOB4 -\& %head1 -\& %head2 -\& 10 -.Ve -.PP -If the header is 2 lines, \-\-header 2 will work: -.PP -.Vb 1 -\& cat num_%header | parallel \-\-header 2 \-\-pipe \-N3 echo JOB{#}\e;cat -.Ve -.PP -Output: Same as above. -.SH "Shebang" -.IX Header "Shebang" -.SS "Input data and parallel command in the same file" -.IX Subsection "Input data and parallel command in the same file" -\&\s-1GNU\s0 Parallel is often called as: -.PP -.Vb 1 -\& cat input_file | parallel command -.Ve -.PP -With \-\-shebang the input_file and parallel can be combined into the same script. -.PP -UNIX-scripts start with a shebang line like: -.PP -.Vb 1 -\& #!/bin/bash -.Ve -.PP -\&\s-1GNU\s0 Parallel can do that, too. With \-\-shebang the arguments can be -listed in the file. The parallel command is the first line of the -script: -.PP -.Vb 1 -\& #!/usr/bin/parallel \-\-shebang \-r echo -\& -\& foo -\& bar -\& baz -.Ve -.PP -Output (the order may be different): -.PP -.Vb 3 -\& foo -\& bar -\& baz -.Ve -.SS "Parallelizing existing scripts" -.IX Subsection "Parallelizing existing scripts" -\&\s-1GNU\s0 Parallel is often called as: -.PP -.Vb 2 -\& cat input_file | parallel command -\& parallel command ::: foo bar -.Ve -.PP -If command is a script parallel can be combined into a single file so: -.PP -.Vb 2 -\& cat input_file | command -\& command foo bar -.Ve -.PP -will run the script in parallel. -.PP -This perl script perl_echo works like echo: -.PP -.Vb 1 -\& #!/usr/bin/perl -\& -\& print "@ARGV\en" -.Ve -.PP -It can be called as: -.PP -.Vb 1 -\& parallel perl_echo ::: foo bar -.Ve -.PP -By changing the #!\-line it can be run in parallel -.PP -.Vb 1 -\& #!/usr/bin/parallel \-\-shebang\-wrap /usr/bin/perl -\& -\& print "@ARGV\en" -.Ve -.PP -Thus this will work: -.PP -.Vb 1 -\& perl_echo foo bar -.Ve -.PP -Output (the order may be different): -.PP -.Vb 2 -\& foo -\& bar -.Ve -.PP -This technique can be used for: -.IP "Perl:" 9 -.IX Item "Perl:" -#!/usr/bin/parallel \-\-shebang\-wrap /usr/bin/perl -.IP "Python:" 9 -.IX Item "Python:" -#!/usr/bin/parallel \-\-shebang\-wrap /usr/bin/python -.IP "Bash:" 9 -.IX Item "Bash:" -#!/usr/bin/parallel \-\-shebang\-wrap /bin/bash -.IP "R:" 9 -.IX Item "R:" -#!/usr/bin/parallel \-\-shebang\-wrap /usr/bin/Rscript \-\-vanilla \-\-slave -.IP "GNUplot:" 9 -.IX Item "GNUplot:" -#!/usr/bin/parallel \-\-shebang\-wrap ARG={} /usr/bin/gnuplot -.IP "Ruby:" 9 -.IX Item "Ruby:" -#!/usr/bin/parallel \-\-shebang\-wrap /usr/bin/ruby -.SH "Semaphore" -.IX Header "Semaphore" -\&\s-1GNU\s0 Parallel can work as a counting semaphore. This is slower and less -efficient than its normal mode. -.PP -An alias for 'parallel \-\-semaphore' is 'sem'. The default is to allow -only one program to run at a time (technically called a mutex). The -program is started in the background. Use \-\-wait for all 'sem's to -finish: -.PP -.Vb 5 -\& sem \*(Aqsleep 1; echo The first finished\*(Aq && -\& echo The first is now running in the background && -\& sem \*(Aqsleep 1; echo The second finished\*(Aq && -\& echo The second is now running in the background -\& sem \-\-wait -.Ve -.PP -Output: -.PP -.Vb 4 -\& The first is now running in the background -\& The first finished -\& The second is now running in the background -\& The second finished -.Ve -.PP -The command can be run in the foreground with \-\-fg: -.PP -.Vb 5 -\& sem \-\-fg \*(Aqsleep 1; echo The first finished\*(Aq && -\& echo The first finished running in the foreground && -\& sem \-\-fg \*(Aqsleep 1; echo The second finished\*(Aq && -\& echo The second finished running in the foreground -\& sem \-\-wait -.Ve -.PP -The difference between this and just running the command, is that a -mutex is set, so if other sems were running in the background only one -would run at the same time. -.PP -To tell the difference between which semaphore is used, use -\&\-\-semaphorename/\-\-id. Run this in one terminal: -.PP -.Vb 1 -\& sem \-\-id my_id \-u \*(Aqecho First started; sleep 10; echo The first finished\*(Aq -.Ve -.PP -and simultaneously this in another terminal: -.PP -.Vb 1 -\& sem \-\-id my_id \-u \*(Aqecho Second started; sleep 10; echo The second finished\*(Aq -.Ve -.PP -Note how the second will only be started when the first has finished. -.SS "Counting semaphore" -.IX Subsection "Counting semaphore" -A mutex is like having a single toilet: When it is in use everyone -else will have to wait. A counting semaphore is like having multiple -toilets: Several people can use the toilets, but when they all are in -use, everyone else will have to wait. -.PP -sem can emulate a counting semaphore. Use \-\-jobs to set the number of -toilets: -.PP -.Vb 5 -\& sem \-\-jobs 3 \-\-id my_id \-u \*(Aqecho First started; sleep 5; echo The first finished\*(Aq && -\& sem \-\-jobs 3 \-\-id my_id \-u \*(Aqecho Second started; sleep 6; echo The second finished\*(Aq && -\& sem \-\-jobs 3 \-\-id my_id \-u \*(Aqecho Third started; sleep 7; echo The third finished\*(Aq && -\& sem \-\-jobs 3 \-\-id my_id \-u \*(Aqecho Fourth started; sleep 8; echo The fourth finished\*(Aq && -\& sem \-\-wait \-\-id my_id -.Ve -.PP -Output: -.PP -.Vb 8 -\& First started -\& Second started -\& Third started -\& The first finished -\& Fourth started -\& The second finished -\& The third finished -\& The fourth finished -.Ve -.SH "Informational" -.IX Header "Informational" -\&\s-1GNU\s0 Parallel has some options to give short information about the -configuration. -.PP -\&\-\-help will print a summary of the most important options: -.PP -.Vb 1 -\& parallel \-\-help -.Ve -.PP -Output: -.PP -.Vb 4 -\& Usage: -\& parallel [options] [command [arguments]] < list_of_arguments -\& parallel [options] [command [arguments]] (::: arguments|:::: argfile(s))... -\& cat ... | parallel \-\-pipe [options] [command [arguments]] -\& -\& \-j n Run n jobs in parallel -\& \-k Keep same order -\& \-X Multiple arguments with context replace -\& \-\-colsep regexp Split input on regexp for positional replacements -\& {} {.} {/} {/.} {#} Replacement strings -\& {3} {3.} {3/} {3/.} Positional replacement strings -\& -\& \-S sshlogin Example: foo@server.example.com -\& \-\-slf .. Use ~/.parallel/sshloginfile as the list of sshlogins -\& \-\-trc {}.bar Shorthand for \-\-transfer \-\-return {}.bar \-\-cleanup -\& \-\-onall Run the given command with argument on all sshlogins -\& \-\-nonall Run the given command with no arguments on all sshlogins -\& -\& \-\-pipe Split stdin (standard input) to multiple jobs. -\& \-\-recend str Record end separator for \-\-pipe. -\& \-\-recstart str Record start separator for \-\-pipe. -\& -\& See \*(Aqman parallel\*(Aq for details -\& -\& When using GNU Parallel for a publication please cite: -\& -\& O. Tange (2011): GNU Parallel \- The Command\-Line Power Tool, -\& ;login: The USENIX Magazine, February 2011:42\-47. -.Ve -.PP -When asking for help, always report the full output of: -.PP -.Vb 1 -\& parallel \-\-version -.Ve -.PP -Output: -.PP -.Vb 5 -\& GNU parallel 20130822 -\& Copyright (C) 2007,2008,2009,2010,2011,2012,2013 Ole Tange and Free Software Foundation, Inc. -\& License GPLv3+: GNU GPL version 3 or later -\& This is free software: you are free to change and redistribute it. -\& GNU parallel comes with no warranty. -\& -\& Web site: http://www.gnu.org/software/parallel -\& -\& When using GNU Parallel for a publication please cite: -\& -\& O. Tange (2011): GNU Parallel \- The Command\-Line Power Tool, -\& ;login: The USENIX Magazine, February 2011:42\-47. -.Ve -.PP -In scripts \-\-minversion can be used to ensure the user has at least -this version: -.PP -.Vb 1 -\& parallel \-\-minversion 20130722 && echo Your version is at least 20130722. -.Ve -.PP -Output: -.PP -.Vb 2 -\& 20130722 -\& Your version is at least 20130722. -.Ve -.PP -If using \s-1GNU\s0 Parallel for research the BibTeX citation can be -generated using \-\-bibtex. -.PP -.Vb 1 -\& parallel \-\-bibtex -.Ve -.PP -Output: -.PP -.Vb 12 -\& @article{Tange2011a, -\& title = {GNU Parallel \- The Command\-Line Power Tool}, -\& author = {O. Tange}, -\& address = {Frederiksberg, Denmark}, -\& journal = {;login: The USENIX Magazine}, -\& month = {Feb}, -\& number = {1}, -\& volume = {36}, -\& url = {http://www.gnu.org/s/parallel}, -\& year = {2011}, -\& pages = {42\-47} -\& } -.Ve -.PP -With \-\-max\-line\-length\-allowed \s-1GNU\s0 Parallel will report the maximal -size of the command line: -.PP -.Vb 1 -\& parallel \-\-max\-line\-length\-allowed -.Ve -.PP -Output (may vary on different systems): -.PP -.Vb 1 -\& 131071 -.Ve -.PP -\&\-\-number\-of\-cpus and \-\-number\-of\-cores run system specific code to -determine the number of CPUs and \s-1CPU\s0 cores on the system. On -unsupported platforms they will return 1: -.PP -.Vb 2 -\& parallel \-\-number\-of\-cpus -\& parallel \-\-number\-of\-cores -.Ve -.PP -Output (may vary on different systems): -.PP -.Vb 2 -\& 4 -\& 64 -.Ve -.SH "Profiles" -.IX Header "Profiles" -The defaults for \s-1GNU\s0 Parallel can be changed systemwise by putting the -command line options in /etc/parallel/config. They can be changed for -a user by putting them in ~/.parallel/config. -.PP -Profiles work the same way, but have to be referred to with \-\-profile: -.PP -.Vb 6 -\& echo \*(Aq\-S :,\*(Aq$SERVER1 > ~/.parallel/cluster -\& echo \*(Aq\-\-nice 17\*(Aq >> ~/.parallel/cluster -\& echo \*(Aq\-\-filter\-hosts\*(Aq >> ~/.parallel/cluster -\& echo \*(Aq\-\-timeout 300%\*(Aq >> ~/.parallel/cluster -\& echo \*(Aq\-\-env _\*(Aq >> ~/.parallel/cluster -\& parallel \-\-profile cluster echo ::: A B C -.Ve -.PP -Output: -.PP -.Vb 3 -\& A -\& B -\& C -.Ve -.PP -Profiles can be combined: -.PP -.Vb 2 -\& echo \*(Aq\-vv \-\-dry\-run\*(Aq > ~/.parallel/dryverbose -\& parallel \-\-profile dryverbose \-\-profile cluster echo ::: A B C -.Ve -.PP -Output: -.PP -.Vb 10 -\& ssh \-tt \-oLogLevel=quiet lo \*(Aqeval \`echo $SHELL | grep "/t\e{0,1\e}csh" > /dev/null && echo setenv PARALLEL_SEQ \*(Aq$PARALLEL_SEQ\*(Aq\e; setenv PARALLEL_PID \*(Aq$PARALLEL_PID\*(Aq || echo PARALLEL_SEQ=\*(Aq$PARALLEL_SEQ\*(Aq\e;export PARALLEL_SEQ\e; PARALLEL_PID=\*(Aq$PARALLEL_PID\*(Aq\e;export PARALLEL_PID\` ;\*(Aq tty\e \e>/dev/null\e \e&\e&\e stty\e isig\e \-onlcr\e \-echo\e;echo\e \e$SHELL\e \e|\e grep\e \e"/t\e\e\e{0,1\e\e\e}csh\e"\e \e>\e /dev/null\e \e&\e&\e setenv\e SERVER1\e lo\e \e&\e&\e setenv\e MYVAR\e foo\e\e\e bar\e \e&\e&\e setenv\e VAR\e foo\e \e&\e&\e setenv\e my_func\e \e\e\e(\e\e\e)\e\e\e \e\e\e{\e\e\e \e\e\e echo\e\e\e in\e\e\e my_func\e\e\e \e\e\e$1\e"\*(Aq -\& \*(Aq\e"\e\e\e}\e \e&\e&\e setenv\e my_func2\e \e\e\e(\e\e\e)\e\e\e \e\e\e{\e\e\e \e\e\e echo\e\e\e in\e\e\e my_func2\e\e\e \e\e\e$VAR\e\e\e \e\e\e$1\e"\*(Aq -\& \*(Aq\e"\e\e\e}\e \e|\e|\e export\e SERVER1=lo\e \e&\e&\e export\e MYVAR=foo\e\e\e bar\e \e&\e&\e export\e VAR=foo\e \e&\e&\e export\e my_func=\e\e\e(\e\e\e)\e\e\e \e\e\e{\e\e\e \e\e\e echo\e\e\e in\e\e\e my_func\e\e\e \e\e\e$1\e"\*(Aq -\& \*(Aq\e"\e\e\e}\e \e&\e&\e export\e my_func2=\e\e\e(\e\e\e)\e\e\e \e\e\e{\e\e\e \e\e\e echo\e\e\e in\e\e\e my_func2\e\e\e \e\e\e$VAR\e\e\e \e\e\e$1\e"\*(Aq -\& \*(Aq\e"\e\e\e}\e \e&\e&\e eval\e my_func\e"\e$my_func\e"\e \e&\e&\e eval\e my_func2\e"\e$my_func2\e"\e;\e\enice\e \-n17\e /bin/bash\e \-c\e echo\e\e\e A; -\& ssh \-tt \-oLogLevel=quiet lo \*(Aqeval \`echo $SHELL | grep "/t\e{0,1\e}csh" > /dev/null && echo setenv PARALLEL_SEQ \*(Aq$PARALLEL_SEQ\*(Aq\e; setenv PARALLEL_PID \*(Aq$PARALLEL_PID\*(Aq || echo PARALLEL_SEQ=\*(Aq$PARALLEL_SEQ\*(Aq\e;export PARALLEL_SEQ\e; PARALLEL_PID=\*(Aq$PARALLEL_PID\*(Aq\e;export PARALLEL_PID\` ;\*(Aq tty\e \e>/dev/null\e \e&\e&\e stty\e isig\e \-onlcr\e \-echo\e;echo\e \e$SHELL\e \e|\e grep\e \e"/t\e\e\e{0,1\e\e\e}csh\e"\e \e>\e /dev/null\e \e&\e&\e setenv\e SERVER1\e lo\e \e&\e&\e setenv\e MYVAR\e foo\e\e\e bar\e \e&\e&\e setenv\e VAR\e foo\e \e&\e&\e setenv\e my_func\e \e\e\e(\e\e\e)\e\e\e \e\e\e{\e\e\e \e\e\e echo\e\e\e in\e\e\e my_func\e\e\e \e\e\e$1\e"\*(Aq -\& \*(Aq\e"\e\e\e}\e \e&\e&\e setenv\e my_func2\e \e\e\e(\e\e\e)\e\e\e \e\e\e{\e\e\e \e\e\e echo\e\e\e in\e\e\e my_func2\e\e\e \e\e\e$VAR\e\e\e \e\e\e$1\e"\*(Aq -\& \*(Aq\e"\e\e\e}\e \e|\e|\e export\e SERVER1=lo\e \e&\e&\e export\e MYVAR=foo\e\e\e bar\e \e&\e&\e export\e VAR=foo\e \e&\e&\e export\e my_func=\e\e\e(\e\e\e)\e\e\e \e\e\e{\e\e\e \e\e\e echo\e\e\e in\e\e\e my_func\e\e\e \e\e\e$1\e"\*(Aq -\& \*(Aq\e"\e\e\e}\e \e&\e&\e export\e my_func2=\e\e\e(\e\e\e)\e\e\e \e\e\e{\e\e\e \e\e\e echo\e\e\e in\e\e\e my_func2\e\e\e \e\e\e$VAR\e\e\e \e\e\e$1\e"\*(Aq -\& \*(Aq\e"\e\e\e}\e \e&\e&\e eval\e my_func\e"\e$my_func\e"\e \e&\e&\e eval\e my_func2\e"\e$my_func2\e"\e;\e\enice\e \-n17\e /bin/bash\e \-c\e echo\e\e\e B; -\& ssh \-tt \-oLogLevel=quiet lo \*(Aqeval \`echo $SHELL | grep "/t\e{0,1\e}csh" > /dev/null && echo setenv PARALLEL_SEQ \*(Aq$PARALLEL_SEQ\*(Aq\e; setenv PARALLEL_PID \*(Aq$PARALLEL_PID\*(Aq || echo PARALLEL_SEQ=\*(Aq$PARALLEL_SEQ\*(Aq\e;export PARALLEL_SEQ\e; PARALLEL_PID=\*(Aq$PARALLEL_PID\*(Aq\e;export PARALLEL_PID\` ;\*(Aq tty\e \e>/dev/null\e \e&\e&\e stty\e isig\e \-onlcr\e \-echo\e;echo\e \e$SHELL\e \e|\e grep\e \e"/t\e\e\e{0,1\e\e\e}csh\e"\e \e>\e /dev/null\e \e&\e&\e setenv\e SERVER1\e lo\e \e&\e&\e setenv\e MYVAR\e foo\e\e\e bar\e \e&\e&\e setenv\e VAR\e foo\e \e&\e&\e setenv\e my_func\e \e\e\e(\e\e\e)\e\e\e \e\e\e{\e\e\e \e\e\e echo\e\e\e in\e\e\e my_func\e\e\e \e\e\e$1\e"\*(Aq -\& \*(Aq\e"\e\e\e}\e \e&\e&\e setenv\e my_func2\e \e\e\e(\e\e\e)\e\e\e \e\e\e{\e\e\e \e\e\e echo\e\e\e in\e\e\e my_func2\e\e\e \e\e\e$VAR\e\e\e \e\e\e$1\e"\*(Aq -\& \*(Aq\e"\e\e\e}\e \e|\e|\e export\e SERVER1=lo\e \e&\e&\e export\e MYVAR=foo\e\e\e bar\e \e&\e&\e export\e VAR=foo\e \e&\e&\e export\e my_func=\e\e\e(\e\e\e)\e\e\e \e\e\e{\e\e\e \e\e\e echo\e\e\e in\e\e\e my_func\e\e\e \e\e\e$1\e"\*(Aq -\& \*(Aq\e"\e\e\e}\e \e&\e&\e export\e my_func2=\e\e\e(\e\e\e)\e\e\e \e\e\e{\e\e\e \e\e\e echo\e\e\e in\e\e\e my_func2\e\e\e \e\e\e$VAR\e\e\e \e\e\e$1\e"\*(Aq -\& \*(Aq\e"\e\e\e}\e \e&\e&\e eval\e my_func\e"\e$my_func\e"\e \e&\e&\e eval\e my_func2\e"\e$my_func2\e"\e;\e\enice\e \-n17\e /bin/bash\e \-c\e echo\e\e\e C; -.Ve -.SH "Spread the word" -.IX Header "Spread the word" -I hope you have learned something from this tutorial. -.PP -If you like \s-1GNU\s0 Parallel: -.IP "\(bu" 2 -(Re\-)walk through the tutorial if you have not done so in the past year -(http://www.gnu.org/software/parallel/parallel_tutorial.html) -.IP "\(bu" 2 -Give a demo at your local user group/team/colleagues -.IP "\(bu" 2 -Post the intro videos and the tutorial on Reddit, Diaspora*, -forums, blogs, Identi.ca, Google+, Twitter, Facebook, Linkedin, -mailing lists -.IP "\(bu" 2 -Request or write a review for your favourite blog or magazine -.IP "\(bu" 2 -Invite me for your next conference -.PP -If you use \s-1GNU\s0 Parallel for research: -.IP "\(bu" 2 -Please cite \s-1GNU\s0 Parallel in you publications (use \-\-bibtex) -.PP -If \s-1GNU\s0 Parallel saves you money: -.IP "\(bu" 2 -(Have your company) donate to \s-1FSF\s0 or become a member -https://my.fsf.org/donate/ -.PP -(C) 2013,2014 Ole Tange, GPLv3 diff --git a/testsuite/REQUIREMENTS b/testsuite/REQUIREMENTS index 9b7c99c8..4dea23a0 100644 --- a/testsuite/REQUIREMENTS +++ b/testsuite/REQUIREMENTS @@ -40,7 +40,7 @@ $INSTALL ash csh fdclone fish fizsh ksh mksh pdksh posh rc rush sash tcsh yash z SSHPASS=`goodpasswd` export SSHPASS #shells="bash sh csh ash tcsh zsh ksh fish fizsh mksh pdksh posh rc sash yash nopathbash nopathcsh" -shells="bash sh csh ash tcsh zsh ksh fish fizsh mksh posh rc sash yash nopathbash nopathcsh" +shells="bash sh csh ash dash tcsh zsh ksh fish fizsh mksh posh rc sash yash nopathbash nopathcsh" create_shell_user() { shell="$1" sudo deluser $shell && sudo mv /home/$shell /tmp/$shell.$RANDOM