From a3947338dadd6b5a45a00b712a69acc721f7f50a Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Mon, 4 Jul 2016 14:54:38 +0200 Subject: [PATCH 01/12] env_parallel.ksh: Fixed for ksh #47483: env_parallel export --env only. --- src/env_parallel.ksh | 67 +++++++++++++++++-- testsuite/tests-to-run/parallel-local-ssh4.sh | 37 ++++++++++ testsuite/wanted-results/parallel-local-ssh1 | 2 +- testsuite/wanted-results/parallel-local-ssh4 | 32 ++++++++- testsuite/wanted-results/parallel-local22 | 2 +- 5 files changed, 133 insertions(+), 7 deletions(-) diff --git a/src/env_parallel.ksh b/src/env_parallel.ksh index 2662a72e..36ab6418 100755 --- a/src/env_parallel.ksh +++ b/src/env_parallel.ksh @@ -26,8 +26,67 @@ # Fifth Floor, Boston, MA 02110-1301 USA env_parallel() { - # env_parallel.ksh - export PARALLEL_ENV="$(alias | perl -pe 's/^/alias /';typeset -p|egrep -v 'typeset( -i)? -r|PIPESTATUS';typeset -f)"; - `which parallel` "$@"; - unset PARALLEL_ENV; + # env_parallel.ksh + + # Get the --env variables if set + # and convert a b c to (a|b|c) + # If --env not set: Match everything (.*) + grep_REGEXP="$( + perl -e 'for(@ARGV){ + $next_is_env and push @envvar, split/,/, $_; + $next_is_env=/^--env$/; + } + $vars = join "|",map { quotemeta $_ } @envvar; + print $vars ? "($vars)" : "(.*)"; + ' -- "$@" + )" + + # Grep alias names + _alias_NAMES="$(alias | perl -pe 's/=.*//' | + egrep "^${grep_REGEXP}\$")" + _list_alias_BODIES="alias $_alias_NAMES | perl -pe 's/^/alias /'" + if [[ "$_alias_NAMES" = "" ]] ; then + # no aliases selected + _list_alias_BODIES="true" + fi + unset _alias_NAMES + + # Grep function names + _function_NAMES="$(typeset +p -f | perl -pe 's/\(\).*//' | + egrep "^${grep_REGEXP}\$")" + _list_function_BODIES="typeset -f $_function_NAMES" + if [[ "$_function_NAMES" = "" ]] ; then + # no functions selected + _list_function_BODIES="true" + fi + unset _function_NAMES + + # Grep variable names + _variable_NAMES="$(typeset +p | perl -pe 's/^typeset .. //' | + egrep "^${grep_REGEXP}\$" | + egrep -v '^(PIPESTATUS)$')" + _list_variable_VALUES="typeset -p $_variable_NAMES" + if [[ "$_variable_NAMES" = "" ]] ; then + # no variables selected + _list_variable_VALUES="true" + fi + unset _variable_NAMES + + # eval is needed for aliases - cannot explain why + export PARALLEL_ENV="$( + eval $_list_alias_BODIES; + $_list_variable_VALUES; + $_list_function_BODIES)"; + unset _list_alias_BODIES + unset _list_variable_VALUES + unset _list_function_BODIES + `which parallel` "$@"; + unset PARALLEL_ENV; } + +# _env_parallel() { +# # env_parallel.ksh +# export PARALLEL_ENV="$(alias | perl -pe 's/^/alias /';typeset -p|egrep -v 'typeset( -i)? -r|PIPESTATUS';typeset -f)"; +# `which parallel` "$@"; +# unset PARALLEL_ENV; +# } diff --git a/testsuite/tests-to-run/parallel-local-ssh4.sh b/testsuite/tests-to-run/parallel-local-ssh4.sh index b4b40957..c76423ad 100644 --- a/testsuite/tests-to-run/parallel-local-ssh4.sh +++ b/testsuite/tests-to-run/parallel-local-ssh4.sh @@ -75,6 +75,43 @@ ssh zsh@lo ' env_parallel -k --env myarray -S server echo "\${myarray[{}]}" ::: 1 2 3 ' +echo '### ksh' +ssh ksh@lo ' + . `which env_parallel.ksh`; + alias myecho="echo aliases"; + env_parallel myecho ::: work; + env_parallel -S server myecho ::: work; + env_parallel --env myecho myecho ::: work; + env_parallel --env myecho -S server myecho ::: work +' + +ssh ksh@lo ' + . `which env_parallel.ksh`; + myfunc() { echo functions $*; }; + env_parallel myfunc ::: work; + env_parallel -S server myfunc ::: work; + env_parallel --env myfunc myfunc ::: work; + env_parallel --env myfunc -S server myfunc ::: work +' + +ssh ksh@lo ' + . `which env_parallel.ksh`; + myvar=variables; + env_parallel echo "\$myvar" ::: work; + env_parallel -S server echo "\$myvar" ::: work; + env_parallel --env myvar echo "\$myvar" ::: work; + env_parallel --env myvar -S server echo "\$myvar" ::: work +' + +ssh ksh@lo ' + . `which env_parallel.ksh`; + myarray=(arrays work, too); + env_parallel -k echo "\${myarray[{}]}" ::: 0 1 2; + env_parallel -k -S server echo "\${myarray[{}]}" ::: 0 1 2; + env_parallel -k --env myarray echo "\${myarray[{}]}" ::: 0 1 2; + env_parallel -k --env myarray -S server echo "\${myarray[{}]}" ::: 0 1 2 +' + echo '### --env _' fUbAr="OK FUBAR" parallel -S parallel@lo --env _ echo '$fUbAr $DEBEMAIL' ::: test fUbAr="OK FUBAR" parallel -S csh@lo --env _ echo '$fUbAr $DEBEMAIL' ::: test diff --git a/testsuite/wanted-results/parallel-local-ssh1 b/testsuite/wanted-results/parallel-local-ssh1 index 4aa2b8c5..649dfe10 100644 --- a/testsuite/wanted-results/parallel-local-ssh1 +++ b/testsuite/wanted-results/parallel-local-ssh1 @@ -192,7 +192,7 @@ ls: cannot access bug46519.ccc: No such file or directory echo '### Test --nice remote' ### Test --nice remote stdout parallel --nice 1 -S lo -vv 'PAR=a bash -c "echo \$PAR {}"' ::: b | perl -pe 's/\S*parallel-server\S*/one-server/;s:[a-z/\\+=0-9]{500,}:base64:i;' -ssh lo -- exec perl -e @GNU_Parallel\\\=\\\(\\\"use\\\",\\\"IPC::Open3\\\;\\\",\\\"use\\\",\\\"MIME::Base64\\\"\\\)\\\;eval\\\"@GNU_Parallel\\\"\\\;my\\\$eval\\\=decode_base64\\\(join\\\"\\\",@ARGV\\\)\\\;eval\\\$eval\\\; base64; +ssh lo -- exec perl -e @GNU_Parallel\\\=\\\(\\\"use\\\",\\\"IPC::Open3\\\;\\\",\\\"use\\\",\\\"MIME::Base64\\\"\\\)\\\;eval\\\"@GNU_Parallel\\\"\\\;\\\$SIG\\\{CHLD\\\}\\\=\\\"IGNORE\\\"\\\;my\\\$zip\\\=\\\(grep\\\{-x\\\$_\\\}\\\"/usr/local/bin/bzip2\\\"\\\)\\\[0\\\]\\\|\\\|\\\"bzip2\\\"\\\;my\\\(\\\$in,\\\$out,\\\$eval\\\)\\\;open3\\\(\\\$in,\\\$out,\\\"\\\>\\\&STDERR\\\",\\\$zip,\\\"-dc\\\"\\\)\\\;if\\\(my\\\$perlpid\\\=fork\\\)\\\{close\\\$in\\\;\\\$eval\\\=join\\\"\\\",\\\<\\\$out\\\>\\\;close\\\$out\\\;\\\}else\\\{close\\\$out\\\;print\\\$in\\\(decode_base64\\\(join\\\"\\\",@ARGV\\\)\\\)\\\;close\\\$in\\\;exit\\\;\\\}wait\\\;eval\\\$eval\\\; base64; a b echo '**' ** diff --git a/testsuite/wanted-results/parallel-local-ssh4 b/testsuite/wanted-results/parallel-local-ssh4 index 441d4cc5..559c0d5e 100644 --- a/testsuite/wanted-results/parallel-local-ssh4 +++ b/testsuite/wanted-results/parallel-local-ssh4 @@ -37,7 +37,7 @@ aliases work aliases work aliases work aliases work - ssh zsh@lo ' myfunc() { echo functions $*; }; env_parallel myfunc ::: work; env_parallel -S server myfunc ::: work; env_parallel --env myfunc myfunc ::: work; env_parallel --env myfunc -S server myfunc ::: work ' +ssh zsh@lo ' myfunc() { echo functions $*; }; env_parallel myfunc ::: work; env_parallel -S server myfunc ::: work; env_parallel --env myfunc myfunc ::: work; env_parallel --env myfunc -S server myfunc ::: work ' functions work functions work functions work @@ -60,6 +60,36 @@ too arrays work, too +echo '### ksh' +### ksh +ssh ksh@lo ' . `which env_parallel.ksh`; alias myecho="echo aliases"; env_parallel myecho ::: work; env_parallel -S server myecho ::: work; env_parallel --env myecho myecho ::: work; env_parallel --env myecho -S server myecho ::: work ' +aliases work +aliases work +aliases work +aliases work +ssh ksh@lo ' . `which env_parallel.ksh`; myfunc() { echo functions $*; }; env_parallel myfunc ::: work; env_parallel -S server myfunc ::: work; env_parallel --env myfunc myfunc ::: work; env_parallel --env myfunc -S server myfunc ::: work ' +functions work +functions work +functions work +functions work +ssh ksh@lo ' . `which env_parallel.ksh`; myvar=variables; env_parallel echo "\$myvar" ::: work; env_parallel -S server echo "\$myvar" ::: work; env_parallel --env myvar echo "\$myvar" ::: work; env_parallel --env myvar -S server echo "\$myvar" ::: work ' +variables work +variables work +variables work +variables work +ssh ksh@lo ' . `which env_parallel.ksh`; myarray=(arrays work, too); env_parallel -k echo "\${myarray[{}]}" ::: 0 1 2; env_parallel -k -S server echo "\${myarray[{}]}" ::: 0 1 2; env_parallel -k --env myarray echo "\${myarray[{}]}" ::: 0 1 2; env_parallel -k --env myarray -S server echo "\${myarray[{}]}" ::: 0 1 2 ' +arrays +work, +too +arrays +work, +too +arrays +work, +too +arrays +work, +too echo '### --env _' ### --env _ fUbAr="OK FUBAR" parallel -S parallel@lo --env _ echo '$fUbAr $DEBEMAIL' ::: test diff --git a/testsuite/wanted-results/parallel-local22 b/testsuite/wanted-results/parallel-local22 index 61622430..055aaa88 100644 --- a/testsuite/wanted-results/parallel-local22 +++ b/testsuite/wanted-results/parallel-local22 @@ -92,7 +92,7 @@ With script in $PARALLEL /bin/bash=/home/tange/privat/parallel/testsuite echo '### bug #42892: parallel -a nonexiting --pipepart' ### bug #42892: parallel -a nonexiting --pipepart parallel --pipepart -a nonexisting wc -parallel: Error: nonexisting is not a seekable file. +parallel: Error: nonexisting is neither a file nor a block device echo '### bug #42913: Dont use $SHELL but the shell currently running' ### bug #42913: Dont use $SHELL but the shell currently running echo '## Unknown shell => $SHELL (bash)' From eacc07bcb04ffb7716dd90d9cd30a4424add59ca Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Tue, 5 Jul 2016 13:28:22 +0200 Subject: [PATCH 02/12] env_parallel.bash: --env _ supported. env_parallel.ksh: --env _ supported. env_parallel.zsh: --env _ supported. testsuite/.../parallel-local-ssh7: Focuses on env_parallel. --- src/env_parallel.bash | 30 +- src/env_parallel.ksh | 22 +- src/env_parallel.zsh | 30 +- testsuite/tests-to-run/parallel-local-ssh4.sh | 272 ------------ testsuite/tests-to-run/parallel-local-ssh7.sh | 402 ++++++++++++++++++ testsuite/wanted-results/parallel-local-ssh4 | 232 ---------- testsuite/wanted-results/parallel-local-ssh7 | 247 +++++++++++ 7 files changed, 714 insertions(+), 521 deletions(-) create mode 100755 testsuite/tests-to-run/parallel-local-ssh7.sh create mode 100644 testsuite/wanted-results/parallel-local-ssh7 diff --git a/src/env_parallel.bash b/src/env_parallel.bash index 0599f9b6..615738b4 100755 --- a/src/env_parallel.bash +++ b/src/env_parallel.bash @@ -25,45 +25,57 @@ # or write to the Free Software Foundation, Inc., 51 Franklin St, # Fifth Floor, Boston, MA 02110-1301 USA -# Supports env of 127426 bytes - env_parallel() { # env_parallel.bash # Get the --env variables if set + # --env _ should be ignored # and convert a b c to (a|b|c) # If --env not set: Match everything (.*) - local grep_REGEXP="$( + local _grep_REGEXP="$( perl -e 'for(@ARGV){ + /^_$/ and $next_is_env = 0; $next_is_env and push @envvar, split/,/, $_; - $next_is_env=/^--env$/; + $next_is_env = /^--env$/; } $vars = join "|",map { quotemeta $_ } @envvar; print $vars ? "($vars)" : "(.*)"; ' -- "$@" )" + # Deal with --env _ + local _ignore_UNDERSCORE="$( + perl -e 'for(@ARGV){ + $next_is_env and push @envvar, split/,/, $_; + $next_is_env=/^--env$/; + } + $underscore = grep { /^_$/ } @envvar; + print $underscore ? "grep -vf $ENV{HOME}/.parallel/ignored_vars" : "cat"; + ' -- "$@" + )" # Grep alias names local _alias_NAMES="$(compgen -a | - egrep "^${grep_REGEXP}\$")" + egrep "^${_grep_REGEXP}\$" | $_ignore_UNDERSCORE)" local _list_alias_BODIES="alias $_alias_NAMES" if [[ "$_alias_NAMES" = "" ]] ; then # no aliases selected _list_alias_BODIES="true" fi + unset _alias_NAMES # Grep function names local _function_NAMES="$(compgen -A function | - egrep "^${grep_REGEXP}\$")" + egrep "^${_grep_REGEXP}\$" | $_ignore_UNDERSCORE)" local _list_function_BODIES="typeset -f $_function_NAMES" if [[ "$_function_NAMES" = "" ]] ; then # no functions selected _list_function_BODIES="true" fi + unset _function_NAMES # Grep variable names local _variable_NAMES="$(compgen -A variable | - egrep "^${grep_REGEXP}\$" | + egrep "^${_grep_REGEXP}\$" | $_ignore_UNDERSCORE | grep -vFf <(readonly) | egrep -v '^(BASHOPTS|BASHPID|EUID|GROUPS|FUNCNAME|DIRSTACK|_|PIPESTATUS|PPID|SHELLOPTS|UID|USERNAME|BASH_[A-Z_]+)$')" local _list_variable_VALUES="typeset -p $_variable_NAMES" @@ -71,6 +83,7 @@ env_parallel() { # no variables selected _list_variable_VALUES="true" fi + unset _variable_NAMES # Copy shopt (so e.g. extended globbing works) # But force expand_aliases as aliases otherwise do not work @@ -82,6 +95,9 @@ env_parallel() { $_list_alias_BODIES; $_list_variable_VALUES; $_list_function_BODIES)"; + unset _list_alias_BODIES + unset _list_variable_VALUES + unset _list_function_BODIES `which parallel` "$@"; unset PARALLEL_ENV; } diff --git a/src/env_parallel.ksh b/src/env_parallel.ksh index 36ab6418..db238ecf 100755 --- a/src/env_parallel.ksh +++ b/src/env_parallel.ksh @@ -25,14 +25,18 @@ # or write to the Free Software Foundation, Inc., 51 Franklin St, # Fifth Floor, Boston, MA 02110-1301 USA +# Supports env of 127426 bytes + env_parallel() { # env_parallel.ksh # Get the --env variables if set + # --env _ should be ignored # and convert a b c to (a|b|c) # If --env not set: Match everything (.*) - grep_REGEXP="$( + _grep_REGEXP="$( perl -e 'for(@ARGV){ + /^_$/ and $next_is_env = 0; $next_is_env and push @envvar, split/,/, $_; $next_is_env=/^--env$/; } @@ -40,10 +44,20 @@ env_parallel() { print $vars ? "($vars)" : "(.*)"; ' -- "$@" )" + # Deal with --env _ + _ignore_UNDERSCORE="$( + perl -e 'for(@ARGV){ + $next_is_env and push @envvar, split/,/, $_; + $next_is_env=/^--env$/; + } + $underscore = grep { /^_$/ } @envvar; + print $underscore ? "grep -vf $ENV{HOME}/.parallel/ignored_vars" : "cat"; + ' -- "$@" + )" # Grep alias names _alias_NAMES="$(alias | perl -pe 's/=.*//' | - egrep "^${grep_REGEXP}\$")" + egrep "^${_grep_REGEXP}\$" | $_ignore_UNDERSCORE)" _list_alias_BODIES="alias $_alias_NAMES | perl -pe 's/^/alias /'" if [[ "$_alias_NAMES" = "" ]] ; then # no aliases selected @@ -53,7 +67,7 @@ env_parallel() { # Grep function names _function_NAMES="$(typeset +p -f | perl -pe 's/\(\).*//' | - egrep "^${grep_REGEXP}\$")" + egrep "^${_grep_REGEXP}\$" | $_ignore_UNDERSCORE)" _list_function_BODIES="typeset -f $_function_NAMES" if [[ "$_function_NAMES" = "" ]] ; then # no functions selected @@ -63,7 +77,7 @@ env_parallel() { # Grep variable names _variable_NAMES="$(typeset +p | perl -pe 's/^typeset .. //' | - egrep "^${grep_REGEXP}\$" | + egrep "^${_grep_REGEXP}\$" | $_ignore_UNDERSCORE | egrep -v '^(PIPESTATUS)$')" _list_variable_VALUES="typeset -p $_variable_NAMES" if [[ "$_variable_NAMES" = "" ]] ; then diff --git a/src/env_parallel.zsh b/src/env_parallel.zsh index e5035045..bf16529e 100755 --- a/src/env_parallel.zsh +++ b/src/env_parallel.zsh @@ -29,30 +29,43 @@ env_parallel() { # env_parallel.zsh # Get the --env variables if set + # --env _ should be ignored # and convert a b c to (a|b|c) # If --env not set: Match everything (.*) - grep_REGEXP="$( + _grep_REGEXP="$( perl -e 'for(@ARGV){ + /^_$/ and $next_is_env = 0; $next_is_env and push @envvar, split/,/, $_; - $next_is_env=/^--env$/; + $next_is_env = /^--env$/; } $vars = join "|",map { quotemeta $_ } @envvar; print $vars ? "($vars)" : "(.*)"; ' -- "$@" )" + # Deal with --env _ + local _ignore_UNDERSCORE="$( + perl -e 'for(@ARGV){ + $next_is_env and push @envvar, split/,/, $_; + $next_is_env=/^--env$/; + } + $underscore = grep { /^_$/ } @envvar; + print $underscore ? "grep -vf $ENV{HOME}/.parallel/ignored_vars" : "cat"; + ' -- "$@" + )" # Grep alias names _alias_NAMES="$(print -l ${(k)aliases} | - egrep "^${grep_REGEXP}\$")" + egrep "^${_grep_REGEXP}\$" | eval $_ignore_UNDERSCORE )" _list_alias_BODIES="alias "$(echo $_alias_NAMES|xargs)" | perl -pe 's/^/alias /'" if [[ "$_alias_NAMES" = "" ]] ; then # no aliases selected _list_alias_BODIES="true" fi + unset _alias_NAMES # Grep function names _function_NAMES="$(print -l ${(k)functions} | - egrep "^${grep_REGEXP}\$" | + egrep "^${_grep_REGEXP}\$" | eval $_ignore_UNDERSCORE | grep -v '=' )" _list_function_BODIES="typeset -f "$(echo $_function_NAMES|xargs) @@ -60,11 +73,12 @@ env_parallel() { # no functions selected _list_function_BODIES="true" fi + unset _function_NAMES # Grep variable names # The egrep -v is crap and should be better _variable_NAMES="$(print -l ${(k)parameters} | - egrep "^${grep_REGEXP}\$" | + egrep "^${_grep_REGEXP}\$" | eval $_ignore_UNDERSCORE | egrep -v '^([-?#!$*@_0]|zsh_eval_context|ZSH_EVAL_CONTEXT|LINENO|IFS|commands|functions|options|aliases|EUID|EGID|UID|GID)$' | egrep -v 'terminfo|funcstack|galiases|keymaps|parameters|jobdirs|dirstack|functrace|funcsourcetrace|zsh_scheduled_events|dis_aliases|dis_reswords|dis_saliases|modules|reswords|saliases|widgets|userdirs|historywords|nameddirs|termcap|dis_builtins|dis_functions|jobtexts|funcfiletrace|dis_galiases|builtins|history|jobstates' )" @@ -75,13 +89,17 @@ env_parallel() { # no variables selected _list_variable_VALUES="true" fi + unset _variable_NAMES + export PARALLEL_ENV="$( eval $_list_alias_BODIES; eval $_list_function_BODIES; eval $_list_variable_VALUES; )"; - + unset _list_alias_BODIES + unset _list_variable_VALUES + unset _list_function_BODIES `which parallel` "$@"; unset PARALLEL_ENV; } diff --git a/testsuite/tests-to-run/parallel-local-ssh4.sh b/testsuite/tests-to-run/parallel-local-ssh4.sh index c76423ad..b59f1c33 100644 --- a/testsuite/tests-to-run/parallel-local-ssh4.sh +++ b/testsuite/tests-to-run/parallel-local-ssh4.sh @@ -1,117 +1,10 @@ #!/bin/bash -# TODO ksh fish - unset run_test # SSH only allowed to localhost/lo # --retries if ssh dies cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | parallel -vj4 --retries 2 -k --joblog /tmp/jl-`basename $0` -L1 -echo '### --env from man env_parallel' -echo '### bash' -ssh bash@lo ' - alias myecho="echo aliases"; - env_parallel myecho ::: work; - env_parallel -S server myecho ::: work; - env_parallel --env myecho myecho ::: work; - env_parallel --env myecho -S server myecho ::: work -' - -ssh bash@lo ' - myfunc() { echo functions $*; }; - env_parallel myfunc ::: work; - env_parallel -S server myfunc ::: work; - env_parallel --env myfunc myfunc ::: work; - env_parallel --env myfunc -S server myfunc ::: work -' - -ssh bash@lo ' - myvar=variables; - env_parallel echo "\$myvar" ::: work; - env_parallel -S server echo "\$myvar" ::: work; - env_parallel --env myvar echo "\$myvar" ::: work; - env_parallel --env myvar -S server echo "\$myvar" ::: work -' - -ssh bash@lo ' - myarray=(arrays work, too); - env_parallel -k echo "\${myarray[{}]}" ::: 0 1 2; - env_parallel -k -S server echo "\${myarray[{}]}" ::: 0 1 2; - env_parallel -k --env myarray echo "\${myarray[{}]}" ::: 0 1 2; - env_parallel -k --env myarray -S server echo "\${myarray[{}]}" ::: 0 1 2 -' - -echo '### zsh' - -ssh zsh@lo ' - alias myecho="echo aliases"; - env_parallel myecho ::: work; - env_parallel -S server myecho ::: work; - env_parallel --env myecho myecho ::: work; - env_parallel --env myecho -S server myecho ::: work -' - -ssh zsh@lo ' - myfunc() { echo functions $*; }; - env_parallel myfunc ::: work; - env_parallel -S server myfunc ::: work; - env_parallel --env myfunc myfunc ::: work; - env_parallel --env myfunc -S server myfunc ::: work -' - -ssh zsh@lo ' - myvar=variables; - env_parallel echo "\$myvar" ::: work; - env_parallel -S server echo "\$myvar" ::: work; - env_parallel --env myvar echo "\$myvar" ::: work; - env_parallel --env myvar -S server echo "\$myvar" ::: work -' - -ssh zsh@lo ' - myarray=(arrays work, too); - env_parallel -k echo "\${myarray[{}]}" ::: 1 2 3; - env_parallel -k -S server echo "\${myarray[{}]}" ::: 1 2 3; - env_parallel -k --env myarray echo "\${myarray[{}]}" ::: 1 2 3; - env_parallel -k --env myarray -S server echo "\${myarray[{}]}" ::: 1 2 3 -' - -echo '### ksh' -ssh ksh@lo ' - . `which env_parallel.ksh`; - alias myecho="echo aliases"; - env_parallel myecho ::: work; - env_parallel -S server myecho ::: work; - env_parallel --env myecho myecho ::: work; - env_parallel --env myecho -S server myecho ::: work -' - -ssh ksh@lo ' - . `which env_parallel.ksh`; - myfunc() { echo functions $*; }; - env_parallel myfunc ::: work; - env_parallel -S server myfunc ::: work; - env_parallel --env myfunc myfunc ::: work; - env_parallel --env myfunc -S server myfunc ::: work -' - -ssh ksh@lo ' - . `which env_parallel.ksh`; - myvar=variables; - env_parallel echo "\$myvar" ::: work; - env_parallel -S server echo "\$myvar" ::: work; - env_parallel --env myvar echo "\$myvar" ::: work; - env_parallel --env myvar -S server echo "\$myvar" ::: work -' - -ssh ksh@lo ' - . `which env_parallel.ksh`; - myarray=(arrays work, too); - env_parallel -k echo "\${myarray[{}]}" ::: 0 1 2; - env_parallel -k -S server echo "\${myarray[{}]}" ::: 0 1 2; - env_parallel -k --env myarray echo "\${myarray[{}]}" ::: 0 1 2; - env_parallel -k --env myarray -S server echo "\${myarray[{}]}" ::: 0 1 2 -' - echo '### --env _' fUbAr="OK FUBAR" parallel -S parallel@lo --env _ echo '$fUbAr $DEBEMAIL' ::: test fUbAr="OK FUBAR" parallel -S csh@lo --env _ echo '$fUbAr $DEBEMAIL' ::: test @@ -124,13 +17,6 @@ echo 'bug #40137: SHELL not bash: Warning when exporting funcs' . <(printf 'myfunc() {\necho $1\n}'); export -f myfunc; parallel --env myfunc -S lo myfunc ::: no_warning . <(printf 'myfunc() {\necho $1\n}'); export -f myfunc; SHELL=/bin/sh parallel --env myfunc -S lo myfunc ::: warning -echo 'env_parallel from man page - transfer non-exported var' - source $(which env_parallel.bash); - var=nonexported env_parallel -S parallel@lo echo '$var' ::: variable - -echo 'compared to parallel - no transfer non-exported var' - var=nonexported parallel -S parallel@lo echo '$var' ::: variable - echo '### zsh' echo 'env in zsh' @@ -226,161 +112,3 @@ echo '### bug #45575: -m and multiple hosts repeats first args' seq 1 3 | parallel -X -S 2/lo,2/: -k echo EOF - -echo -echo Test env_parallel: -echo + for each shell -echo + remote, locally -echo + variables, variables with funky content, arrays, assoc array, functions, aliases -echo -echo "### Bash environment" -#stdout ssh -t lo <<'EOS' -myvar="myvar works" -funky=$(perl -e 'print pack "c*", 1..255') -myarray=('' array_val2 3 '' 5 ' space 6 ') -declare -A assocarr -assocarr[a]=assoc_val_a -assocarr[b]=assoc_val_b -alias alias_echo="echo 3 arg"; -func_echo() { - echo $*; - echo "$myvar" - echo "${myarray[5]}" - echo ${assocarr[a]} - echo Funky-"$funky"-funky -} -. `which env_parallel.bash` -env_parallel alias_echo ::: alias_works -env_parallel func_echo ::: function_works -env_parallel -S lo alias_echo ::: alias_works_over_ssh -env_parallel -S lo func_echo ::: function_works_over_ssh -echo -echo "$funky" | parallel --shellquote -#EOS - - -echo -echo "### Zsh environment" -stdout ssh -q zsh@lo <<'EOS' | egrep -v 'Welcome to |packages can be updated|security updates' -myvar="myvar works" -funky=$(perl -e 'print pack "c*", 1..255') -myarray=('' array_val2 3 '' 5 ' space 6 ') -declare -A assocarr -assocarr[a]=assoc_val_a -assocarr[b]=assoc_val_b -alias alias_echo="echo 3 arg"; -func_echo() { - echo $*; - echo "$myvar" - echo "$myarray[6]" - echo ${assocarr[a]} - echo Funky-"$funky"-funky -} - -env_parallel alias_echo ::: alias_works -env_parallel func_echo ::: function_works -env_parallel -S zsh@lo alias_echo ::: alias_works_over_ssh -env_parallel -S zsh@lo func_echo ::: function_works_over_ssh -echo -echo "$funky" | parallel --shellquote -EOS - -echo -echo "### Ksh environment" -stdout ssh -q ksh@lo <<'EOS' | egrep -v 'Welcome to |packages can be updated|security updates' -myvar="myvar works" -funky=$(perl -e 'print pack "c*", 1..255') -myarray=('' array_val2 3 '' 5 ' space 6 ') -typeset -A assocarr -assocarr[a]=assoc_val_a -assocarr[b]=assoc_val_b -alias alias_echo="echo 3 arg"; - -func_echo() { - echo $*; - echo "$myvar" - echo "${myarray[5]}" - echo ${assocarr[a]} - echo Funky-"$funky"-funky -} - -env_parallel alias_echo ::: alias_works -env_parallel func_echo ::: function_works -env_parallel -S ksh@lo alias_echo ::: alias_works_over_ssh -env_parallel -S ksh@lo func_echo ::: function_works_over_ssh -echo -echo "$funky" | parallel --shellquote -EOS - -echo -echo "### Fish environment" -stdout ssh -q fish@lo <<'EOS' | egrep -v 'Welcome to |packages can be updated|security updates' -set myvar "myvar works" -setenv myenvvar "myenvvar works" - -set funky (perl -e 'print pack "c*", 1..255') -setenv funkyenv (perl -e 'print pack "c*", 1..255') - -set myarray '' array_val2 3 '' 5 ' space 6 ' - -# Assoc arrays do not exist -#typeset -A assocarr -#assocarr[a]=assoc_val_a -#assocarr[b]=assoc_val_b -alias alias_echo="echo 3 arg"; - -function func_echo - echo $argv; - echo "$myvar" - echo "$myenvvar" - echo "$myarray[6]" -# Assoc arrays do not exist in fish -# echo ${assocarr[a]} - echo - echo - echo - echo Funky-"$funky"-funky - echo Funkyenv-"$funkyenv"-funkyenv - echo - echo - echo -end - -env_parallel alias_echo ::: alias_works -env_parallel func_echo ::: function_works -env_parallel -S fish@lo alias_echo ::: alias_works_over_ssh -env_parallel -S fish@lo func_echo ::: function_works_over_ssh -echo -echo "$funky" | parallel --shellquote -EOS - -echo -echo "### csh environment" -stdout ssh -q csh@lo <<'EOS' | egrep -v 'Welcome to |packages can be updated|security updates' -set myvar = "myvar works" -set funky = "`perl -e 'print pack q(c*), 2..255'`" -set myarray = ('' 'array_val2' '3' '' '5' ' space 6 ') -#declare -A assocarr -#assocarr[a]=assoc_val_a -#assocarr[b]=assoc_val_b -alias alias_echo echo 3 arg; -alias alias_echo_var 'echo $argv; echo "$myvar"; echo "${myarray[4]} special chars problem"; echo Funky-"$funky"-funky' - -#function func_echo -# echo $argv; -# echo $myvar; -# echo ${myarray[2]} -# #echo ${assocarr[a]} -# echo Funky-"$funky"-funky -#end - -env_parallel alias_echo ::: alias_works -env_parallel alias_echo_var ::: alias_var_works -env_parallel func_echo ::: function_does_not_work -env_parallel -S csh@lo alias_echo ::: alias_works_over_ssh -env_parallel -S csh@lo alias_echo_var ::: alias_var_works_over_ssh -env_parallel -S csh@lo func_echo ::: function_does_not_work_over_ssh -echo -echo "$funky" | parallel --shellquote -EOS - diff --git a/testsuite/tests-to-run/parallel-local-ssh7.sh b/testsuite/tests-to-run/parallel-local-ssh7.sh new file mode 100755 index 00000000..a0db5f9c --- /dev/null +++ b/testsuite/tests-to-run/parallel-local-ssh7.sh @@ -0,0 +1,402 @@ +#!/bin/bash + +echo '### test --env _' +echo 'Both test that variables are copied,' +echo 'but also that they are NOT copied, if ignored' + +par_bash_man() { + echo '### bash' + + myscript=$(cat <<'_EOF' + echo "### From man env_parallel" + + alias myecho="echo aliases"; + env_parallel myecho ::: work; + env_parallel -S server myecho ::: work; + env_parallel --env myecho myecho ::: work; + env_parallel --env myecho -S server myecho ::: work + + myfunc() { echo functions $*; }; + env_parallel myfunc ::: work; + env_parallel -S server myfunc ::: work; + env_parallel --env myfunc myfunc ::: work; + env_parallel --env myfunc -S server myfunc ::: work + + myvar=variables; + env_parallel echo "\$myvar" ::: work; + env_parallel -S server echo "\$myvar" ::: work; + env_parallel --env myvar echo "\$myvar" ::: work; + env_parallel --env myvar -S server echo "\$myvar" ::: work + + myarray=(arrays work, too); + env_parallel -k echo "\${myarray[{}]}" ::: 0 1 2; + env_parallel -k -S server echo "\${myarray[{}]}" ::: 0 1 2; + env_parallel -k --env myarray echo "\${myarray[{}]}" ::: 0 1 2; + env_parallel -k --env myarray -S server echo "\${myarray[{}]}" ::: 0 1 2 +_EOF + ) + ssh bash@lo "$myscript" +} + +par_zsh_man() { + echo '### zsh' + myscript=$(cat <<'_EOF' + echo "### From man env_parallel" + + alias myecho="echo aliases"; + env_parallel myecho ::: work; + env_parallel -S server myecho ::: work; + env_parallel --env myecho myecho ::: work; + env_parallel --env myecho -S server myecho ::: work + + myfunc() { echo functions $*; }; + env_parallel myfunc ::: work; + env_parallel -S server myfunc ::: work; + env_parallel --env myfunc myfunc ::: work; + env_parallel --env myfunc -S server myfunc ::: work + + myvar=variables; + env_parallel echo "\$myvar" ::: work; + env_parallel -S server echo "\$myvar" ::: work; + env_parallel --env myvar echo "\$myvar" ::: work; + env_parallel --env myvar -S server echo "\$myvar" ::: work + + myarray=(arrays work, too); + env_parallel -k echo "\${myarray[{}]}" ::: 1 2 3; + env_parallel -k -S server echo "\${myarray[{}]}" ::: 1 2 3; + env_parallel -k --env myarray echo "\${myarray[{}]}" ::: 1 2 3; + env_parallel -k --env myarray -S server echo "\${myarray[{}]}" ::: 1 2 3 +_EOF + ) + ssh zsh@lo "$myscript" +} + +par_ksh_man() { + echo '### ksh' + myscript=$(cat <<'_EOF' + echo "### From man env_parallel" + + . `which env_parallel.ksh`; + alias myecho="echo aliases"; + env_parallel myecho ::: work; + env_parallel -S server myecho ::: work; + env_parallel --env myecho myecho ::: work; + env_parallel --env myecho -S server myecho ::: work + + . `which env_parallel.ksh`; + myfunc() { echo functions $*; }; + env_parallel myfunc ::: work; + env_parallel -S server myfunc ::: work; + env_parallel --env myfunc myfunc ::: work; + env_parallel --env myfunc -S server myfunc ::: work + + . `which env_parallel.ksh`; + myvar=variables; + env_parallel echo "\$myvar" ::: work; + env_parallel -S server echo "\$myvar" ::: work; + env_parallel --env myvar echo "\$myvar" ::: work; + env_parallel --env myvar -S server echo "\$myvar" ::: work + + . `which env_parallel.ksh`; + myarray=(arrays work, too); + env_parallel -k echo "\${myarray[{}]}" ::: 0 1 2; + env_parallel -k -S server echo "\${myarray[{}]}" ::: 0 1 2; + env_parallel -k --env myarray echo "\${myarray[{}]}" ::: 0 1 2; + env_parallel -k --env myarray -S server echo "\${myarray[{}]}" ::: 0 1 2 +_EOF + ) + ssh ksh@lo "$myscript" +} + +par_bash_underscore() { + echo '### bash' + myscript=$(cat <<'_EOF' + echo "### Testing of --env _" + + env_parallel --record-env; + alias myecho="echo \$myvar aliases in"; + myfunc() { myecho ${myarray[@]} functions $*; }; + myvar="variables in"; + myarray=(and arrays in); + env_parallel myfunc ::: work; + env_parallel -S server myfunc ::: work; + env_parallel --env myfunc,myvar,myarray,myecho myfunc ::: work; + env_parallel --env myfunc,myvar,myarray,myecho -S server myfunc ::: work; + env_parallel --env _ myfunc ::: work; + env_parallel --env _ -S server myfunc ::: work; + + echo myvar >> ~/.parallel/ignored_vars; + env_parallel --env _ myfunc ::: work; + env_parallel --env _ -S server myfunc ::: work; + echo myarray >> ~/.parallel/ignored_vars; + env_parallel --env _ myfunc ::: work; + env_parallel --env _ -S server myfunc ::: work; + echo myecho >> ~/.parallel/ignored_vars; + env_parallel --env _ myfunc ::: work; + echo "OK if no myecho ^^^^^^^^^^^^^^^^^^^^^^^^^" >&2; + env_parallel --env _ -S server myfunc ::: work; + echo "OK if no myecho ^^^^^^^^^^^^^^^^^^^^^^^^^" >&2; + echo myfunc >> ~/.parallel/ignored_vars; + env_parallel --env _ myfunc ::: work; + echo "OK if no myfunc ^^^^^^^^^^^^^^^^^^^^^^^^^" >&2; + env_parallel --env _ -S server myfunc ::: work; + echo "OK if no myfunc ^^^^^^^^^^^^^^^^^^^^^^^^^" >&2; +_EOF + ) + ssh bash@lo "$myscript" +} + +par_zsh_underscore() { + echo '### zsh' + myscript=$(cat <<'_EOF' + echo "### Testing of --env _" + + . `which env_parallel.zsh`; + env_parallel --record-env; + alias myecho="echo \$myvar aliases in"; + eval `cat <<"_EOS"; + myfunc() { myecho ${myarray[@]} functions $*; }; + myvar="variables in"; + myarray=(and arrays in); + env_parallel myfunc ::: work; + env_parallel -S server myfunc ::: work; + env_parallel --env myfunc,myvar,myarray,myecho myfunc ::: work; + env_parallel --env myfunc,myvar,myarray,myecho -S server myfunc ::: work; + env_parallel --env _ myfunc ::: work; + env_parallel --env _ -S server myfunc ::: work; + + echo myvar >> ~/.parallel/ignored_vars; + env_parallel --env _ myfunc ::: work; + env_parallel --env _ -S server myfunc ::: work; + echo myarray >> ~/.parallel/ignored_vars; + env_parallel --env _ myfunc ::: work; + env_parallel --env _ -S server myfunc ::: work; + echo myecho >> ~/.parallel/ignored_vars; + : Not using the function, because aliases are expanded in functions; + env_parallel --env _ myecho ::: work; + echo OK if no .^^^^^^^^^^^^^^^^^^^^^^^^^ myecho >&2; + env_parallel --env _ -S server myecho ::: work; + echo OK if no .^^^^^^^^^^^^^^^^^^^^^^^^^ myecho >&2; + echo myfunc >> ~/.parallel/ignored_vars; + env_parallel --env _ myfunc ::: work; + echo OK if no .^^^^^^^^^^^^^^^^^^^^^^^^^ myfunc >&2; + env_parallel --env _ -S server myfunc ::: work; + echo OK if no .^^^^^^^^^^^^^^^^^^^^^^^^^ myfunc >&2; +_EOS` +_EOF + ) + ssh zsh@lo "$myscript" +} + +par_ksh_underscore() { + echo '### ksh' + myscript=$(cat <<'_EOF' + echo "### Testing of --env _" + + . `which env_parallel.ksh`; + env_parallel --record-env; + alias myecho="echo \$myvar aliases in"; + myfunc() { myecho ${myarray[@]} functions $*; }; + myvar="variables in"; + myarray=(and arrays in); + env_parallel myfunc ::: work; + env_parallel -S server myfunc ::: work; + env_parallel --env myfunc,myvar,myarray,myecho myfunc ::: work; + env_parallel --env myfunc,myvar,myarray,myecho -S server myfunc ::: work; + env_parallel --env _ myfunc ::: work; + env_parallel --env _ -S server myfunc ::: work; + + echo myvar >> ~/.parallel/ignored_vars; + env_parallel --env _ myfunc ::: work; + env_parallel --env _ -S server myfunc ::: work; + echo myarray >> ~/.parallel/ignored_vars; + env_parallel --env _ myfunc ::: work; + env_parallel --env _ -S server myfunc ::: work; + echo myecho >> ~/.parallel/ignored_vars; + env_parallel --env _ myfunc ::: work; + echo "OK if no myecho ^^^^^^^^^^^^^^^^^" >&2; + env_parallel --env _ -S server myfunc ::: work; + echo "OK if no myecho ^^^^^^^^^^^^^^^^^" >&2; + echo myfunc >> ~/.parallel/ignored_vars; + env_parallel --env _ myfunc ::: work; + echo "OK if no myfunc ^^^^^^^^^^^^^^^^^" >&2; + env_parallel --env _ -S server myfunc ::: work; + echo "OK if no myfunc ^^^^^^^^^^^^^^^^^" >&2; +_EOF + ) + ssh ksh@lo "$myscript" +} + +# Test env_parallel: +# + for each shell +# + remote, locally +# + variables, variables with funky content, arrays, assoc array, functions, aliases + +par_bash_funky() { + myscript=$(cat <<'_EOF' + myvar="myvar works" + funky=$(perl -e "print pack \"c*\", 1..255") + myarray=("" array_val2 3 "" 5 " space 6 ") + declare -A assocarr + assocarr[a]=assoc_val_a + assocarr[b]=assoc_val_b + alias alias_echo="echo 3 arg"; + func_echo() { + echo $*; + echo "$myvar" + echo "${myarray[5]}" + echo ${assocarr[a]} + echo Funky-"$funky"-funky + } + . `which env_parallel.bash` + env_parallel alias_echo ::: alias_works + env_parallel func_echo ::: function_works + env_parallel -S lo alias_echo ::: alias_works_over_ssh + env_parallel -S lo func_echo ::: function_works_over_ssh + echo + echo "$funky" | parallel --shellquote +_EOF + ) + ssh bash@lo "$myscript" +} + +par_zsh_funky() { + myscript=$(cat <<'_EOF' + myvar="myvar works" + funky=$(perl -e "print pack \"c*\", 1..255") + myarray=("" array_val2 3 "" 5 " space 6 ") + declare -A assocarr + assocarr[a]=assoc_val_a + assocarr[b]=assoc_val_b + alias alias_echo="echo 3 arg"; + func_echo() { + echo $*; + echo "$myvar" + echo "$myarray[6]" + echo ${assocarr[a]} + echo Funky-"$funky"-funky + } + + env_parallel alias_echo ::: alias_works + env_parallel func_echo ::: function_works + env_parallel -S zsh@lo alias_echo ::: alias_works_over_ssh + env_parallel -S zsh@lo func_echo ::: function_works_over_ssh + echo + echo "$funky" | parallel --shellquote +_EOF + ) + ssh zsh@lo "$myscript" +} + +par_ksh_funky() { + myscript=$(cat <<'_EOF' + . `which env_parallel.ksh`; + + myvar="myvar works" + funky=$(perl -e "print pack \"c*\", 1..255") + myarray=("" array_val2 3 "" 5 " space 6 ") + typeset -A assocarr + assocarr[a]=assoc_val_a + assocarr[b]=assoc_val_b + alias alias_echo="echo 3 arg"; + + func_echo() { + echo $*; + echo "$myvar" + echo "${myarray[5]}" + echo ${assocarr[a]} + echo Funky-"$funky"-funky + } + + env_parallel alias_echo ::: alias_works + env_parallel func_echo ::: function_works + env_parallel -S ksh@lo alias_echo ::: alias_works_over_ssh + env_parallel -S ksh@lo func_echo ::: function_works_over_ssh + echo + echo "$funky" | parallel --shellquote +_EOF + ) + ssh ksh@lo "$myscript" +} + +par_fish_funky() { + myscript=$(cat <<'_EOF' + set myvar "myvar works" + setenv myenvvar "myenvvar works" + + set funky (perl -e "print pack \"c*\", 1..255") + setenv funkyenv (perl -e "print pack \"c*\", 1..255") + + set myarray "" array_val2 3 "" 5 " space 6 " + + # Assoc arrays do not exist + #typeset -A assocarr + #assocarr[a]=assoc_val_a + #assocarr[b]=assoc_val_b + alias alias_echo="echo 3 arg"; + + function func_echo + echo $argv; + echo "$myvar" + echo "$myenvvar" + echo "$myarray[6]" + # Assoc arrays do not exist in fish + # echo ${assocarr[a]} + echo + echo + echo + echo Funky-"$funky"-funky + echo Funkyenv-"$funkyenv"-funkyenv + echo + echo + echo + end + + env_parallel alias_echo ::: alias_works + env_parallel func_echo ::: function_works + env_parallel -S fish@lo alias_echo ::: alias_works_over_ssh + env_parallel -S fish@lo func_echo ::: function_works_over_ssh + echo + echo "$funky" | parallel --shellquote +_EOF + ) + ssh fish@lo "$myscript" +} + +par_csh_funky() { + myscript=$(cat <<'_EOF' + set myvar = "myvar works" + set funky = "`perl -e 'print pack q(c*), 2..255'`" + set myarray = ('' 'array_val2' '3' '' '5' ' space 6 ') + #declare -A assocarr + #assocarr[a]=assoc_val_a + #assocarr[b]=assoc_val_b + alias alias_echo echo 3 arg; + alias alias_echo_var 'echo $argv; echo "$myvar"; echo "${myarray[4]} special chars problem"; echo Funky-"$funky"-funky' + + #function func_echo + # echo $argv; + # echo $myvar; + # echo ${myarray[2]} + # #echo ${assocarr[a]} + # echo Funky-"$funky"-funky + #end + + env_parallel alias_echo ::: alias_works + env_parallel alias_echo_var ::: alias_var_works + env_parallel func_echo ::: function_does_not_work + env_parallel -S csh@lo alias_echo ::: alias_works_over_ssh + env_parallel -S csh@lo alias_echo_var ::: alias_var_works_over_ssh + env_parallel -S csh@lo func_echo ::: function_does_not_work_over_ssh + echo + echo "$funky" | parallel --shellquote +_EOF + ) + ssh csh@lo "$myscript" +} + + +export -f $(compgen -A function | grep par_) +# Tested with -j1..8 +# -j6 was fastest +compgen -A function | grep par_ | sort | parallel -j6 --tag -k '{} 2>&1' diff --git a/testsuite/wanted-results/parallel-local-ssh4 b/testsuite/wanted-results/parallel-local-ssh4 index 559c0d5e..595edf58 100644 --- a/testsuite/wanted-results/parallel-local-ssh4 +++ b/testsuite/wanted-results/parallel-local-ssh4 @@ -1,95 +1,3 @@ -echo '### --env from man env_parallel' -### --env from man env_parallel -echo '### bash' -### bash -ssh bash@lo ' alias myecho="echo aliases"; env_parallel myecho ::: work; env_parallel -S server myecho ::: work; env_parallel --env myecho myecho ::: work; env_parallel --env myecho -S server myecho ::: work ' -aliases work -aliases work -aliases work -aliases work -ssh bash@lo ' myfunc() { echo functions $*; }; env_parallel myfunc ::: work; env_parallel -S server myfunc ::: work; env_parallel --env myfunc myfunc ::: work; env_parallel --env myfunc -S server myfunc ::: work ' -functions work -functions work -functions work -functions work -ssh bash@lo ' myvar=variables; env_parallel echo "\$myvar" ::: work; env_parallel -S server echo "\$myvar" ::: work; env_parallel --env myvar echo "\$myvar" ::: work; env_parallel --env myvar -S server echo "\$myvar" ::: work ' -variables work -variables work -variables work -variables work -ssh bash@lo ' myarray=(arrays work, too); env_parallel -k echo "\${myarray[{}]}" ::: 0 1 2; env_parallel -k -S server echo "\${myarray[{}]}" ::: 0 1 2; env_parallel -k --env myarray echo "\${myarray[{}]}" ::: 0 1 2; env_parallel -k --env myarray -S server echo "\${myarray[{}]}" ::: 0 1 2 ' -arrays -work, -too -arrays -work, -too -arrays -work, -too -arrays -work, -too -echo '### zsh' -### zsh -ssh zsh@lo ' alias myecho="echo aliases"; env_parallel myecho ::: work; env_parallel -S server myecho ::: work; env_parallel --env myecho myecho ::: work; env_parallel --env myecho -S server myecho ::: work ' -aliases work -aliases work -aliases work -aliases work -ssh zsh@lo ' myfunc() { echo functions $*; }; env_parallel myfunc ::: work; env_parallel -S server myfunc ::: work; env_parallel --env myfunc myfunc ::: work; env_parallel --env myfunc -S server myfunc ::: work ' -functions work -functions work -functions work -functions work -ssh zsh@lo ' myvar=variables; env_parallel echo "\$myvar" ::: work; env_parallel -S server echo "\$myvar" ::: work; env_parallel --env myvar echo "\$myvar" ::: work; env_parallel --env myvar -S server echo "\$myvar" ::: work ' -variables work -variables work -variables work -variables work -ssh zsh@lo ' myarray=(arrays work, too); env_parallel -k echo "\${myarray[{}]}" ::: 1 2 3; env_parallel -k -S server echo "\${myarray[{}]}" ::: 1 2 3; env_parallel -k --env myarray echo "\${myarray[{}]}" ::: 1 2 3; env_parallel -k --env myarray -S server echo "\${myarray[{}]}" ::: 1 2 3 ' -arrays -work, -too -arrays -work, -too -arrays -work, -too -arrays -work, -too -echo '### ksh' -### ksh -ssh ksh@lo ' . `which env_parallel.ksh`; alias myecho="echo aliases"; env_parallel myecho ::: work; env_parallel -S server myecho ::: work; env_parallel --env myecho myecho ::: work; env_parallel --env myecho -S server myecho ::: work ' -aliases work -aliases work -aliases work -aliases work -ssh ksh@lo ' . `which env_parallel.ksh`; myfunc() { echo functions $*; }; env_parallel myfunc ::: work; env_parallel -S server myfunc ::: work; env_parallel --env myfunc myfunc ::: work; env_parallel --env myfunc -S server myfunc ::: work ' -functions work -functions work -functions work -functions work -ssh ksh@lo ' . `which env_parallel.ksh`; myvar=variables; env_parallel echo "\$myvar" ::: work; env_parallel -S server echo "\$myvar" ::: work; env_parallel --env myvar echo "\$myvar" ::: work; env_parallel --env myvar -S server echo "\$myvar" ::: work ' -variables work -variables work -variables work -variables work -ssh ksh@lo ' . `which env_parallel.ksh`; myarray=(arrays work, too); env_parallel -k echo "\${myarray[{}]}" ::: 0 1 2; env_parallel -k -S server echo "\${myarray[{}]}" ::: 0 1 2; env_parallel -k --env myarray echo "\${myarray[{}]}" ::: 0 1 2; env_parallel -k --env myarray -S server echo "\${myarray[{}]}" ::: 0 1 2 ' -arrays -work, -too -arrays -work, -too -arrays -work, -too -arrays -work, -too echo '### --env _' ### --env _ fUbAr="OK FUBAR" parallel -S parallel@lo --env _ echo '$fUbAr $DEBEMAIL' ::: test @@ -107,14 +15,6 @@ bug #40137: SHELL not bash: Warning when exporting funcs no_warning . <(printf 'myfunc() {\necho $1\n}'); export -f myfunc; SHELL=/bin/sh parallel --env myfunc -S lo myfunc ::: warning warning -echo 'env_parallel from man page - transfer non-exported var' -env_parallel from man page - transfer non-exported var - source $(which env_parallel.bash); var=nonexported env_parallel -S parallel@lo echo '$var' ::: variable -nonexported variable -echo 'compared to parallel - no transfer non-exported var' -compared to parallel - no transfer non-exported var - var=nonexported parallel -S parallel@lo echo '$var' ::: variable -variable echo '### zsh' ### zsh echo 'env in zsh' @@ -224,135 +124,3 @@ echo '### bug #45575: -m and multiple hosts repeats first args' 1 2 3 - -Test env_parallel: -+ for each shell -+ remote, locally -+ variables, variables with funky content, arrays, assoc array, functions, aliases - -### Bash environment -3 arg alias_works -function_works -myvar works - space 6 -assoc_val_a -Funky- - - -  !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~ --funky -3 arg alias_works_over_ssh -function_works_over_ssh -myvar works - space 6 -assoc_val_a -Funky- - - -  !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~ --funky - -\\\\\\\\ -\ \ \ \\\\\\\\\\\\\\ \!\"\#\$%\&\'\(\)\*+,-./0123456789:\;\<\=\>\?@ABCDEFGHIJKLMNOPQRSTUVWXYZ\[\\\]\^_\`abcdefghijklmnopqrstuvwxyz\{\|\}\~\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\ - -### Zsh environment - - * Documentation: http://www.linuxmint.com -3 arg alias_works -function_works -myvar works - space 6 -assoc_val_a -Funky- - -  !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~-funky -3 arg alias_works_over_ssh -function_works_over_ssh -myvar works - space 6 -assoc_val_a -Funky- - -  !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~-funky - -\\\\\\\\ -\ \ \ \\\\\\\\\\\\\\ \!\"\#\$%\&\'\(\)\*+,-./0123456789:\;\<\=\>\?@ABCDEFGHIJKLMNOPQRSTUVWXYZ\[\\\]\^_\`abcdefghijklmnopqrstuvwxyz\{\|\}\~\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\ - -### Ksh environment - - * Documentation: http://www.linuxmint.com -3 arg alias_works -function_works -myvar works - space 6 -assoc_val_a -Funky- -  !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~-funky -3 arg alias_works_over_ssh -function_works_over_ssh -myvar works - space 6 -assoc_val_a -Funky- -  !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~-funky - -\\\\\\\\ -\ \ \ \\\\\\\\\\\\\\ \!\"\#\$%\&\'\(\)\*+,-./0123456789:\;\<\=\>\?@ABCDEFGHIJKLMNOPQRSTUVWXYZ\[\\\]\^_\`abcdefghijklmnopqrstuvwxyz\{\|\}\~\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\ - -### Fish environment - - * Documentation: http://www.linuxmint.com -env_parallel: Warning: ASCII value 1 in variables is not supported -env_parallel: Warning: ASCII value 1 in variables is not supported -3 arg alias_works -env_parallel: Warning: ASCII value 1 in variables is not supported -env_parallel: Warning: ASCII value 1 in variables is not supported -function_works -myvar works -myenvvar works - space 6 - - - -Funky-   !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~-funky -Funkyenv-   !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~-funkyenv - - - -env_parallel: Warning: ASCII value 1 in variables is not supported -env_parallel: Warning: ASCII value 1 in variables is not supported -3 arg alias_works_over_ssh -env_parallel: Warning: ASCII value 1 in variables is not supported -env_parallel: Warning: ASCII value 1 in variables is not supported -function_works_over_ssh -myvar works -myenvvar works - space 6 - - - -Funky-   !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~-funky -Funkyenv-   !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~-funkyenv - - - - -\\\\\\\\ \ \ \ \ \\\\\\\\\\\\\\ \ \!\"\#\$%\&\'\(\)\*+,-./0123456789:\;\<\=\>\?@ABCDEFGHIJKLMNOPQRSTUVWXYZ\[\\\]\^_\`abcdefghijklmnopqrstuvwxyz\{\|\}\~\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\ - -### csh environment - - * Documentation: http://www.linuxmint.com -Warning: no access to tty (Bad file descriptor). -Thus no job control in this shell. -3 arg alias_works - -myvar works -space special chars problem -Funky-  !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~-funky alias_var_works -func_echo: Command not found. -3 arg alias_works_over_ssh -myvar works -space special chars problem -Funky-  !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~-funky alias_var_works_over_ssh -func_echo: Command not found. -\\\\\\\\ \ \ \ \ \\\\\\\\\\\\\\ \!\"\#\$%\&\'\(\)\*+,-./0123456789:\;\<\=\>\?@ABCDEFGHIJKLMNOPQRSTUVWXYZ\[\\\]\^_\`abcdefghijklmnopqrstuvwxyz\{\|\}\~\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\ diff --git a/testsuite/wanted-results/parallel-local-ssh7 b/testsuite/wanted-results/parallel-local-ssh7 new file mode 100644 index 00000000..9d1001a6 --- /dev/null +++ b/testsuite/wanted-results/parallel-local-ssh7 @@ -0,0 +1,247 @@ +### test --env _ +Both test that variables are copied, +but also that they are NOT copied, if ignored +par_bash_funky 3 arg alias_works +par_bash_funky function_works +par_bash_funky myvar works +par_bash_funky space 6 +par_bash_funky assoc_val_a +par_bash_funky Funky- +par_bash_funky +par_bash_funky  +par_bash_funky  !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~ +par_bash_funky -funky +par_bash_funky 3 arg alias_works_over_ssh +par_bash_funky function_works_over_ssh +par_bash_funky myvar works +par_bash_funky space 6 +par_bash_funky assoc_val_a +par_bash_funky Funky- +par_bash_funky +par_bash_funky  +par_bash_funky  !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~ +par_bash_funky -funky +par_bash_funky +par_bash_funky \\\\\\\\ +par_bash_funky \ \ \ \\\\\\\\\\\\\\ \!\"\#\$%\&\'\(\)\*+,-./0123456789:\;\<\=\>\?@ABCDEFGHIJKLMNOPQRSTUVWXYZ\[\\\]\^_\`abcdefghijklmnopqrstuvwxyz\{\|\}\~\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\ +par_bash_man ### bash +par_bash_man ### From man env_parallel +par_bash_man aliases work +par_bash_man aliases work +par_bash_man aliases work +par_bash_man aliases work +par_bash_man functions work +par_bash_man functions work +par_bash_man functions work +par_bash_man functions work +par_bash_man variables work +par_bash_man variables work +par_bash_man variables work +par_bash_man variables work +par_bash_man arrays +par_bash_man work, +par_bash_man too +par_bash_man arrays +par_bash_man work, +par_bash_man too +par_bash_man arrays +par_bash_man work, +par_bash_man too +par_bash_man arrays +par_bash_man work, +par_bash_man too +par_bash_underscore ### bash +par_bash_underscore ### Testing of --env _ +par_bash_underscore variables in aliases in and arrays in functions work +par_bash_underscore variables in aliases in and arrays in functions work +par_bash_underscore variables in aliases in and arrays in functions work +par_bash_underscore variables in aliases in and arrays in functions work +par_bash_underscore variables in aliases in and arrays in functions work +par_bash_underscore variables in aliases in and arrays in functions work +par_bash_underscore aliases in and arrays in functions work +par_bash_underscore aliases in and arrays in functions work +par_bash_underscore aliases in functions work +par_bash_underscore aliases in functions work +par_bash_underscore /bin/bash: line 64: myecho: command not found +par_bash_underscore OK if no myecho ^^^^^^^^^^^^^^^^^^^^^^^^^ +par_bash_underscore /bin/bash: line 64: myecho: command not found +par_bash_underscore OK if no myecho ^^^^^^^^^^^^^^^^^^^^^^^^^ +par_bash_underscore /bin/bash: line 62: myfunc: command not found +par_bash_underscore OK if no myfunc ^^^^^^^^^^^^^^^^^^^^^^^^^ +par_bash_underscore /bin/bash: line 62: myfunc: command not found +par_bash_underscore OK if no myfunc ^^^^^^^^^^^^^^^^^^^^^^^^^ +par_csh_funky 3 arg alias_works +par_csh_funky myvar works +par_csh_funky space special chars problem +par_csh_funky Funky-  !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~-funky alias_var_works +par_csh_funky func_echo: Command not found. +par_csh_funky 3 arg alias_works_over_ssh +par_csh_funky myvar works +par_csh_funky space special chars problem +par_csh_funky Funky-  !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~-funky alias_var_works_over_ssh +par_csh_funky func_echo: Command not found. +par_csh_funky \\\\\\\\ \ \ \ \ \\\\\\\\\\\\\\ \!\"\#\$%\&\'\(\)\*+,-./0123456789:\;\<\=\>\?@ABCDEFGHIJKLMNOPQRSTUVWXYZ\[\\\]\^_\`abcdefghijklmnopqrstuvwxyz\{\|\}\~\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\ +par_fish_funky env_parallel: Warning: ASCII value 1 in variables is not supported +par_fish_funky env_parallel: Warning: ASCII value 1 in variables is not supported +par_fish_funky 3 arg alias_works +par_fish_funky env_parallel: Warning: ASCII value 1 in variables is not supported +par_fish_funky env_parallel: Warning: ASCII value 1 in variables is not supported +par_fish_funky function_works +par_fish_funky myvar works +par_fish_funky myenvvar works +par_fish_funky space 6 +par_fish_funky +par_fish_funky +par_fish_funky +par_fish_funky Funky-   !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~-funky +par_fish_funky Funkyenv-   !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~-funkyenv +par_fish_funky +par_fish_funky +par_fish_funky +par_fish_funky env_parallel: Warning: ASCII value 1 in variables is not supported +par_fish_funky env_parallel: Warning: ASCII value 1 in variables is not supported +par_fish_funky 3 arg alias_works_over_ssh +par_fish_funky env_parallel: Warning: ASCII value 1 in variables is not supported +par_fish_funky env_parallel: Warning: ASCII value 1 in variables is not supported +par_fish_funky function_works_over_ssh +par_fish_funky myvar works +par_fish_funky myenvvar works +par_fish_funky space 6 +par_fish_funky +par_fish_funky +par_fish_funky +par_fish_funky Funky-   !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~-funky +par_fish_funky Funkyenv-   !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~-funkyenv +par_fish_funky +par_fish_funky +par_fish_funky +par_fish_funky +par_fish_funky \\\\\\\\ \ \ \ \ \\\\\\\\\\\\\\ \ \!\"\#\$%\&\'\(\)\*+,-./0123456789:\;\<\=\>\?@ABCDEFGHIJKLMNOPQRSTUVWXYZ\[\\\]\^_\`abcdefghijklmnopqrstuvwxyz\{\|\}\~\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\ +par_ksh_funky 3 arg alias_works +par_ksh_funky function_works +par_ksh_funky myvar works +par_ksh_funky space 6 +par_ksh_funky assoc_val_a +par_ksh_funky Funky- +par_ksh_funky  !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~-funky +par_ksh_funky 3 arg alias_works_over_ssh +par_ksh_funky function_works_over_ssh +par_ksh_funky myvar works +par_ksh_funky space 6 +par_ksh_funky assoc_val_a +par_ksh_funky Funky- +par_ksh_funky  !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~-funky +par_ksh_funky +par_ksh_funky \\\\\\\\ +par_ksh_funky \ \ \ \\\\\\\\\\\\\\ \!\"\#\$%\&\'\(\)\*+,-./0123456789:\;\<\=\>\?@ABCDEFGHIJKLMNOPQRSTUVWXYZ\[\\\]\^_\`abcdefghijklmnopqrstuvwxyz\{\|\}\~\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\ +par_ksh_man ### ksh +par_ksh_man ### From man env_parallel +par_ksh_man aliases work +par_ksh_man aliases work +par_ksh_man aliases work +par_ksh_man aliases work +par_ksh_man functions work +par_ksh_man functions work +par_ksh_man functions work +par_ksh_man functions work +par_ksh_man variables work +par_ksh_man variables work +par_ksh_man variables work +par_ksh_man variables work +par_ksh_man arrays +par_ksh_man work, +par_ksh_man too +par_ksh_man arrays +par_ksh_man work, +par_ksh_man too +par_ksh_man arrays +par_ksh_man work, +par_ksh_man too +par_ksh_man arrays +par_ksh_man work, +par_ksh_man too +par_ksh_underscore ### ksh +par_ksh_underscore ### Testing of --env _ +par_ksh_underscore variables in aliases in and arrays in functions work +par_ksh_underscore variables in aliases in and arrays in functions work +par_ksh_underscore variables in aliases in and arrays in functions work +par_ksh_underscore variables in aliases in and arrays in functions work +par_ksh_underscore variables in aliases in and arrays in functions work +par_ksh_underscore variables in aliases in and arrays in functions work +par_ksh_underscore aliases in and arrays in functions work +par_ksh_underscore aliases in and arrays in functions work +par_ksh_underscore aliases in functions work +par_ksh_underscore aliases in functions work +par_ksh_underscore /usr/bin/ksh[100]: myecho: not found [No such file or directory] +par_ksh_underscore OK if no myecho ^^^^^^^^^^^^^^^^^ +par_ksh_underscore /usr/bin/ksh[100]: myecho: not found [No such file or directory] +par_ksh_underscore OK if no myecho ^^^^^^^^^^^^^^^^^ +par_ksh_underscore /usr/bin/ksh: line 100: myfunc: not found +par_ksh_underscore OK if no myfunc ^^^^^^^^^^^^^^^^^ +par_ksh_underscore /usr/bin/ksh: line 100: myfunc: not found +par_ksh_underscore OK if no myfunc ^^^^^^^^^^^^^^^^^ +par_zsh_funky 3 arg alias_works +par_zsh_funky function_works +par_zsh_funky myvar works +par_zsh_funky space 6 +par_zsh_funky assoc_val_a +par_zsh_funky Funky- +par_zsh_funky  +par_zsh_funky  !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~-funky +par_zsh_funky 3 arg alias_works_over_ssh +par_zsh_funky function_works_over_ssh +par_zsh_funky myvar works +par_zsh_funky space 6 +par_zsh_funky assoc_val_a +par_zsh_funky Funky- +par_zsh_funky  +par_zsh_funky  !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~-funky +par_zsh_funky +par_zsh_funky \\\\\\\\ +par_zsh_funky \ \ \ \\\\\\\\\\\\\\ \!\"\#\$%\&\'\(\)\*+,-./0123456789:\;\<\=\>\?@ABCDEFGHIJKLMNOPQRSTUVWXYZ\[\\\]\^_\`abcdefghijklmnopqrstuvwxyz\{\|\}\~\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\ +par_zsh_man ### zsh +par_zsh_man ### From man env_parallel +par_zsh_man aliases work +par_zsh_man aliases work +par_zsh_man aliases work +par_zsh_man aliases work +par_zsh_man functions work +par_zsh_man functions work +par_zsh_man functions work +par_zsh_man functions work +par_zsh_man variables work +par_zsh_man variables work +par_zsh_man variables work +par_zsh_man variables work +par_zsh_man arrays +par_zsh_man work, +par_zsh_man too +par_zsh_man arrays +par_zsh_man work, +par_zsh_man too +par_zsh_man arrays +par_zsh_man work, +par_zsh_man too +par_zsh_man arrays +par_zsh_man work, +par_zsh_man too +par_zsh_underscore ### zsh +par_zsh_underscore ### Testing of --env _ +par_zsh_underscore variables in aliases in and arrays in functions work +par_zsh_underscore variables in aliases in and arrays in functions work +par_zsh_underscore variables in aliases in and arrays in functions work +par_zsh_underscore variables in aliases in and arrays in functions work +par_zsh_underscore variables in aliases in and arrays in functions work +par_zsh_underscore variables in aliases in and arrays in functions work +par_zsh_underscore aliases in and arrays in functions work +par_zsh_underscore aliases in and arrays in functions work +par_zsh_underscore aliases in functions work +par_zsh_underscore aliases in functions work +par_zsh_underscore (eval):1: command not found: myecho +par_zsh_underscore OK if no .^^^^^^^^^^^^^^^^^^^^^^^^^ myecho +par_zsh_underscore (eval):1: command not found: myecho +par_zsh_underscore OK if no .^^^^^^^^^^^^^^^^^^^^^^^^^ myecho +par_zsh_underscore (eval):1: command not found: myfunc +par_zsh_underscore OK if no .^^^^^^^^^^^^^^^^^^^^^^^^^ myfunc +par_zsh_underscore (eval):1: command not found: myfunc +par_zsh_underscore OK if no .^^^^^^^^^^^^^^^^^^^^^^^^^ myfunc From 11a8800751bd80959826b868444229e8c5f0b853 Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Wed, 6 Jul 2016 15:58:47 +0200 Subject: [PATCH 03/12] parallel: Fixed locking bug in --sql for SQLite. testsuite: Reworked parallel-local-sql. --- src/parallel | 16 +- src/parallel.pod | 94 +- testsuite/tests-to-run/parallel-local-sql.sh | 132 +-- testsuite/wanted-results/parallel-local-sql | 851 ++++++++++++------- 4 files changed, 710 insertions(+), 383 deletions(-) mode change 100644 => 100755 testsuite/tests-to-run/parallel-local-sql.sh diff --git a/src/parallel b/src/parallel index a03f8907..8e1ac7ee 100755 --- a/src/parallel +++ b/src/parallel @@ -1953,7 +1953,7 @@ sub save_stdin_stdout_stderr { # $Global::original_stderr # $Global::original_stdin # Returns: N/A - + # TODO Disabled until we have an open3 that will take n filehandles # for my $fdno (1..61) { # # /dev/fd/62 and above are used by bash for <(cmd) @@ -3181,7 +3181,7 @@ sub parallelized_host_filtering { ::debug("init", $cmd, "\n"); my @out; my $prepend = ""; - + my ($host_fh,$in,$err); open3($in, $host_fh, $err, $cmd) || ::die_bug("parallel host check: $cmd"); if(not fork()) { @@ -9904,6 +9904,10 @@ sub new { my $dbh = DBI->connect($dsn, $userid, $password, { RaiseError => 1, AutoInactiveDestroy => 1 }) or die $DBI::errstr; + $dbh->{'PrintWarn'} = $Global::debug || 0; + $dbh->{'PrintError'} = $Global::debug || 0; + $dbh->{'RaiseError'} = $Global::debug || 0; + return bless { 'dbh' => $dbh, 'driver' => $driver, @@ -10085,13 +10089,19 @@ sub run { if($DBI::errstr =~ /locked/) { ::debug("sql","Lock retry: $lockretry"); $lockretry++; + ::usleep(rand()*300); } else { ::error($DBI::errstr); + wait_and_exit(255); } } } + if($lockretry >= 10) { + ::die_bug("lockretry > 10"); + } if($rv < 0){ - print $DBI::errstr; + ::error($DBI::errstr); + ::wait_and_exit(255); } return $sth; } diff --git a/src/parallel.pod b/src/parallel.pod index ec3e51d3..2eaa8565 100644 --- a/src/parallel.pod +++ b/src/parallel.pod @@ -2902,30 +2902,87 @@ Or if the regexps are fixed strings: grep -F -f regexps.txt bigfile -There are 2 limiting factors: CPU and disk I/O. CPU is easy to -measure: If the B takes >90% CPU (e.g. when running top), then the -CPU is a limiting factor, and parallelization will speed this up. If -not, then disk I/O is the limiting factor, and depending on the disk -system it may be faster or slower to parallelize. The only way to know -for certain is to measure. +There are 3 limiting factors: CPU, RAM, and disk I/O. -If the CPU is the limiting factor parallelization should be done on the regexps: +RAM is easy to measure: If the B process takes up most of your +free memory (e.g. when running B), then RAM is a limiting factor. - cat regexp.txt | parallel --pipe -L1000 --round-robin grep -f - bigfile +CPU is also easy to measure: If the B takes >90% CPU in B, +then the CPU is a limiting factor, and parallelization will speed this +up. -If a line matches multiple regexps, the line may be duplicated. The -command will start one B per CPU and read I one time -per CPU, but as that is done in parallel, all reads except the first -will be cached in RAM. Depending on the size of I it may -be faster to use B<--block 10m> instead of B<-L1000>. If I -is too big to fit in RAM, remove B<--round-robin> and adjust -B<-L1000>. This will cause I to be read more times. +It is harder to see if disk I/O is the limiting factor, and depending +on the disk system it may be faster or slower to parallelize. The only +way to know for certain is to test and measure. + + +=head2 Limiting factor: RAM + +The normal B works no matter the size of +bigfile, but if regexps.txt is so big it cannot fit into memory, then +you need to split this. + +B takes around 100 bytes of RAM and B takes about 500 +bytes of RAM per 1 byte of regexp. So if regexps.txt is 1% of your +RAM, then it may be too big. + +If you can convert your regexps into fixed strings do that. E.g. if +the lines you are looking for in bigfile all looks like: + + ID1 foo bar baz Identifier1 quux + fubar ID2 foo bar baz Identifier2 + +then your regexps.txt can be converted from: + + ID1.*Identifier1 + ID2.*Identifier2 + +into: + + ID1 foo bar baz Identifier1 + ID2 foo bar baz Identifier2 + +This way you can use B which takes around 80% less memory and +is much faster. + +If it still does not fit in memory you can do this: + + parallel --pipepart -a regexps.txt --block 1M grep -F -f - -n bigfile | + sort -un | perl -pe 's/^\d+://' + +The 1M should be your free memory divided by the number of cores and +divided by 200 for B and by 1000 for normal B. On +GNU/Linux you can do: + + free=$(awk '/^((Swap)?Cached|MemFree|Buffers):/ { sum += $2 } + END { print sum }' /proc/meminfo) + percpu=$((free / 200 / $(parallel --number-of-cores)))k + + parallel --pipepart -a regexps.txt --block $percpu --compress grep -F -f - -n bigfile | + sort -un | perl -pe 's/^\d+://' + +If you can live with duplicated lines and wrong order, it is faster to do: + + parallel --pipepart -a regexps.txt --block $percpu --compress grep -F -f - bigfile + +=head2 Limiting factor: CPU + +If the CPU is the limiting factor parallelization should be done on +the regexps: + + cat regexp.txt | parallel --pipe -L1000 --round-robin --compress grep -f - -n bigfile | + sort -un | perl -pe 's/^\d+://' + +The command will start one B per CPU and read I one +time per CPU, but as that is done in parallel, all reads except the +first will be cached in RAM. Depending on the size of I it +may be faster to use B<--block 10m> instead of B<-L1000>. Some storage systems perform better when reading multiple chunks in parallel. This is true for some RAID systems and for some network file systems. To parallelize the reading of I: - parallel --pipepart --block 100M -a bigfile -k grep -f regexp.txt + parallel --pipepart --block 100M -a bigfile -k --compress grep -f regexp.txt This will split I into 100MB chunks and run B on each of these chunks. To parallelize both reading of I and I @@ -2936,6 +2993,11 @@ combine the two using B<--fifo>: If a line matches multiple regexps, the line may be duplicated. +=head2 Bigger problem + +If the problem is too big to be solved by this, you are probably ready +for Lucene. + =head1 EXAMPLE: Using remote computers diff --git a/testsuite/tests-to-run/parallel-local-sql.sh b/testsuite/tests-to-run/parallel-local-sql.sh old mode 100644 new mode 100755 index c1fd7458..12f4da79 --- a/testsuite/tests-to-run/parallel-local-sql.sh +++ b/testsuite/tests-to-run/parallel-local-sql.sh @@ -1,75 +1,81 @@ #!/bin/bash export SQLITE=sqlite3:///%2Frun%2Fshm%2Fparallel.db -export SQLITETBL=$SQLITE/parsql -export PG=pg://tange:tange@lo/tange -export PGTBL=$PG/parsql -export MYSQL=mysql://`whoami`:`whoami`@lo/tange -export MYSQLTBL=$MYSQL/parsql -export PGTBL2=${PGTBL}2 -export PGTBL3=${PGTBL}3 -export PGTBL4=${PGTBL}4 -export PGTBL5=${PGTBL}5 -export T1=$(tempfile) -export T2=$(tempfile) -export T3=$(tempfile) -export T4=$(tempfile) -export T5=$(tempfile) -export T6=$(tempfile) -export T7=$(tempfile) -export T8=$(tempfile) -export T9=$(tempfile) -export T10=$(tempfile) -export T11=$(tempfile) -export T12=$(tempfile) -export T13=$(tempfile) -export T14=$(tempfile) +export PG=pg://`whoami`:`whoami`@lo/`whoami` +export MYSQL=mysql://`whoami`:`whoami`@lo/`whoami` -#sql mysql://tange:tange@lo/ 'create database tange;'; -cat <<'EOF' | sed -e 's/;$/; /;' | stdout parallel -vj0 -k --joblog /tmp/jl-`basename $0` -L1 | perl -pe 's/\s*\d+\.?\d+\s*/999/g;s/999e+999.\s+.\s+/999e+999|999/g;' -echo '### --sqlandworker mysql' - (sleep 2; parallel --sqlworker $MYSQLTBL sleep .3\;echo >$T1) & - parallel --sqlandworker $MYSQLTBL sleep .3\;echo ::: {1..5} ::: {a..e} >$T2; - true sort -u $T1 $T2; - sleep 1; sql $MYSQL 'select * from parsql order by seq;' +export DEBUG=false -echo '### --sqlandworker postgresql' - (sleep 2; parallel --sqlworker $PGTBL sleep .3\;echo >$T3) & - parallel --sqlandworker $PGTBL sleep .3\;echo ::: {1..5} ::: {a..e} >$T4; - true sort -u $T3 $T4; - sleep 1; sql $PG 'select * from parsql order by seq;' +p_showsqlresult() { + SERVERURL=$1 + TABLE=$2 + sql $SERVERURL "select Host,Command,V1,V2,Stdout,Stderr from $TABLE order by seq;" +} -echo '### --sqlandworker sqlite' - (sleep 2; parallel --sqlworker $SQLITETBL sleep .3\;echo >$T5) & - parallel --sqlandworker $SQLITETBL sleep .3\;echo ::: {1..5} ::: {a..e} >$T6; - true sort -u $T5 $T6; - sleep 1; sql $SQLITE 'select * from parsql order by seq;' +p_wrapper() { + INNER=$1 + SERVERURL=$2 + TABLE=TBL$RANDOM + DBURL=$SERVERURL/$TABLE + T1=$(tempfile) + T2=$(tempfile) + eval "$INNER" + echo Exit=$? + wait + echo Exit=$? + $DEBUG && sort -u $T1 $T2; + rm $T1 $T2 + p_showsqlresult $SERVERURL $TABLE + $DEBUG || sql $SERVERURL "drop table $TABLE;" >/dev/null +} -echo '### --sqlandworker postgresql -S lo' - (sleep 2; parallel -S lo --sqlworker $PGTBL2 sleep .3\;echo >$T7) & - parallel -S lo --sqlandworker $PGTBL2 sleep .3\;echo ::: {1..5} ::: {a..e} >$T8; - true sort -u $T7 $T8; - sleep 1; sql $PG 'select * from parsql2 order by seq;' +p_sqlandworker() { + (sleep 2; parallel --sqlworker $DBURL sleep .3\;echo >$T1) & + parallel --sqlandworker $DBURL sleep .3\;echo ::: {1..5} ::: {a..e} >$T2; +} +export -f p_sqlandworker -echo '### --sqlandworker postgresql --results' - mkdir -p /tmp/out--sql - (sleep 2; parallel --results /tmp/out--sql --sqlworker $PGTBL3 sleep .3\;echo >$T9) & - parallel --results /tmp/out--sql --sqlandworker $PGTBL3 sleep .3\;echo ::: {1..5} ::: {a..e} >$T10; - true sort -u $T9 $T10; - sleep 1; sql $PG 'select * from parsql3 order by seq;' +par_sqlandworker() { + p_wrapper p_sqlandworker $1 +} -echo '### --sqlandworker postgresql --linebuffer' - (sleep 2; parallel --linebuffer --sqlworker $PGTBL4 sleep .3\;echo >$T11) & - parallel --linebuffer --sqlandworker $PGTBL4 sleep .3\;echo ::: {1..5} ::: {a..e} >$T12; - true sort -u $T11 $T12; - sleep 1; sql $PG 'select * from parsql4 order by seq;' +p_sqlandworker_lo() { + (sleep 2; parallel -S lo --sqlworker $DBURL sleep .3\;echo >$T1) & + parallel -S lo --sqlandworker $DBURL sleep .3\;echo ::: {1..5} ::: {a..e} >$T2; +} -echo '### --sqlandworker postgresql -u' - (sleep 2; parallel -u --sqlworker $PGTBL5 sleep .3\;echo >$T13) & - parallel -u --sqlandworker $PGTBL5 sleep .3\;echo ::: {1..5} ::: {a..e} >$T14; - true sort -u $T13 $T14; - sleep 1; sql $PG 'select * from parsql5 order by seq;' +par_sqlandworker_lo() { + p_wrapper p_sqlandworker_lo $1 +} -EOF +p_sqlandworker_results() { + (sleep 2; parallel --results /tmp/out--sql --sqlworker $DBURL sleep .3\;echo >$T1) & + parallel --results /tmp/out--sql --sqlandworker $DBURL sleep .3\;echo ::: {1..5} ::: {a..e} >$T2; +} -eval rm '$T'{1..14} +par_sqlandworker_results() { + p_wrapper p_sqlandworker_results $1 +} + +p_sqlandworker_linebuffer() { + (sleep 2; parallel --linebuffer --sqlworker $DBURL sleep .3\;echo >$T1) & + parallel --linebuffer --sqlandworker $DBURL sleep .3\;echo ::: {1..5} ::: {a..e} >$T2; +} + +par_sqlandworker_linebuffer() { + p_wrapper p_sqlandworker_linebuffer $1 +} + +p_sqlandworker_unbuffer() { + (sleep 2; parallel -u --sqlworker $DBURL sleep .3\;echo >$T1) & + parallel -u --sqlandworker $DBURL sleep .3\;echo ::: {1..5} ::: {a..e} >$T2; +} + +par_sqlandworker_unbuffer() { + p_wrapper p_sqlandworker_unbuffer $1 +} + +export -f $(compgen -A function | egrep 'p_|par_') +# Tested that -j0 in parallel is fastest (up to 15 jobs) +compgen -A function | grep par_ | sort | + stdout parallel -vj0 -k --tag --joblog /tmp/jl-`basename $0` :::: - ::: \$MYSQL \$PG \$SQLITE diff --git a/testsuite/wanted-results/parallel-local-sql b/testsuite/wanted-results/parallel-local-sql index f0b047e3..7f9aa6a6 100644 --- a/testsuite/wanted-results/parallel-local-sql +++ b/testsuite/wanted-results/parallel-local-sql @@ -1,301 +1,550 @@ -echo '### --sqlandworker mysql' -### --sqlandworker mysql - (sleep 2; parallel --sqlworker $MYSQLTBL sleep .3\;echo >$T1) & - parallel --sqlandworker $MYSQLTBL sleep .3\;echo ::: {1..5} ::: {a..e} >$T2; true sort -u $T1 $T2; sleep 1; sql $MYSQL 'select * from parsql order by seq;' -Seq Host Starttime JobRuntime Send Receive Exitval _Signal Command V1 V2 Stdout Stderr -1 :9999990 4 0 0 sleep .3;echo 1 a 1 a 1 a\n -2 :9999990 4 0 0 sleep .3;echo 1 b 1 b 1 b\n -3 :9999990 4 0 0 sleep .3;echo 1 c 1 c 1 c\n -4 :9999990 4 0 0 sleep .3;echo 1 d 1 d 1 d\n -5 :9999990 4 0 0 sleep .3;echo 1 e 1 e 1 e\n -6 :9999990 4 0 0 sleep .3;echo 2 a 2 a 2 a\n -7 :9999990 4 0 0 sleep .3;echo 2 b 2 b 2 b\n -8 :9999990 4 0 0 sleep .3;echo 2 c 2 c 2 c\n -9 :9999990 4 0 0 sleep .3;echo 2 d 2 d 2 d\n -999:9999990 4 0 0 sleep .3;echo 2 e 2 e 2 e\n -999:9999990 4 0 0 sleep .3;echo 3 a 3 a 3 a\n -999:9999990 4 0 0 sleep .3;echo 3 b 3 b 3 b\n -999:9999990 4 0 0 sleep .3;echo 3 c 3 c 3 c\n -999:9999990 4 0 0 sleep .3;echo 3 d 3 d 3 d\n -999:9999990 4 0 0 sleep .3;echo 3 e 3 e 3 e\n -999:9999990 4 0 0 sleep .3;echo 4 a 4 a 4 a\n -999:9999990 4 0 0 sleep .3;echo 4 b 4 b 4 b\n -999:9999990 4 0 0 sleep .3;echo 4 c 4 c 4 c\n -999:9999990 4 0 0 sleep .3;echo 4 d 4 d 4 d\n -999:9999990 4 0 0 sleep .3;echo 4 e 4 e 4 e\n -999:9999990 4 0 0 sleep .3;echo 5 a 5 a 5 a\n -999:9999990 4 0 0 sleep .3;echo 5 b 5 b 5 b\n -999:9999990 4 0 0 sleep .3;echo 5 c 5 c 5 c\n -999:9999990 4 0 0 sleep .3;echo 5 d 5 d 5 d\n -999:9999990 4 0 0 sleep .3;echo 5 e 5 e 5 e\n -echo '### --sqlandworker postgresql' -### --sqlandworker postgresql - (sleep 2; parallel --sqlworker $PGTBL sleep .3\;echo >$T3) & - parallel --sqlandworker $PGTBL sleep .3\;echo ::: {1..5} ::: {a..e} >$T4; true sort -u $T3 $T4; sleep 1; sql $PG 'select * from parsql order by seq;' - seq | host | starttime | jobruntime | send | receive | exitval | _signal | command | v1 | v2 | stdout | stderr ------+------+----------------+------------+------+---------+---------+---------+-------------------+----+----+--------+-------- - 1 | : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 1 a | 1 | a | 1 a +| - | | | | | | | | | | | | - 2 | : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 1 b | 1 | b | 1 b +| - | | | | | | | | | | | | - 3 | : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 1 c | 1 | c | 1 c +| - | | | | | | | | | | | | - 4 | : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 1 d | 1 | d | 1 d +| - | | | | | | | | | | | | - 5 | : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 1 e | 1 | e | 1 e +| - | | | | | | | | | | | | - 6 | : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 2 a | 2 | a | 2 a +| - | | | | | | | | | | | | - 7 | : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 2 b | 2 | b | 2 b +| - | | | | | | | | | | | | - 8 | : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 2 c | 2 | c | 2 c +| - | | | | | | | | | | | | - 9 | : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 2 d | 2 | d | 2 d +| - | | | | | | | | | | | | -999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 2 e | 2 | e | 2 e +| - | | | | | | | | | | | | -999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 3 a | 3 | a | 3 a +| - | | | | | | | | | | | | -999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 3 b | 3 | b | 3 b +| - | | | | | | | | | | | | -999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 3 c | 3 | c | 3 c +| - | | | | | | | | | | | | -999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 3 d | 3 | d | 3 d +| - | | | | | | | | | | | | -999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 3 e | 3 | e | 3 e +| - | | | | | | | | | | | | -999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 4 a | 4 | a | 4 a +| - | | | | | | | | | | | | -999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 4 b | 4 | b | 4 b +| - | | | | | | | | | | | | -999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 4 c | 4 | c | 4 c +| - | | | | | | | | | | | | -999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 4 d | 4 | d | 4 d +| - | | | | | | | | | | | | -999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 4 e | 4 | e | 4 e +| - | | | | | | | | | | | | -999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 5 a | 5 | a | 5 a +| - | | | | | | | | | | | | -999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 5 b | 5 | b | 5 b +| - | | | | | | | | | | | | -999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 5 c | 5 | c | 5 c +| - | | | | | | | | | | | | -999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 5 d | 5 | d | 5 d +| - | | | | | | | | | | | | -999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 5 e | 5 | e | 5 e +| - | | | | | | | | | | | | -(999rows) - -echo '### --sqlandworker sqlite' -### --sqlandworker sqlite - (sleep 2; parallel --sqlworker $SQLITETBL sleep .3\;echo >$T5) & - parallel --sqlandworker $SQLITETBL sleep .3\;echo ::: {1..5} ::: {a..e} >$T6; true sort -u $T5 $T6; sleep 1; sql $SQLITE 'select * from parsql order by seq;' -Seq|Host|Starttime|JobRuntime|Send|Receive|Exitval|_Signal|Command|V1|V2|Stdout|Stderr -1|:|999|999|0|4|0|0|sleep .3;echo 1 a|1|a|1 a -| -2|:|999|999|0|4|0|0|sleep .3;echo 1 b|1|b|1 b -| -3|:|999|999|0|4|0|0|sleep .3;echo 1 c|1|c|1 c -| -4|:|999|999|0|4|0|0|sleep .3;echo 1 d|1|d|1 d -| -5|:|999|999|0|4|0|0|sleep .3;echo 1 e|1|e|1 e -| -6|:|999|999|0|4|0|0|sleep .3;echo 2 a|2|a|2 a -| -7|:|999|999|0|4|0|0|sleep .3;echo 2 b|2|b|2 b -| -8|:|999|999|0|4|0|0|sleep .3;echo 2 c|2|c|2 c -| -9|:|999|999|0|4|0|0|sleep .3;echo 2 d|2|d|2 d -| -999|:|999|999|0|4|0|0|sleep .3;echo 2 e|2|e|2 e -| -999|:|999|999|0|4|0|0|sleep .3;echo 3 a|3|a|3 a -| -999|:|999|999|0|4|0|0|sleep .3;echo 3 b|3|b|3 b -| -999|:|999|999|0|4|0|0|sleep .3;echo 3 c|3|c|3 c -| -999|:|999|999|0|4|0|0|sleep .3;echo 3 d|3|d|3 d -| -999|:|999|999|0|4|0|0|sleep .3;echo 3 e|3|e|3 e -| -999|:|999|999|0|4|0|0|sleep .3;echo 4 a|4|a|4 a -| -999|:|999|999|0|4|0|0|sleep .3;echo 4 b|4|b|4 b -| -999|:|999|999|0|4|0|0|sleep .3;echo 4 c|4|c|4 c -| -999|:|999|999|0|4|0|0|sleep .3;echo 4 d|4|d|4 d -| -999|:|999|999|0|4|0|0|sleep .3;echo 4 e|4|e|4 e -| -999|:|999|999|0|4|0|0|sleep .3;echo 5 a|5|a|5 a -| -999|:|999|999|0|4|0|0|sleep .3;echo 5 b|5|b|5 b -| -999|:|999|999|0|4|0|0|sleep .3;echo 5 c|5|c|5 c -| -999|:|999|999|0|4|0|0|sleep .3;echo 5 d|5|d|5 d -| -999|:|999|999|0|4|0|0|sleep .3;echo 5 e|5|e|5 e -| -echo '### --sqlandworker postgresql -S lo' -### --sqlandworker postgresql -S lo - (sleep 2; parallel -S lo --sqlworker $PGTBL2 sleep .3\;echo >$T7) & - parallel -S lo --sqlandworker $PGTBL2 sleep .3\;echo ::: {1..5} ::: {a..e} >$T8; true sort -u $T7 $T8; sleep 1; sql $PG 'select * from parsql2 order by seq;' - seq | host | starttime | jobruntime | send | receive | exitval | _signal | command | v1 | v2 | stdout | stderr ------+------+----------------+------------+------+---------+---------+---------+-------------------+----+----+--------+-------- - 1 | lo |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 1 a | 1 | a | 1 a +| - | | | | | | | | | | | | - 2 | lo |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 1 b | 1 | b | 1 b +| - | | | | | | | | | | | | - 3 | lo |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 1 c | 1 | c | 1 c +| - | | | | | | | | | | | | - 4 | lo |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 1 d | 1 | d | 1 d +| - | | | | | | | | | | | | - 5 | lo |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 1 e | 1 | e | 1 e +| - | | | | | | | | | | | | - 6 | lo |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 2 a | 2 | a | 2 a +| - | | | | | | | | | | | | - 7 | lo |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 2 b | 2 | b | 2 b +| - | | | | | | | | | | | | - 8 | lo |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 2 c | 2 | c | 2 c +| - | | | | | | | | | | | | - 9 | lo |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 2 d | 2 | d | 2 d +| - | | | | | | | | | | | | -999| lo |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 2 e | 2 | e | 2 e +| - | | | | | | | | | | | | -999| lo |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 3 a | 3 | a | 3 a +| - | | | | | | | | | | | | -999| lo |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 3 b | 3 | b | 3 b +| - | | | | | | | | | | | | -999| lo |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 3 c | 3 | c | 3 c +| - | | | | | | | | | | | | -999| lo |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 3 d | 3 | d | 3 d +| - | | | | | | | | | | | | -999| lo |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 3 e | 3 | e | 3 e +| - | | | | | | | | | | | | -999| lo |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 4 a | 4 | a | 4 a +| - | | | | | | | | | | | | -999| lo |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 4 b | 4 | b | 4 b +| - | | | | | | | | | | | | -999| lo |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 4 c | 4 | c | 4 c +| - | | | | | | | | | | | | -999| lo |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 4 d | 4 | d | 4 d +| - | | | | | | | | | | | | -999| lo |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 4 e | 4 | e | 4 e +| - | | | | | | | | | | | | -999| lo |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 5 a | 5 | a | 5 a +| - | | | | | | | | | | | | -999| lo |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 5 b | 5 | b | 5 b +| - | | | | | | | | | | | | -999| lo |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 5 c | 5 | c | 5 c +| - | | | | | | | | | | | | -999| lo |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 5 d | 5 | d | 5 d +| - | | | | | | | | | | | | -999| lo |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 5 e | 5 | e | 5 e +| - | | | | | | | | | | | | -(999rows) - -echo '### --sqlandworker postgresql --results' -### --sqlandworker postgresql --results - mkdir -p /tmp/out--sql - (sleep 2; parallel --results /tmp/out--sql --sqlworker $PGTBL3 sleep .3\;echo >$T9) & - parallel --results /tmp/out--sql --sqlandworker $PGTBL3 sleep .3\;echo ::: {1..5} ::: {a..e} >$T999; true sort -u $T9 $T999; sleep 1; sql $PG 'select * from parsql3 order by seq;' - seq | host | starttime | jobruntime | send | receive | exitval | _signal | command | v1 | v2 | stdout | stderr ------+------+----------------+------------+------+---------+---------+---------+-------------------+----+----+------------------------------+------------------------------ - 1 | : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 1 a | 1 | a | /tmp/out--sql/1/1/2/a/stdout | /tmp/out--sql/1/1/2/a/stderr - 2 | : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 1 b | 1 | b | /tmp/out--sql/1/1/2/b/stdout | /tmp/out--sql/1/1/2/b/stderr - 3 | : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 1 c | 1 | c | /tmp/out--sql/1/1/2/c/stdout | /tmp/out--sql/1/1/2/c/stderr - 4 | : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 1 d | 1 | d | /tmp/out--sql/1/1/2/d/stdout | /tmp/out--sql/1/1/2/d/stderr - 5 | : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 1 e | 1 | e | /tmp/out--sql/1/1/2/e/stdout | /tmp/out--sql/1/1/2/e/stderr - 6 | : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 2 a | 2 | a | /tmp/out--sql/1/2/2/a/stdout | /tmp/out--sql/1/2/2/a/stderr - 7 | : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 2 b | 2 | b | /tmp/out--sql/1/2/2/b/stdout | /tmp/out--sql/1/2/2/b/stderr - 8 | : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 2 c | 2 | c | /tmp/out--sql/1/2/2/c/stdout | /tmp/out--sql/1/2/2/c/stderr - 9 | : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 2 d | 2 | d | /tmp/out--sql/1/2/2/d/stdout | /tmp/out--sql/1/2/2/d/stderr -999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 2 e | 2 | e | /tmp/out--sql/1/2/2/e/stdout | /tmp/out--sql/1/2/2/e/stderr -999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 3 a | 3 | a | /tmp/out--sql/1/3/2/a/stdout | /tmp/out--sql/1/3/2/a/stderr -999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 3 b | 3 | b | /tmp/out--sql/1/3/2/b/stdout | /tmp/out--sql/1/3/2/b/stderr -999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 3 c | 3 | c | /tmp/out--sql/1/3/2/c/stdout | /tmp/out--sql/1/3/2/c/stderr -999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 3 d | 3 | d | /tmp/out--sql/1/3/2/d/stdout | /tmp/out--sql/1/3/2/d/stderr -999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 3 e | 3 | e | /tmp/out--sql/1/3/2/e/stdout | /tmp/out--sql/1/3/2/e/stderr -999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 4 a | 4 | a | /tmp/out--sql/1/4/2/a/stdout | /tmp/out--sql/1/4/2/a/stderr -999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 4 b | 4 | b | /tmp/out--sql/1/4/2/b/stdout | /tmp/out--sql/1/4/2/b/stderr -999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 4 c | 4 | c | /tmp/out--sql/1/4/2/c/stdout | /tmp/out--sql/1/4/2/c/stderr -999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 4 d | 4 | d | /tmp/out--sql/1/4/2/d/stdout | /tmp/out--sql/1/4/2/d/stderr -999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 4 e | 4 | e | /tmp/out--sql/1/4/2/e/stdout | /tmp/out--sql/1/4/2/e/stderr -999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 5 a | 5 | a | /tmp/out--sql/1/5/2/a/stdout | /tmp/out--sql/1/5/2/a/stderr -999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 5 b | 5 | b | /tmp/out--sql/1/5/2/b/stdout | /tmp/out--sql/1/5/2/b/stderr -999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 5 c | 5 | c | /tmp/out--sql/1/5/2/c/stdout | /tmp/out--sql/1/5/2/c/stderr -999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 5 d | 5 | d | /tmp/out--sql/1/5/2/d/stdout | /tmp/out--sql/1/5/2/d/stderr -999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 5 e | 5 | e | /tmp/out--sql/1/5/2/e/stdout | /tmp/out--sql/1/5/2/e/stderr -(999rows) - -echo '### --sqlandworker postgresql --linebuffer' -### --sqlandworker postgresql --linebuffer - (sleep 2; parallel --linebuffer --sqlworker $PGTBL4 sleep .3\;echo >$T999) & - parallel --linebuffer --sqlandworker $PGTBL4 sleep .3\;echo ::: {1..5} ::: {a..e} >$T999; true sort -u $T999$T999; sleep 1; sql $PG 'select * from parsql4 order by seq;' - seq | host | starttime | jobruntime | send | receive | exitval | _signal | command | v1 | v2 | stdout | stderr ------+------+----------------+------------+------+---------+---------+---------+-------------------+----+----+--------+-------- - 1 | : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 1 a | 1 | a | | - 2 | : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 1 b | 1 | b | | - 3 | : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 1 c | 1 | c | | - 4 | : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 1 d | 1 | d | | - 5 | : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 1 e | 1 | e | | - 6 | : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 2 a | 2 | a | | - 7 | : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 2 b | 2 | b | | - 8 | : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 2 c | 2 | c | | - 9 | : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 2 d | 2 | d | | -999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 2 e | 2 | e | | -999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 3 a | 3 | a | | -999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 3 b | 3 | b | | -999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 3 c | 3 | c | | -999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 3 d | 3 | d | | -999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 3 e | 3 | e | | -999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 4 a | 4 | a | | -999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 4 b | 4 | b | | -999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 4 c | 4 | c | | -999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 4 d | 4 | d | | -999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 4 e | 4 | e | | -999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 5 a | 5 | a | | -999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 5 b | 5 | b | | -999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 5 c | 5 | c | | -999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 5 d | 5 | d | | -999| : |999|999| 0 | 4 | 0 | 0 | sleep .3;echo 5 e | 5 | e | | -(999rows) - -echo '### --sqlandworker postgresql -u' -### --sqlandworker postgresql -u - (sleep 2; parallel -u --sqlworker $PGTBL5 sleep .3\;echo >$T999) & - parallel -u --sqlandworker $PGTBL5 sleep .3\;echo ::: {1..5} ::: {a..e} >$T999; true sort -u $T999$T999; sleep 1; sql $PG 'select * from parsql5 order by seq;' - seq | host | starttime | jobruntime | send | receive | exitval | _signal | command | v1 | v2 | stdout | stderr ------+------+----------------+------------+------+---------+---------+---------+-------------------+----+----+--------+-------- - 1 | : |999|999| 0 | 0 | 0 | 0 | sleep .3;echo 1 a | 1 | a | | - 2 | : |999|999| 0 | 0 | 0 | 0 | sleep .3;echo 1 b | 1 | b | | - 3 | : |999|999| 0 | 0 | 0 | 0 | sleep .3;echo 1 c | 1 | c | | - 4 | : |999|999| 0 | 0 | 0 | 0 | sleep .3;echo 1 d | 1 | d | | - 5 | : |999|999| 0 | 0 | 0 | 0 | sleep .3;echo 1 e | 1 | e | | - 6 | : |999|999| 0 | 0 | 0 | 0 | sleep .3;echo 2 a | 2 | a | | - 7 | : |999|999| 0 | 0 | 0 | 0 | sleep .3;echo 2 b | 2 | b | | - 8 | : |999|999| 0 | 0 | 0 | 0 | sleep .3;echo 2 c | 2 | c | | - 9 | : |999|999| 0 | 0 | 0 | 0 | sleep .3;echo 2 d | 2 | d | | -999| : |999|999| 0 | 0 | 0 | 0 | sleep .3;echo 2 e | 2 | e | | -999| : |999|999| 0 | 0 | 0 | 0 | sleep .3;echo 3 a | 3 | a | | -999| : |999|999| 0 | 0 | 0 | 0 | sleep .3;echo 3 b | 3 | b | | -999| : |999|999| 0 | 0 | 0 | 0 | sleep .3;echo 3 c | 3 | c | | -999| : |999|999| 0 | 0 | 0 | 0 | sleep .3;echo 3 d | 3 | d | | -999| : |999|999| 0 | 0 | 0 | 0 | sleep .3;echo 3 e | 3 | e | | -999| : |999|999| 0 | 0 | 0 | 0 | sleep .3;echo 4 a | 4 | a | | -999| : |999|999| 0 | 0 | 0 | 0 | sleep .3;echo 4 b | 4 | b | | -999| : |999|999| 0 | 0 | 0 | 0 | sleep .3;echo 4 c | 4 | c | | -999| : |999|999| 0 | 0 | 0 | 0 | sleep .3;echo 4 d | 4 | d | | -999| : |999|999| 0 | 0 | 0 | 0 | sleep .3;echo 4 e | 4 | e | | -999| : |999|999| 0 | 0 | 0 | 0 | sleep .3;echo 5 a | 5 | a | | -999| : |999|999| 0 | 0 | 0 | 0 | sleep .3;echo 5 b | 5 | b | | -999| : |999|999| 0 | 0 | 0 | 0 | sleep .3;echo 5 c | 5 | c | | -999| : |999|999| 0 | 0 | 0 | 0 | sleep .3;echo 5 d | 5 | d | | -999| : |999|999| 0 | 0 | 0 | 0 | sleep .3;echo 5 e | 5 | e | | -(999rows) - +par_sqlandworker $MYSQL +par_sqlandworker $MYSQL Exit=0 +par_sqlandworker $MYSQL Exit=0 +par_sqlandworker $MYSQL Host Command V1 V2 Stdout Stderr +par_sqlandworker $MYSQL : sleep .3;echo 1 a 1 a 1 a\n +par_sqlandworker $MYSQL : sleep .3;echo 1 b 1 b 1 b\n +par_sqlandworker $MYSQL : sleep .3;echo 1 c 1 c 1 c\n +par_sqlandworker $MYSQL : sleep .3;echo 1 d 1 d 1 d\n +par_sqlandworker $MYSQL : sleep .3;echo 1 e 1 e 1 e\n +par_sqlandworker $MYSQL : sleep .3;echo 2 a 2 a 2 a\n +par_sqlandworker $MYSQL : sleep .3;echo 2 b 2 b 2 b\n +par_sqlandworker $MYSQL : sleep .3;echo 2 c 2 c 2 c\n +par_sqlandworker $MYSQL : sleep .3;echo 2 d 2 d 2 d\n +par_sqlandworker $MYSQL : sleep .3;echo 2 e 2 e 2 e\n +par_sqlandworker $MYSQL : sleep .3;echo 3 a 3 a 3 a\n +par_sqlandworker $MYSQL : sleep .3;echo 3 b 3 b 3 b\n +par_sqlandworker $MYSQL : sleep .3;echo 3 c 3 c 3 c\n +par_sqlandworker $MYSQL : sleep .3;echo 3 d 3 d 3 d\n +par_sqlandworker $MYSQL : sleep .3;echo 3 e 3 e 3 e\n +par_sqlandworker $MYSQL : sleep .3;echo 4 a 4 a 4 a\n +par_sqlandworker $MYSQL : sleep .3;echo 4 b 4 b 4 b\n +par_sqlandworker $MYSQL : sleep .3;echo 4 c 4 c 4 c\n +par_sqlandworker $MYSQL : sleep .3;echo 4 d 4 d 4 d\n +par_sqlandworker $MYSQL : sleep .3;echo 4 e 4 e 4 e\n +par_sqlandworker $MYSQL : sleep .3;echo 5 a 5 a 5 a\n +par_sqlandworker $MYSQL : sleep .3;echo 5 b 5 b 5 b\n +par_sqlandworker $MYSQL : sleep .3;echo 5 c 5 c 5 c\n +par_sqlandworker $MYSQL : sleep .3;echo 5 d 5 d 5 d\n +par_sqlandworker $MYSQL : sleep .3;echo 5 e 5 e 5 e\n +par_sqlandworker $PG +par_sqlandworker $PG Exit=0 +par_sqlandworker $PG Exit=0 +par_sqlandworker $PG host | command | v1 | v2 | stdout | stderr +par_sqlandworker $PG ------+-------------------+----+----+--------+-------- +par_sqlandworker $PG : | sleep .3;echo 1 a | 1 | a | 1 a +| +par_sqlandworker $PG | | | | | +par_sqlandworker $PG : | sleep .3;echo 1 b | 1 | b | 1 b +| +par_sqlandworker $PG | | | | | +par_sqlandworker $PG : | sleep .3;echo 1 c | 1 | c | 1 c +| +par_sqlandworker $PG | | | | | +par_sqlandworker $PG : | sleep .3;echo 1 d | 1 | d | 1 d +| +par_sqlandworker $PG | | | | | +par_sqlandworker $PG : | sleep .3;echo 1 e | 1 | e | 1 e +| +par_sqlandworker $PG | | | | | +par_sqlandworker $PG : | sleep .3;echo 2 a | 2 | a | 2 a +| +par_sqlandworker $PG | | | | | +par_sqlandworker $PG : | sleep .3;echo 2 b | 2 | b | 2 b +| +par_sqlandworker $PG | | | | | +par_sqlandworker $PG : | sleep .3;echo 2 c | 2 | c | 2 c +| +par_sqlandworker $PG | | | | | +par_sqlandworker $PG : | sleep .3;echo 2 d | 2 | d | 2 d +| +par_sqlandworker $PG | | | | | +par_sqlandworker $PG : | sleep .3;echo 2 e | 2 | e | 2 e +| +par_sqlandworker $PG | | | | | +par_sqlandworker $PG : | sleep .3;echo 3 a | 3 | a | 3 a +| +par_sqlandworker $PG | | | | | +par_sqlandworker $PG : | sleep .3;echo 3 b | 3 | b | 3 b +| +par_sqlandworker $PG | | | | | +par_sqlandworker $PG : | sleep .3;echo 3 c | 3 | c | 3 c +| +par_sqlandworker $PG | | | | | +par_sqlandworker $PG : | sleep .3;echo 3 d | 3 | d | 3 d +| +par_sqlandworker $PG | | | | | +par_sqlandworker $PG : | sleep .3;echo 3 e | 3 | e | 3 e +| +par_sqlandworker $PG | | | | | +par_sqlandworker $PG : | sleep .3;echo 4 a | 4 | a | 4 a +| +par_sqlandworker $PG | | | | | +par_sqlandworker $PG : | sleep .3;echo 4 b | 4 | b | 4 b +| +par_sqlandworker $PG | | | | | +par_sqlandworker $PG : | sleep .3;echo 4 c | 4 | c | 4 c +| +par_sqlandworker $PG | | | | | +par_sqlandworker $PG : | sleep .3;echo 4 d | 4 | d | 4 d +| +par_sqlandworker $PG | | | | | +par_sqlandworker $PG : | sleep .3;echo 4 e | 4 | e | 4 e +| +par_sqlandworker $PG | | | | | +par_sqlandworker $PG : | sleep .3;echo 5 a | 5 | a | 5 a +| +par_sqlandworker $PG | | | | | +par_sqlandworker $PG : | sleep .3;echo 5 b | 5 | b | 5 b +| +par_sqlandworker $PG | | | | | +par_sqlandworker $PG : | sleep .3;echo 5 c | 5 | c | 5 c +| +par_sqlandworker $PG | | | | | +par_sqlandworker $PG : | sleep .3;echo 5 d | 5 | d | 5 d +| +par_sqlandworker $PG | | | | | +par_sqlandworker $PG : | sleep .3;echo 5 e | 5 | e | 5 e +| +par_sqlandworker $PG | | | | | +par_sqlandworker $PG (25 rows) +par_sqlandworker $PG +par_sqlandworker $SQLITE +par_sqlandworker $SQLITE Exit=0 +par_sqlandworker $SQLITE Exit=0 +par_sqlandworker $SQLITE Host|Command|V1|V2|Stdout|Stderr +par_sqlandworker $SQLITE :|sleep .3;echo 1 a|1|a|1 a +par_sqlandworker $SQLITE | +par_sqlandworker $SQLITE :|sleep .3;echo 1 b|1|b|1 b +par_sqlandworker $SQLITE | +par_sqlandworker $SQLITE :|sleep .3;echo 1 c|1|c|1 c +par_sqlandworker $SQLITE | +par_sqlandworker $SQLITE :|sleep .3;echo 1 d|1|d|1 d +par_sqlandworker $SQLITE | +par_sqlandworker $SQLITE :|sleep .3;echo 1 e|1|e|1 e +par_sqlandworker $SQLITE | +par_sqlandworker $SQLITE :|sleep .3;echo 2 a|2|a|2 a +par_sqlandworker $SQLITE | +par_sqlandworker $SQLITE :|sleep .3;echo 2 b|2|b|2 b +par_sqlandworker $SQLITE | +par_sqlandworker $SQLITE :|sleep .3;echo 2 c|2|c|2 c +par_sqlandworker $SQLITE | +par_sqlandworker $SQLITE :|sleep .3;echo 2 d|2|d|2 d +par_sqlandworker $SQLITE | +par_sqlandworker $SQLITE :|sleep .3;echo 2 e|2|e|2 e +par_sqlandworker $SQLITE | +par_sqlandworker $SQLITE :|sleep .3;echo 3 a|3|a|3 a +par_sqlandworker $SQLITE | +par_sqlandworker $SQLITE :|sleep .3;echo 3 b|3|b|3 b +par_sqlandworker $SQLITE | +par_sqlandworker $SQLITE :|sleep .3;echo 3 c|3|c|3 c +par_sqlandworker $SQLITE | +par_sqlandworker $SQLITE :|sleep .3;echo 3 d|3|d|3 d +par_sqlandworker $SQLITE | +par_sqlandworker $SQLITE :|sleep .3;echo 3 e|3|e|3 e +par_sqlandworker $SQLITE | +par_sqlandworker $SQLITE :|sleep .3;echo 4 a|4|a|4 a +par_sqlandworker $SQLITE | +par_sqlandworker $SQLITE :|sleep .3;echo 4 b|4|b|4 b +par_sqlandworker $SQLITE | +par_sqlandworker $SQLITE :|sleep .3;echo 4 c|4|c|4 c +par_sqlandworker $SQLITE | +par_sqlandworker $SQLITE :|sleep .3;echo 4 d|4|d|4 d +par_sqlandworker $SQLITE | +par_sqlandworker $SQLITE :|sleep .3;echo 4 e|4|e|4 e +par_sqlandworker $SQLITE | +par_sqlandworker $SQLITE :|sleep .3;echo 5 a|5|a|5 a +par_sqlandworker $SQLITE | +par_sqlandworker $SQLITE :|sleep .3;echo 5 b|5|b|5 b +par_sqlandworker $SQLITE | +par_sqlandworker $SQLITE :|sleep .3;echo 5 c|5|c|5 c +par_sqlandworker $SQLITE | +par_sqlandworker $SQLITE :|sleep .3;echo 5 d|5|d|5 d +par_sqlandworker $SQLITE | +par_sqlandworker $SQLITE :|sleep .3;echo 5 e|5|e|5 e +par_sqlandworker $SQLITE | +par_sqlandworker_linebuffer $MYSQL +par_sqlandworker_linebuffer $MYSQL Exit=0 +par_sqlandworker_linebuffer $MYSQL Exit=0 +par_sqlandworker_linebuffer $MYSQL Host Command V1 V2 Stdout Stderr +par_sqlandworker_linebuffer $MYSQL : sleep .3;echo 1 a 1 a NULL NULL +par_sqlandworker_linebuffer $MYSQL : sleep .3;echo 1 b 1 b NULL NULL +par_sqlandworker_linebuffer $MYSQL : sleep .3;echo 1 c 1 c NULL NULL +par_sqlandworker_linebuffer $MYSQL : sleep .3;echo 1 d 1 d NULL NULL +par_sqlandworker_linebuffer $MYSQL : sleep .3;echo 1 e 1 e NULL NULL +par_sqlandworker_linebuffer $MYSQL : sleep .3;echo 2 a 2 a NULL NULL +par_sqlandworker_linebuffer $MYSQL : sleep .3;echo 2 b 2 b NULL NULL +par_sqlandworker_linebuffer $MYSQL : sleep .3;echo 2 c 2 c NULL NULL +par_sqlandworker_linebuffer $MYSQL : sleep .3;echo 2 d 2 d NULL NULL +par_sqlandworker_linebuffer $MYSQL : sleep .3;echo 2 e 2 e NULL NULL +par_sqlandworker_linebuffer $MYSQL : sleep .3;echo 3 a 3 a NULL NULL +par_sqlandworker_linebuffer $MYSQL : sleep .3;echo 3 b 3 b NULL NULL +par_sqlandworker_linebuffer $MYSQL : sleep .3;echo 3 c 3 c NULL NULL +par_sqlandworker_linebuffer $MYSQL : sleep .3;echo 3 d 3 d NULL NULL +par_sqlandworker_linebuffer $MYSQL : sleep .3;echo 3 e 3 e NULL NULL +par_sqlandworker_linebuffer $MYSQL : sleep .3;echo 4 a 4 a NULL NULL +par_sqlandworker_linebuffer $MYSQL : sleep .3;echo 4 b 4 b NULL NULL +par_sqlandworker_linebuffer $MYSQL : sleep .3;echo 4 c 4 c NULL NULL +par_sqlandworker_linebuffer $MYSQL : sleep .3;echo 4 d 4 d NULL NULL +par_sqlandworker_linebuffer $MYSQL : sleep .3;echo 4 e 4 e NULL NULL +par_sqlandworker_linebuffer $MYSQL : sleep .3;echo 5 a 5 a NULL NULL +par_sqlandworker_linebuffer $MYSQL : sleep .3;echo 5 b 5 b NULL NULL +par_sqlandworker_linebuffer $MYSQL : sleep .3;echo 5 c 5 c NULL NULL +par_sqlandworker_linebuffer $MYSQL : sleep .3;echo 5 d 5 d NULL NULL +par_sqlandworker_linebuffer $MYSQL : sleep .3;echo 5 e 5 e NULL NULL +par_sqlandworker_linebuffer $PG +par_sqlandworker_linebuffer $PG Exit=0 +par_sqlandworker_linebuffer $PG Exit=0 +par_sqlandworker_linebuffer $PG host | command | v1 | v2 | stdout | stderr +par_sqlandworker_linebuffer $PG ------+-------------------+----+----+--------+-------- +par_sqlandworker_linebuffer $PG : | sleep .3;echo 1 a | 1 | a | | +par_sqlandworker_linebuffer $PG : | sleep .3;echo 1 b | 1 | b | | +par_sqlandworker_linebuffer $PG : | sleep .3;echo 1 c | 1 | c | | +par_sqlandworker_linebuffer $PG : | sleep .3;echo 1 d | 1 | d | | +par_sqlandworker_linebuffer $PG : | sleep .3;echo 1 e | 1 | e | | +par_sqlandworker_linebuffer $PG : | sleep .3;echo 2 a | 2 | a | | +par_sqlandworker_linebuffer $PG : | sleep .3;echo 2 b | 2 | b | | +par_sqlandworker_linebuffer $PG : | sleep .3;echo 2 c | 2 | c | | +par_sqlandworker_linebuffer $PG : | sleep .3;echo 2 d | 2 | d | | +par_sqlandworker_linebuffer $PG : | sleep .3;echo 2 e | 2 | e | | +par_sqlandworker_linebuffer $PG : | sleep .3;echo 3 a | 3 | a | | +par_sqlandworker_linebuffer $PG : | sleep .3;echo 3 b | 3 | b | | +par_sqlandworker_linebuffer $PG : | sleep .3;echo 3 c | 3 | c | | +par_sqlandworker_linebuffer $PG : | sleep .3;echo 3 d | 3 | d | | +par_sqlandworker_linebuffer $PG : | sleep .3;echo 3 e | 3 | e | | +par_sqlandworker_linebuffer $PG : | sleep .3;echo 4 a | 4 | a | | +par_sqlandworker_linebuffer $PG : | sleep .3;echo 4 b | 4 | b | | +par_sqlandworker_linebuffer $PG : | sleep .3;echo 4 c | 4 | c | | +par_sqlandworker_linebuffer $PG : | sleep .3;echo 4 d | 4 | d | | +par_sqlandworker_linebuffer $PG : | sleep .3;echo 4 e | 4 | e | | +par_sqlandworker_linebuffer $PG : | sleep .3;echo 5 a | 5 | a | | +par_sqlandworker_linebuffer $PG : | sleep .3;echo 5 b | 5 | b | | +par_sqlandworker_linebuffer $PG : | sleep .3;echo 5 c | 5 | c | | +par_sqlandworker_linebuffer $PG : | sleep .3;echo 5 d | 5 | d | | +par_sqlandworker_linebuffer $PG : | sleep .3;echo 5 e | 5 | e | | +par_sqlandworker_linebuffer $PG (25 rows) +par_sqlandworker_linebuffer $PG +par_sqlandworker_linebuffer $SQLITE +par_sqlandworker_linebuffer $SQLITE Exit=0 +par_sqlandworker_linebuffer $SQLITE Exit=0 +par_sqlandworker_linebuffer $SQLITE Host|Command|V1|V2|Stdout|Stderr +par_sqlandworker_linebuffer $SQLITE :|sleep .3;echo 1 a|1|a|| +par_sqlandworker_linebuffer $SQLITE :|sleep .3;echo 1 b|1|b|| +par_sqlandworker_linebuffer $SQLITE :|sleep .3;echo 1 c|1|c|| +par_sqlandworker_linebuffer $SQLITE :|sleep .3;echo 1 d|1|d|| +par_sqlandworker_linebuffer $SQLITE :|sleep .3;echo 1 e|1|e|| +par_sqlandworker_linebuffer $SQLITE :|sleep .3;echo 2 a|2|a|| +par_sqlandworker_linebuffer $SQLITE :|sleep .3;echo 2 b|2|b|| +par_sqlandworker_linebuffer $SQLITE :|sleep .3;echo 2 c|2|c|| +par_sqlandworker_linebuffer $SQLITE :|sleep .3;echo 2 d|2|d|| +par_sqlandworker_linebuffer $SQLITE :|sleep .3;echo 2 e|2|e|| +par_sqlandworker_linebuffer $SQLITE :|sleep .3;echo 3 a|3|a|| +par_sqlandworker_linebuffer $SQLITE :|sleep .3;echo 3 b|3|b|| +par_sqlandworker_linebuffer $SQLITE :|sleep .3;echo 3 c|3|c|| +par_sqlandworker_linebuffer $SQLITE :|sleep .3;echo 3 d|3|d|| +par_sqlandworker_linebuffer $SQLITE :|sleep .3;echo 3 e|3|e|| +par_sqlandworker_linebuffer $SQLITE :|sleep .3;echo 4 a|4|a|| +par_sqlandworker_linebuffer $SQLITE :|sleep .3;echo 4 b|4|b|| +par_sqlandworker_linebuffer $SQLITE :|sleep .3;echo 4 c|4|c|| +par_sqlandworker_linebuffer $SQLITE :|sleep .3;echo 4 d|4|d|| +par_sqlandworker_linebuffer $SQLITE :|sleep .3;echo 4 e|4|e|| +par_sqlandworker_linebuffer $SQLITE :|sleep .3;echo 5 a|5|a|| +par_sqlandworker_linebuffer $SQLITE :|sleep .3;echo 5 b|5|b|| +par_sqlandworker_linebuffer $SQLITE :|sleep .3;echo 5 c|5|c|| +par_sqlandworker_linebuffer $SQLITE :|sleep .3;echo 5 d|5|d|| +par_sqlandworker_linebuffer $SQLITE :|sleep .3;echo 5 e|5|e|| +par_sqlandworker_lo $MYSQL +par_sqlandworker_lo $MYSQL Exit=0 +par_sqlandworker_lo $MYSQL Exit=0 +par_sqlandworker_lo $MYSQL Host Command V1 V2 Stdout Stderr +par_sqlandworker_lo $MYSQL lo sleep .3;echo 1 a 1 a 1 a\n +par_sqlandworker_lo $MYSQL lo sleep .3;echo 1 b 1 b 1 b\n +par_sqlandworker_lo $MYSQL lo sleep .3;echo 1 c 1 c 1 c\n +par_sqlandworker_lo $MYSQL lo sleep .3;echo 1 d 1 d 1 d\n +par_sqlandworker_lo $MYSQL lo sleep .3;echo 1 e 1 e 1 e\n +par_sqlandworker_lo $MYSQL lo sleep .3;echo 2 a 2 a 2 a\n +par_sqlandworker_lo $MYSQL lo sleep .3;echo 2 b 2 b 2 b\n +par_sqlandworker_lo $MYSQL lo sleep .3;echo 2 c 2 c 2 c\n +par_sqlandworker_lo $MYSQL lo sleep .3;echo 2 d 2 d 2 d\n +par_sqlandworker_lo $MYSQL lo sleep .3;echo 2 e 2 e 2 e\n +par_sqlandworker_lo $MYSQL lo sleep .3;echo 3 a 3 a 3 a\n +par_sqlandworker_lo $MYSQL lo sleep .3;echo 3 b 3 b 3 b\n +par_sqlandworker_lo $MYSQL lo sleep .3;echo 3 c 3 c 3 c\n +par_sqlandworker_lo $MYSQL lo sleep .3;echo 3 d 3 d 3 d\n +par_sqlandworker_lo $MYSQL lo sleep .3;echo 3 e 3 e 3 e\n +par_sqlandworker_lo $MYSQL lo sleep .3;echo 4 a 4 a 4 a\n +par_sqlandworker_lo $MYSQL lo sleep .3;echo 4 b 4 b 4 b\n +par_sqlandworker_lo $MYSQL lo sleep .3;echo 4 c 4 c 4 c\n +par_sqlandworker_lo $MYSQL lo sleep .3;echo 4 d 4 d 4 d\n +par_sqlandworker_lo $MYSQL lo sleep .3;echo 4 e 4 e 4 e\n +par_sqlandworker_lo $MYSQL lo sleep .3;echo 5 a 5 a 5 a\n +par_sqlandworker_lo $MYSQL lo sleep .3;echo 5 b 5 b 5 b\n +par_sqlandworker_lo $MYSQL lo sleep .3;echo 5 c 5 c 5 c\n +par_sqlandworker_lo $MYSQL lo sleep .3;echo 5 d 5 d 5 d\n +par_sqlandworker_lo $MYSQL lo sleep .3;echo 5 e 5 e 5 e\n +par_sqlandworker_lo $PG +par_sqlandworker_lo $PG Exit=0 +par_sqlandworker_lo $PG Exit=0 +par_sqlandworker_lo $PG host | command | v1 | v2 | stdout | stderr +par_sqlandworker_lo $PG ------+-------------------+----+----+--------+-------- +par_sqlandworker_lo $PG lo | sleep .3;echo 1 a | 1 | a | 1 a +| +par_sqlandworker_lo $PG | | | | | +par_sqlandworker_lo $PG lo | sleep .3;echo 1 b | 1 | b | 1 b +| +par_sqlandworker_lo $PG | | | | | +par_sqlandworker_lo $PG lo | sleep .3;echo 1 c | 1 | c | 1 c +| +par_sqlandworker_lo $PG | | | | | +par_sqlandworker_lo $PG lo | sleep .3;echo 1 d | 1 | d | 1 d +| +par_sqlandworker_lo $PG | | | | | +par_sqlandworker_lo $PG lo | sleep .3;echo 1 e | 1 | e | 1 e +| +par_sqlandworker_lo $PG | | | | | +par_sqlandworker_lo $PG lo | sleep .3;echo 2 a | 2 | a | 2 a +| +par_sqlandworker_lo $PG | | | | | +par_sqlandworker_lo $PG lo | sleep .3;echo 2 b | 2 | b | 2 b +| +par_sqlandworker_lo $PG | | | | | +par_sqlandworker_lo $PG lo | sleep .3;echo 2 c | 2 | c | 2 c +| +par_sqlandworker_lo $PG | | | | | +par_sqlandworker_lo $PG lo | sleep .3;echo 2 d | 2 | d | 2 d +| +par_sqlandworker_lo $PG | | | | | +par_sqlandworker_lo $PG lo | sleep .3;echo 2 e | 2 | e | 2 e +| +par_sqlandworker_lo $PG | | | | | +par_sqlandworker_lo $PG lo | sleep .3;echo 3 a | 3 | a | 3 a +| +par_sqlandworker_lo $PG | | | | | +par_sqlandworker_lo $PG lo | sleep .3;echo 3 b | 3 | b | 3 b +| +par_sqlandworker_lo $PG | | | | | +par_sqlandworker_lo $PG lo | sleep .3;echo 3 c | 3 | c | 3 c +| +par_sqlandworker_lo $PG | | | | | +par_sqlandworker_lo $PG lo | sleep .3;echo 3 d | 3 | d | 3 d +| +par_sqlandworker_lo $PG | | | | | +par_sqlandworker_lo $PG lo | sleep .3;echo 3 e | 3 | e | 3 e +| +par_sqlandworker_lo $PG | | | | | +par_sqlandworker_lo $PG lo | sleep .3;echo 4 a | 4 | a | 4 a +| +par_sqlandworker_lo $PG | | | | | +par_sqlandworker_lo $PG lo | sleep .3;echo 4 b | 4 | b | 4 b +| +par_sqlandworker_lo $PG | | | | | +par_sqlandworker_lo $PG lo | sleep .3;echo 4 c | 4 | c | 4 c +| +par_sqlandworker_lo $PG | | | | | +par_sqlandworker_lo $PG lo | sleep .3;echo 4 d | 4 | d | 4 d +| +par_sqlandworker_lo $PG | | | | | +par_sqlandworker_lo $PG lo | sleep .3;echo 4 e | 4 | e | 4 e +| +par_sqlandworker_lo $PG | | | | | +par_sqlandworker_lo $PG lo | sleep .3;echo 5 a | 5 | a | 5 a +| +par_sqlandworker_lo $PG | | | | | +par_sqlandworker_lo $PG lo | sleep .3;echo 5 b | 5 | b | 5 b +| +par_sqlandworker_lo $PG | | | | | +par_sqlandworker_lo $PG lo | sleep .3;echo 5 c | 5 | c | 5 c +| +par_sqlandworker_lo $PG | | | | | +par_sqlandworker_lo $PG lo | sleep .3;echo 5 d | 5 | d | 5 d +| +par_sqlandworker_lo $PG | | | | | +par_sqlandworker_lo $PG lo | sleep .3;echo 5 e | 5 | e | 5 e +| +par_sqlandworker_lo $PG | | | | | +par_sqlandworker_lo $PG (25 rows) +par_sqlandworker_lo $PG +par_sqlandworker_lo $SQLITE +par_sqlandworker_lo $SQLITE Exit=0 +par_sqlandworker_lo $SQLITE Exit=0 +par_sqlandworker_lo $SQLITE Host|Command|V1|V2|Stdout|Stderr +par_sqlandworker_lo $SQLITE lo|sleep .3;echo 1 a|1|a|1 a +par_sqlandworker_lo $SQLITE | +par_sqlandworker_lo $SQLITE lo|sleep .3;echo 1 b|1|b|1 b +par_sqlandworker_lo $SQLITE | +par_sqlandworker_lo $SQLITE lo|sleep .3;echo 1 c|1|c|1 c +par_sqlandworker_lo $SQLITE | +par_sqlandworker_lo $SQLITE lo|sleep .3;echo 1 d|1|d|1 d +par_sqlandworker_lo $SQLITE | +par_sqlandworker_lo $SQLITE lo|sleep .3;echo 1 e|1|e|1 e +par_sqlandworker_lo $SQLITE | +par_sqlandworker_lo $SQLITE lo|sleep .3;echo 2 a|2|a|2 a +par_sqlandworker_lo $SQLITE | +par_sqlandworker_lo $SQLITE lo|sleep .3;echo 2 b|2|b|2 b +par_sqlandworker_lo $SQLITE | +par_sqlandworker_lo $SQLITE lo|sleep .3;echo 2 c|2|c|2 c +par_sqlandworker_lo $SQLITE | +par_sqlandworker_lo $SQLITE lo|sleep .3;echo 2 d|2|d|2 d +par_sqlandworker_lo $SQLITE | +par_sqlandworker_lo $SQLITE lo|sleep .3;echo 2 e|2|e|2 e +par_sqlandworker_lo $SQLITE | +par_sqlandworker_lo $SQLITE lo|sleep .3;echo 3 a|3|a|3 a +par_sqlandworker_lo $SQLITE | +par_sqlandworker_lo $SQLITE lo|sleep .3;echo 3 b|3|b|3 b +par_sqlandworker_lo $SQLITE | +par_sqlandworker_lo $SQLITE lo|sleep .3;echo 3 c|3|c|3 c +par_sqlandworker_lo $SQLITE | +par_sqlandworker_lo $SQLITE lo|sleep .3;echo 3 d|3|d|3 d +par_sqlandworker_lo $SQLITE | +par_sqlandworker_lo $SQLITE lo|sleep .3;echo 3 e|3|e|3 e +par_sqlandworker_lo $SQLITE | +par_sqlandworker_lo $SQLITE lo|sleep .3;echo 4 a|4|a|4 a +par_sqlandworker_lo $SQLITE | +par_sqlandworker_lo $SQLITE lo|sleep .3;echo 4 b|4|b|4 b +par_sqlandworker_lo $SQLITE | +par_sqlandworker_lo $SQLITE lo|sleep .3;echo 4 c|4|c|4 c +par_sqlandworker_lo $SQLITE | +par_sqlandworker_lo $SQLITE lo|sleep .3;echo 4 d|4|d|4 d +par_sqlandworker_lo $SQLITE | +par_sqlandworker_lo $SQLITE lo|sleep .3;echo 4 e|4|e|4 e +par_sqlandworker_lo $SQLITE | +par_sqlandworker_lo $SQLITE lo|sleep .3;echo 5 a|5|a|5 a +par_sqlandworker_lo $SQLITE | +par_sqlandworker_lo $SQLITE lo|sleep .3;echo 5 b|5|b|5 b +par_sqlandworker_lo $SQLITE | +par_sqlandworker_lo $SQLITE lo|sleep .3;echo 5 c|5|c|5 c +par_sqlandworker_lo $SQLITE | +par_sqlandworker_lo $SQLITE lo|sleep .3;echo 5 d|5|d|5 d +par_sqlandworker_lo $SQLITE | +par_sqlandworker_lo $SQLITE lo|sleep .3;echo 5 e|5|e|5 e +par_sqlandworker_lo $SQLITE | +par_sqlandworker_results $MYSQL +par_sqlandworker_results $MYSQL Exit=0 +par_sqlandworker_results $MYSQL Exit=0 +par_sqlandworker_results $MYSQL Host Command V1 V2 Stdout Stderr +par_sqlandworker_results $MYSQL : sleep .3;echo 1 a 1 a /tmp/out--sql/1/1/2/a/stdout /tmp/out--sql/1/1/2/a/stderr +par_sqlandworker_results $MYSQL : sleep .3;echo 1 b 1 b /tmp/out--sql/1/1/2/b/stdout /tmp/out--sql/1/1/2/b/stderr +par_sqlandworker_results $MYSQL : sleep .3;echo 1 c 1 c /tmp/out--sql/1/1/2/c/stdout /tmp/out--sql/1/1/2/c/stderr +par_sqlandworker_results $MYSQL : sleep .3;echo 1 d 1 d /tmp/out--sql/1/1/2/d/stdout /tmp/out--sql/1/1/2/d/stderr +par_sqlandworker_results $MYSQL : sleep .3;echo 1 e 1 e /tmp/out--sql/1/1/2/e/stdout /tmp/out--sql/1/1/2/e/stderr +par_sqlandworker_results $MYSQL : sleep .3;echo 2 a 2 a /tmp/out--sql/1/2/2/a/stdout /tmp/out--sql/1/2/2/a/stderr +par_sqlandworker_results $MYSQL : sleep .3;echo 2 b 2 b /tmp/out--sql/1/2/2/b/stdout /tmp/out--sql/1/2/2/b/stderr +par_sqlandworker_results $MYSQL : sleep .3;echo 2 c 2 c /tmp/out--sql/1/2/2/c/stdout /tmp/out--sql/1/2/2/c/stderr +par_sqlandworker_results $MYSQL : sleep .3;echo 2 d 2 d /tmp/out--sql/1/2/2/d/stdout /tmp/out--sql/1/2/2/d/stderr +par_sqlandworker_results $MYSQL : sleep .3;echo 2 e 2 e /tmp/out--sql/1/2/2/e/stdout /tmp/out--sql/1/2/2/e/stderr +par_sqlandworker_results $MYSQL : sleep .3;echo 3 a 3 a /tmp/out--sql/1/3/2/a/stdout /tmp/out--sql/1/3/2/a/stderr +par_sqlandworker_results $MYSQL : sleep .3;echo 3 b 3 b /tmp/out--sql/1/3/2/b/stdout /tmp/out--sql/1/3/2/b/stderr +par_sqlandworker_results $MYSQL : sleep .3;echo 3 c 3 c /tmp/out--sql/1/3/2/c/stdout /tmp/out--sql/1/3/2/c/stderr +par_sqlandworker_results $MYSQL : sleep .3;echo 3 d 3 d /tmp/out--sql/1/3/2/d/stdout /tmp/out--sql/1/3/2/d/stderr +par_sqlandworker_results $MYSQL : sleep .3;echo 3 e 3 e /tmp/out--sql/1/3/2/e/stdout /tmp/out--sql/1/3/2/e/stderr +par_sqlandworker_results $MYSQL : sleep .3;echo 4 a 4 a /tmp/out--sql/1/4/2/a/stdout /tmp/out--sql/1/4/2/a/stderr +par_sqlandworker_results $MYSQL : sleep .3;echo 4 b 4 b /tmp/out--sql/1/4/2/b/stdout /tmp/out--sql/1/4/2/b/stderr +par_sqlandworker_results $MYSQL : sleep .3;echo 4 c 4 c /tmp/out--sql/1/4/2/c/stdout /tmp/out--sql/1/4/2/c/stderr +par_sqlandworker_results $MYSQL : sleep .3;echo 4 d 4 d /tmp/out--sql/1/4/2/d/stdout /tmp/out--sql/1/4/2/d/stderr +par_sqlandworker_results $MYSQL : sleep .3;echo 4 e 4 e /tmp/out--sql/1/4/2/e/stdout /tmp/out--sql/1/4/2/e/stderr +par_sqlandworker_results $MYSQL : sleep .3;echo 5 a 5 a /tmp/out--sql/1/5/2/a/stdout /tmp/out--sql/1/5/2/a/stderr +par_sqlandworker_results $MYSQL : sleep .3;echo 5 b 5 b /tmp/out--sql/1/5/2/b/stdout /tmp/out--sql/1/5/2/b/stderr +par_sqlandworker_results $MYSQL : sleep .3;echo 5 c 5 c /tmp/out--sql/1/5/2/c/stdout /tmp/out--sql/1/5/2/c/stderr +par_sqlandworker_results $MYSQL : sleep .3;echo 5 d 5 d /tmp/out--sql/1/5/2/d/stdout /tmp/out--sql/1/5/2/d/stderr +par_sqlandworker_results $MYSQL : sleep .3;echo 5 e 5 e /tmp/out--sql/1/5/2/e/stdout /tmp/out--sql/1/5/2/e/stderr +par_sqlandworker_results $PG +par_sqlandworker_results $PG Exit=0 +par_sqlandworker_results $PG Exit=0 +par_sqlandworker_results $PG host | command | v1 | v2 | stdout | stderr +par_sqlandworker_results $PG ------+-------------------+----+----+------------------------------+------------------------------ +par_sqlandworker_results $PG : | sleep .3;echo 1 a | 1 | a | /tmp/out--sql/1/1/2/a/stdout | /tmp/out--sql/1/1/2/a/stderr +par_sqlandworker_results $PG : | sleep .3;echo 1 b | 1 | b | /tmp/out--sql/1/1/2/b/stdout | /tmp/out--sql/1/1/2/b/stderr +par_sqlandworker_results $PG : | sleep .3;echo 1 c | 1 | c | /tmp/out--sql/1/1/2/c/stdout | /tmp/out--sql/1/1/2/c/stderr +par_sqlandworker_results $PG : | sleep .3;echo 1 d | 1 | d | /tmp/out--sql/1/1/2/d/stdout | /tmp/out--sql/1/1/2/d/stderr +par_sqlandworker_results $PG : | sleep .3;echo 1 e | 1 | e | /tmp/out--sql/1/1/2/e/stdout | /tmp/out--sql/1/1/2/e/stderr +par_sqlandworker_results $PG : | sleep .3;echo 2 a | 2 | a | /tmp/out--sql/1/2/2/a/stdout | /tmp/out--sql/1/2/2/a/stderr +par_sqlandworker_results $PG : | sleep .3;echo 2 b | 2 | b | /tmp/out--sql/1/2/2/b/stdout | /tmp/out--sql/1/2/2/b/stderr +par_sqlandworker_results $PG : | sleep .3;echo 2 c | 2 | c | /tmp/out--sql/1/2/2/c/stdout | /tmp/out--sql/1/2/2/c/stderr +par_sqlandworker_results $PG : | sleep .3;echo 2 d | 2 | d | /tmp/out--sql/1/2/2/d/stdout | /tmp/out--sql/1/2/2/d/stderr +par_sqlandworker_results $PG : | sleep .3;echo 2 e | 2 | e | /tmp/out--sql/1/2/2/e/stdout | /tmp/out--sql/1/2/2/e/stderr +par_sqlandworker_results $PG : | sleep .3;echo 3 a | 3 | a | /tmp/out--sql/1/3/2/a/stdout | /tmp/out--sql/1/3/2/a/stderr +par_sqlandworker_results $PG : | sleep .3;echo 3 b | 3 | b | /tmp/out--sql/1/3/2/b/stdout | /tmp/out--sql/1/3/2/b/stderr +par_sqlandworker_results $PG : | sleep .3;echo 3 c | 3 | c | /tmp/out--sql/1/3/2/c/stdout | /tmp/out--sql/1/3/2/c/stderr +par_sqlandworker_results $PG : | sleep .3;echo 3 d | 3 | d | /tmp/out--sql/1/3/2/d/stdout | /tmp/out--sql/1/3/2/d/stderr +par_sqlandworker_results $PG : | sleep .3;echo 3 e | 3 | e | /tmp/out--sql/1/3/2/e/stdout | /tmp/out--sql/1/3/2/e/stderr +par_sqlandworker_results $PG : | sleep .3;echo 4 a | 4 | a | /tmp/out--sql/1/4/2/a/stdout | /tmp/out--sql/1/4/2/a/stderr +par_sqlandworker_results $PG : | sleep .3;echo 4 b | 4 | b | /tmp/out--sql/1/4/2/b/stdout | /tmp/out--sql/1/4/2/b/stderr +par_sqlandworker_results $PG : | sleep .3;echo 4 c | 4 | c | /tmp/out--sql/1/4/2/c/stdout | /tmp/out--sql/1/4/2/c/stderr +par_sqlandworker_results $PG : | sleep .3;echo 4 d | 4 | d | /tmp/out--sql/1/4/2/d/stdout | /tmp/out--sql/1/4/2/d/stderr +par_sqlandworker_results $PG : | sleep .3;echo 4 e | 4 | e | /tmp/out--sql/1/4/2/e/stdout | /tmp/out--sql/1/4/2/e/stderr +par_sqlandworker_results $PG : | sleep .3;echo 5 a | 5 | a | /tmp/out--sql/1/5/2/a/stdout | /tmp/out--sql/1/5/2/a/stderr +par_sqlandworker_results $PG : | sleep .3;echo 5 b | 5 | b | /tmp/out--sql/1/5/2/b/stdout | /tmp/out--sql/1/5/2/b/stderr +par_sqlandworker_results $PG : | sleep .3;echo 5 c | 5 | c | /tmp/out--sql/1/5/2/c/stdout | /tmp/out--sql/1/5/2/c/stderr +par_sqlandworker_results $PG : | sleep .3;echo 5 d | 5 | d | /tmp/out--sql/1/5/2/d/stdout | /tmp/out--sql/1/5/2/d/stderr +par_sqlandworker_results $PG : | sleep .3;echo 5 e | 5 | e | /tmp/out--sql/1/5/2/e/stdout | /tmp/out--sql/1/5/2/e/stderr +par_sqlandworker_results $PG (25 rows) +par_sqlandworker_results $PG +par_sqlandworker_results $SQLITE +par_sqlandworker_results $SQLITE Exit=0 +par_sqlandworker_results $SQLITE Exit=0 +par_sqlandworker_results $SQLITE Host|Command|V1|V2|Stdout|Stderr +par_sqlandworker_results $SQLITE :|sleep .3;echo 1 a|1|a|/tmp/out--sql/1/1/2/a/stdout|/tmp/out--sql/1/1/2/a/stderr +par_sqlandworker_results $SQLITE :|sleep .3;echo 1 b|1|b|/tmp/out--sql/1/1/2/b/stdout|/tmp/out--sql/1/1/2/b/stderr +par_sqlandworker_results $SQLITE :|sleep .3;echo 1 c|1|c|/tmp/out--sql/1/1/2/c/stdout|/tmp/out--sql/1/1/2/c/stderr +par_sqlandworker_results $SQLITE :|sleep .3;echo 1 d|1|d|/tmp/out--sql/1/1/2/d/stdout|/tmp/out--sql/1/1/2/d/stderr +par_sqlandworker_results $SQLITE :|sleep .3;echo 1 e|1|e|/tmp/out--sql/1/1/2/e/stdout|/tmp/out--sql/1/1/2/e/stderr +par_sqlandworker_results $SQLITE :|sleep .3;echo 2 a|2|a|/tmp/out--sql/1/2/2/a/stdout|/tmp/out--sql/1/2/2/a/stderr +par_sqlandworker_results $SQLITE :|sleep .3;echo 2 b|2|b|/tmp/out--sql/1/2/2/b/stdout|/tmp/out--sql/1/2/2/b/stderr +par_sqlandworker_results $SQLITE :|sleep .3;echo 2 c|2|c|/tmp/out--sql/1/2/2/c/stdout|/tmp/out--sql/1/2/2/c/stderr +par_sqlandworker_results $SQLITE :|sleep .3;echo 2 d|2|d|/tmp/out--sql/1/2/2/d/stdout|/tmp/out--sql/1/2/2/d/stderr +par_sqlandworker_results $SQLITE :|sleep .3;echo 2 e|2|e|/tmp/out--sql/1/2/2/e/stdout|/tmp/out--sql/1/2/2/e/stderr +par_sqlandworker_results $SQLITE :|sleep .3;echo 3 a|3|a|/tmp/out--sql/1/3/2/a/stdout|/tmp/out--sql/1/3/2/a/stderr +par_sqlandworker_results $SQLITE :|sleep .3;echo 3 b|3|b|/tmp/out--sql/1/3/2/b/stdout|/tmp/out--sql/1/3/2/b/stderr +par_sqlandworker_results $SQLITE :|sleep .3;echo 3 c|3|c|/tmp/out--sql/1/3/2/c/stdout|/tmp/out--sql/1/3/2/c/stderr +par_sqlandworker_results $SQLITE :|sleep .3;echo 3 d|3|d|/tmp/out--sql/1/3/2/d/stdout|/tmp/out--sql/1/3/2/d/stderr +par_sqlandworker_results $SQLITE :|sleep .3;echo 3 e|3|e|/tmp/out--sql/1/3/2/e/stdout|/tmp/out--sql/1/3/2/e/stderr +par_sqlandworker_results $SQLITE :|sleep .3;echo 4 a|4|a|/tmp/out--sql/1/4/2/a/stdout|/tmp/out--sql/1/4/2/a/stderr +par_sqlandworker_results $SQLITE :|sleep .3;echo 4 b|4|b|/tmp/out--sql/1/4/2/b/stdout|/tmp/out--sql/1/4/2/b/stderr +par_sqlandworker_results $SQLITE :|sleep .3;echo 4 c|4|c|/tmp/out--sql/1/4/2/c/stdout|/tmp/out--sql/1/4/2/c/stderr +par_sqlandworker_results $SQLITE :|sleep .3;echo 4 d|4|d|/tmp/out--sql/1/4/2/d/stdout|/tmp/out--sql/1/4/2/d/stderr +par_sqlandworker_results $SQLITE :|sleep .3;echo 4 e|4|e|/tmp/out--sql/1/4/2/e/stdout|/tmp/out--sql/1/4/2/e/stderr +par_sqlandworker_results $SQLITE :|sleep .3;echo 5 a|5|a|/tmp/out--sql/1/5/2/a/stdout|/tmp/out--sql/1/5/2/a/stderr +par_sqlandworker_results $SQLITE :|sleep .3;echo 5 b|5|b|/tmp/out--sql/1/5/2/b/stdout|/tmp/out--sql/1/5/2/b/stderr +par_sqlandworker_results $SQLITE :|sleep .3;echo 5 c|5|c|/tmp/out--sql/1/5/2/c/stdout|/tmp/out--sql/1/5/2/c/stderr +par_sqlandworker_results $SQLITE :|sleep .3;echo 5 d|5|d|/tmp/out--sql/1/5/2/d/stdout|/tmp/out--sql/1/5/2/d/stderr +par_sqlandworker_results $SQLITE :|sleep .3;echo 5 e|5|e|/tmp/out--sql/1/5/2/e/stdout|/tmp/out--sql/1/5/2/e/stderr +par_sqlandworker_unbuffer $MYSQL +par_sqlandworker_unbuffer $MYSQL Exit=0 +par_sqlandworker_unbuffer $MYSQL Exit=0 +par_sqlandworker_unbuffer $MYSQL Host Command V1 V2 Stdout Stderr +par_sqlandworker_unbuffer $MYSQL : sleep .3;echo 1 a 1 a NULL NULL +par_sqlandworker_unbuffer $MYSQL : sleep .3;echo 1 b 1 b NULL NULL +par_sqlandworker_unbuffer $MYSQL : sleep .3;echo 1 c 1 c NULL NULL +par_sqlandworker_unbuffer $MYSQL : sleep .3;echo 1 d 1 d NULL NULL +par_sqlandworker_unbuffer $MYSQL : sleep .3;echo 1 e 1 e NULL NULL +par_sqlandworker_unbuffer $MYSQL : sleep .3;echo 2 a 2 a NULL NULL +par_sqlandworker_unbuffer $MYSQL : sleep .3;echo 2 b 2 b NULL NULL +par_sqlandworker_unbuffer $MYSQL : sleep .3;echo 2 c 2 c NULL NULL +par_sqlandworker_unbuffer $MYSQL : sleep .3;echo 2 d 2 d NULL NULL +par_sqlandworker_unbuffer $MYSQL : sleep .3;echo 2 e 2 e NULL NULL +par_sqlandworker_unbuffer $MYSQL : sleep .3;echo 3 a 3 a NULL NULL +par_sqlandworker_unbuffer $MYSQL : sleep .3;echo 3 b 3 b NULL NULL +par_sqlandworker_unbuffer $MYSQL : sleep .3;echo 3 c 3 c NULL NULL +par_sqlandworker_unbuffer $MYSQL : sleep .3;echo 3 d 3 d NULL NULL +par_sqlandworker_unbuffer $MYSQL : sleep .3;echo 3 e 3 e NULL NULL +par_sqlandworker_unbuffer $MYSQL : sleep .3;echo 4 a 4 a NULL NULL +par_sqlandworker_unbuffer $MYSQL : sleep .3;echo 4 b 4 b NULL NULL +par_sqlandworker_unbuffer $MYSQL : sleep .3;echo 4 c 4 c NULL NULL +par_sqlandworker_unbuffer $MYSQL : sleep .3;echo 4 d 4 d NULL NULL +par_sqlandworker_unbuffer $MYSQL : sleep .3;echo 4 e 4 e NULL NULL +par_sqlandworker_unbuffer $MYSQL : sleep .3;echo 5 a 5 a NULL NULL +par_sqlandworker_unbuffer $MYSQL : sleep .3;echo 5 b 5 b NULL NULL +par_sqlandworker_unbuffer $MYSQL : sleep .3;echo 5 c 5 c NULL NULL +par_sqlandworker_unbuffer $MYSQL : sleep .3;echo 5 d 5 d NULL NULL +par_sqlandworker_unbuffer $MYSQL : sleep .3;echo 5 e 5 e NULL NULL +par_sqlandworker_unbuffer $PG +par_sqlandworker_unbuffer $PG Exit=0 +par_sqlandworker_unbuffer $PG Exit=0 +par_sqlandworker_unbuffer $PG host | command | v1 | v2 | stdout | stderr +par_sqlandworker_unbuffer $PG ------+-------------------+----+----+--------+-------- +par_sqlandworker_unbuffer $PG : | sleep .3;echo 1 a | 1 | a | | +par_sqlandworker_unbuffer $PG : | sleep .3;echo 1 b | 1 | b | | +par_sqlandworker_unbuffer $PG : | sleep .3;echo 1 c | 1 | c | | +par_sqlandworker_unbuffer $PG : | sleep .3;echo 1 d | 1 | d | | +par_sqlandworker_unbuffer $PG : | sleep .3;echo 1 e | 1 | e | | +par_sqlandworker_unbuffer $PG : | sleep .3;echo 2 a | 2 | a | | +par_sqlandworker_unbuffer $PG : | sleep .3;echo 2 b | 2 | b | | +par_sqlandworker_unbuffer $PG : | sleep .3;echo 2 c | 2 | c | | +par_sqlandworker_unbuffer $PG : | sleep .3;echo 2 d | 2 | d | | +par_sqlandworker_unbuffer $PG : | sleep .3;echo 2 e | 2 | e | | +par_sqlandworker_unbuffer $PG : | sleep .3;echo 3 a | 3 | a | | +par_sqlandworker_unbuffer $PG : | sleep .3;echo 3 b | 3 | b | | +par_sqlandworker_unbuffer $PG : | sleep .3;echo 3 c | 3 | c | | +par_sqlandworker_unbuffer $PG : | sleep .3;echo 3 d | 3 | d | | +par_sqlandworker_unbuffer $PG : | sleep .3;echo 3 e | 3 | e | | +par_sqlandworker_unbuffer $PG : | sleep .3;echo 4 a | 4 | a | | +par_sqlandworker_unbuffer $PG : | sleep .3;echo 4 b | 4 | b | | +par_sqlandworker_unbuffer $PG : | sleep .3;echo 4 c | 4 | c | | +par_sqlandworker_unbuffer $PG : | sleep .3;echo 4 d | 4 | d | | +par_sqlandworker_unbuffer $PG : | sleep .3;echo 4 e | 4 | e | | +par_sqlandworker_unbuffer $PG : | sleep .3;echo 5 a | 5 | a | | +par_sqlandworker_unbuffer $PG : | sleep .3;echo 5 b | 5 | b | | +par_sqlandworker_unbuffer $PG : | sleep .3;echo 5 c | 5 | c | | +par_sqlandworker_unbuffer $PG : | sleep .3;echo 5 d | 5 | d | | +par_sqlandworker_unbuffer $PG : | sleep .3;echo 5 e | 5 | e | | +par_sqlandworker_unbuffer $PG (25 rows) +par_sqlandworker_unbuffer $PG +par_sqlandworker_unbuffer $SQLITE +par_sqlandworker_unbuffer $SQLITE Exit=0 +par_sqlandworker_unbuffer $SQLITE Exit=0 +par_sqlandworker_unbuffer $SQLITE Host|Command|V1|V2|Stdout|Stderr +par_sqlandworker_unbuffer $SQLITE :|sleep .3;echo 1 a|1|a|| +par_sqlandworker_unbuffer $SQLITE :|sleep .3;echo 1 b|1|b|| +par_sqlandworker_unbuffer $SQLITE :|sleep .3;echo 1 c|1|c|| +par_sqlandworker_unbuffer $SQLITE :|sleep .3;echo 1 d|1|d|| +par_sqlandworker_unbuffer $SQLITE :|sleep .3;echo 1 e|1|e|| +par_sqlandworker_unbuffer $SQLITE :|sleep .3;echo 2 a|2|a|| +par_sqlandworker_unbuffer $SQLITE :|sleep .3;echo 2 b|2|b|| +par_sqlandworker_unbuffer $SQLITE :|sleep .3;echo 2 c|2|c|| +par_sqlandworker_unbuffer $SQLITE :|sleep .3;echo 2 d|2|d|| +par_sqlandworker_unbuffer $SQLITE :|sleep .3;echo 2 e|2|e|| +par_sqlandworker_unbuffer $SQLITE :|sleep .3;echo 3 a|3|a|| +par_sqlandworker_unbuffer $SQLITE :|sleep .3;echo 3 b|3|b|| +par_sqlandworker_unbuffer $SQLITE :|sleep .3;echo 3 c|3|c|| +par_sqlandworker_unbuffer $SQLITE :|sleep .3;echo 3 d|3|d|| +par_sqlandworker_unbuffer $SQLITE :|sleep .3;echo 3 e|3|e|| +par_sqlandworker_unbuffer $SQLITE :|sleep .3;echo 4 a|4|a|| +par_sqlandworker_unbuffer $SQLITE :|sleep .3;echo 4 b|4|b|| +par_sqlandworker_unbuffer $SQLITE :|sleep .3;echo 4 c|4|c|| +par_sqlandworker_unbuffer $SQLITE :|sleep .3;echo 4 d|4|d|| +par_sqlandworker_unbuffer $SQLITE :|sleep .3;echo 4 e|4|e|| +par_sqlandworker_unbuffer $SQLITE :|sleep .3;echo 5 a|5|a|| +par_sqlandworker_unbuffer $SQLITE :|sleep .3;echo 5 b|5|b|| +par_sqlandworker_unbuffer $SQLITE :|sleep .3;echo 5 c|5|c|| +par_sqlandworker_unbuffer $SQLITE :|sleep .3;echo 5 d|5|d|| +par_sqlandworker_unbuffer $SQLITE :|sleep .3;echo 5 e|5|e|| From 8a5729cebd1b5db9c3ff8539badabd81ee27a16e Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Wed, 6 Jul 2016 19:04:52 +0200 Subject: [PATCH 04/12] Fixed bug #48414: --sqlandworker --linebuffer does not save output in sql. --- src/parallel | 73 ++- testsuite/Start.sh | 0 testsuite/tests-to-run/parallel-local-sql.sh | 52 +- testsuite/wanted-results/parallel-local-sql | 600 ++++++++++++++++--- 4 files changed, 595 insertions(+), 130 deletions(-) mode change 100644 => 100755 testsuite/Start.sh diff --git a/src/parallel b/src/parallel index 8e1ac7ee..c55da1b0 100755 --- a/src/parallel +++ b/src/parallel @@ -1208,7 +1208,7 @@ sub check_invalid_option_combinations { sub init_globals { # Defaults: - $Global::version = 20160627; + $Global::version = 20160704; $Global::progname = 'parallel'; $Global::infinity = 2**31; $Global::debug = 0; @@ -8000,7 +8000,6 @@ sub linebuffer_print { } if($opt::compress) { # Blocked reading in final round - $Global::use{"Fcntl"} ||= eval "use Fcntl qw(:DEFAULT :flock); 1;"; for my $fdno (1,2) { ::set_fh_blocking($self->fh($fdno,'r')); } @@ -8051,7 +8050,28 @@ sub linebuffer_print { if($self->fh($fdno,"rpid") and CORE::kill 0, $self->fh($fdno,"rpid")) { # decompress still running } else { - # decompress done: close fh + # decompress done: + # copy to sql (if needed) + # then close fh + if($opt::sqlworker) { + my @output; + my $tag = $self->tag(); + seek $in_fh, 0, 0; + while(<$in_fh>) { + push @output, $tag,$_; + } + if($fdno == 1) { + if(not $opt::results) { + $Global::sql->update("SET Stdout = ? WHERE Seq = ".$self->seq(), + join("",@output)); + } + } else { + if(not $opt::results) { + $Global::sql->update("SET Stderr = ? WHERE Seq = ".$self->seq(), + join("",@output)); + } + } + } close $in_fh; if($? and $opt::compress) { ::error($opt::decompress_program." failed."); @@ -8075,12 +8095,25 @@ sub tag_print { # $in_fh is now ready for reading at position 0 my $tag = $self->tag(); my $outputlength = 0; + my @output; while(<$in_fh>) { print $out_fd $tag,$_; $outputlength += length $_; + if($opt::sqlworker) { + push @output, $tag,$_; + } } if($fdno == 1) { $self->add_returnsize($outputlength); + if($opt::sqlworker and not $opt::results) { + $Global::sql->update("SET Stdout = ? WHERE Seq = ".$self->seq(), + join("",@output)); + } + } else { + if($opt::sqlworker and not $opt::results) { + $Global::sql->update("SET Stderr = ? WHERE Seq = ".$self->seq(), + join("",@output)); + } } close $in_fh; if($? and $opt::compress) { @@ -8149,13 +8182,13 @@ sub print_joblog { sub tag { my $self = shift; - if($opt::tag or defined $opt::tagstring) { - if(not defined $self->{'tag'}) { + if(not defined $self->{'tag'}) { + if($opt::tag or defined $opt::tagstring) { $self->{'tag'} = $self->{'commandline'}-> replace_placeholders([$opt::tagstring],0,0)."\t"; + } else { + $self->{'tag'} = ""; } - } else { - return ""; } return $self->{'tag'}; } @@ -9906,7 +9939,10 @@ sub new { or die $DBI::errstr; $dbh->{'PrintWarn'} = $Global::debug || 0; $dbh->{'PrintError'} = $Global::debug || 0; - $dbh->{'RaiseError'} = $Global::debug || 0; + $dbh->{'RaiseError'} = 1; + $dbh->{'ShowErrorStatement'} = 1; + $dbh->{'HandleError'} = sub {}; + return bless { 'dbh' => $dbh, @@ -10083,21 +10119,36 @@ sub run { my $lockretry = 0; while($lockretry < 10) { $sth = $dbh->prepare($stmt); - if($rv = $sth->execute(@_)) { + if($sth + and + eval { $rv = $sth->execute(@_) }) { last; } else { + if($@ =~ /no such table/ + or + $DBI::errstr =~ /no such table/) { + # This is fine: + # It is just a worker that reported back too late - + # another worker had finished the job first + # and the table was then dropped + $rv = $sth = 0; + last; + } if($DBI::errstr =~ /locked/) { ::debug("sql","Lock retry: $lockretry"); $lockretry++; ::usleep(rand()*300); + } elsif(not $sth) { + # Try again + $lockretry++; } else { ::error($DBI::errstr); - wait_and_exit(255); + ::wait_and_exit(255); } } } if($lockretry >= 10) { - ::die_bug("lockretry > 10"); + ::die_bug("retry > 10: $DBI::errstr"); } if($rv < 0){ ::error($DBI::errstr); diff --git a/testsuite/Start.sh b/testsuite/Start.sh old mode 100644 new mode 100755 diff --git a/testsuite/tests-to-run/parallel-local-sql.sh b/testsuite/tests-to-run/parallel-local-sql.sh index 12f4da79..8042d724 100755 --- a/testsuite/tests-to-run/parallel-local-sql.sh +++ b/testsuite/tests-to-run/parallel-local-sql.sh @@ -14,7 +14,7 @@ p_showsqlresult() { p_wrapper() { INNER=$1 - SERVERURL=$2 + SERVERURL=$(eval echo $2) TABLE=TBL$RANDOM DBURL=$SERVERURL/$TABLE T1=$(tempfile) @@ -29,53 +29,47 @@ p_wrapper() { $DEBUG || sql $SERVERURL "drop table $TABLE;" >/dev/null } -p_sqlandworker() { - (sleep 2; parallel --sqlworker $DBURL sleep .3\;echo >$T1) & - parallel --sqlandworker $DBURL sleep .3\;echo ::: {1..5} ::: {a..e} >$T2; +p_template() { + (sleep 2; parallel "$@" --sqlworker $DBURL sleep .3\;echo >$T1) & + parallel "$@" --sqlandworker $DBURL sleep .3\;echo ::: {1..5} ::: {a..e} >$T2; } -export -f p_sqlandworker +export -f p_template par_sqlandworker() { - p_wrapper p_sqlandworker $1 -} - -p_sqlandworker_lo() { - (sleep 2; parallel -S lo --sqlworker $DBURL sleep .3\;echo >$T1) & - parallel -S lo --sqlandworker $DBURL sleep .3\;echo ::: {1..5} ::: {a..e} >$T2; + p_template } par_sqlandworker_lo() { - p_wrapper p_sqlandworker_lo $1 -} - -p_sqlandworker_results() { - (sleep 2; parallel --results /tmp/out--sql --sqlworker $DBURL sleep .3\;echo >$T1) & - parallel --results /tmp/out--sql --sqlandworker $DBURL sleep .3\;echo ::: {1..5} ::: {a..e} >$T2; + p_template -S lo } par_sqlandworker_results() { - p_wrapper p_sqlandworker_results $1 -} - -p_sqlandworker_linebuffer() { - (sleep 2; parallel --linebuffer --sqlworker $DBURL sleep .3\;echo >$T1) & - parallel --linebuffer --sqlandworker $DBURL sleep .3\;echo ::: {1..5} ::: {a..e} >$T2; + p_template --results /tmp/out--sql } par_sqlandworker_linebuffer() { - p_wrapper p_sqlandworker_linebuffer $1 + p_template --linebuffer } -p_sqlandworker_unbuffer() { - (sleep 2; parallel -u --sqlworker $DBURL sleep .3\;echo >$T1) & - parallel -u --sqlandworker $DBURL sleep .3\;echo ::: {1..5} ::: {a..e} >$T2; +par_sqlandworker_tag() { + p_template --tag +} + +par_sqlandworker_linebuffer_tag() { + p_template --linebuffer --tag +} + +par_sqlandworker_compress_linebuffer_tag() { + p_template --compress --linebuffer --tag } par_sqlandworker_unbuffer() { - p_wrapper p_sqlandworker_unbuffer $1 + p_template -u } + export -f $(compgen -A function | egrep 'p_|par_') # Tested that -j0 in parallel is fastest (up to 15 jobs) compgen -A function | grep par_ | sort | - stdout parallel -vj0 -k --tag --joblog /tmp/jl-`basename $0` :::: - ::: \$MYSQL \$PG \$SQLITE + stdout parallel -vj5 -k --tag --joblog /tmp/jl-`basename $0` p_wrapper \ + :::: - ::: \$MYSQL \$PG \$SQLITE diff --git a/testsuite/wanted-results/parallel-local-sql b/testsuite/wanted-results/parallel-local-sql index 7f9aa6a6..b26ce5ce 100644 --- a/testsuite/wanted-results/parallel-local-sql +++ b/testsuite/wanted-results/parallel-local-sql @@ -1,4 +1,4 @@ -par_sqlandworker $MYSQL +p_wrapper par_sqlandworker \$MYSQL par_sqlandworker $MYSQL Exit=0 par_sqlandworker $MYSQL Exit=0 par_sqlandworker $MYSQL Host Command V1 V2 Stdout Stderr @@ -27,7 +27,7 @@ par_sqlandworker $MYSQL : sleep .3;echo 5 b 5 b 5 b\n par_sqlandworker $MYSQL : sleep .3;echo 5 c 5 c 5 c\n par_sqlandworker $MYSQL : sleep .3;echo 5 d 5 d 5 d\n par_sqlandworker $MYSQL : sleep .3;echo 5 e 5 e 5 e\n -par_sqlandworker $PG +p_wrapper par_sqlandworker \$PG par_sqlandworker $PG Exit=0 par_sqlandworker $PG Exit=0 par_sqlandworker $PG host | command | v1 | v2 | stdout | stderr @@ -84,7 +84,7 @@ par_sqlandworker $PG : | sleep .3;echo 5 e | 5 | e | 5 e +| par_sqlandworker $PG | | | | | par_sqlandworker $PG (25 rows) par_sqlandworker $PG -par_sqlandworker $SQLITE +p_wrapper par_sqlandworker \$SQLITE par_sqlandworker $SQLITE Exit=0 par_sqlandworker $SQLITE Exit=0 par_sqlandworker $SQLITE Host|Command|V1|V2|Stdout|Stderr @@ -138,97 +138,377 @@ par_sqlandworker $SQLITE :|sleep .3;echo 5 d|5|d|5 d par_sqlandworker $SQLITE | par_sqlandworker $SQLITE :|sleep .3;echo 5 e|5|e|5 e par_sqlandworker $SQLITE | -par_sqlandworker_linebuffer $MYSQL +p_wrapper par_sqlandworker_compress_linebuffer_tag \$MYSQL +par_sqlandworker_compress_linebuffer_tag $MYSQL Exit=0 +par_sqlandworker_compress_linebuffer_tag $MYSQL Exit=0 +par_sqlandworker_compress_linebuffer_tag $MYSQL Host Command V1 V2 Stdout Stderr +par_sqlandworker_compress_linebuffer_tag $MYSQL : sleep .3;echo 1 a 1 a NULL NULL +par_sqlandworker_compress_linebuffer_tag $MYSQL : sleep .3;echo 1 b 1 b NULL NULL +par_sqlandworker_compress_linebuffer_tag $MYSQL : sleep .3;echo 1 c 1 c NULL NULL +par_sqlandworker_compress_linebuffer_tag $MYSQL : sleep .3;echo 1 d 1 d NULL NULL +par_sqlandworker_compress_linebuffer_tag $MYSQL : sleep .3;echo 1 e 1 e NULL NULL +par_sqlandworker_compress_linebuffer_tag $MYSQL : sleep .3;echo 2 a 2 a NULL NULL +par_sqlandworker_compress_linebuffer_tag $MYSQL : sleep .3;echo 2 b 2 b NULL NULL +par_sqlandworker_compress_linebuffer_tag $MYSQL : sleep .3;echo 2 c 2 c NULL NULL +par_sqlandworker_compress_linebuffer_tag $MYSQL : sleep .3;echo 2 d 2 d NULL NULL +par_sqlandworker_compress_linebuffer_tag $MYSQL : sleep .3;echo 2 e 2 e NULL NULL +par_sqlandworker_compress_linebuffer_tag $MYSQL : sleep .3;echo 3 a 3 a NULL NULL +par_sqlandworker_compress_linebuffer_tag $MYSQL : sleep .3;echo 3 b 3 b NULL NULL +par_sqlandworker_compress_linebuffer_tag $MYSQL : sleep .3;echo 3 c 3 c NULL NULL +par_sqlandworker_compress_linebuffer_tag $MYSQL : sleep .3;echo 3 d 3 d NULL NULL +par_sqlandworker_compress_linebuffer_tag $MYSQL : sleep .3;echo 3 e 3 e NULL NULL +par_sqlandworker_compress_linebuffer_tag $MYSQL : sleep .3;echo 4 a 4 a NULL NULL +par_sqlandworker_compress_linebuffer_tag $MYSQL : sleep .3;echo 4 b 4 b NULL NULL +par_sqlandworker_compress_linebuffer_tag $MYSQL : sleep .3;echo 4 c 4 c NULL NULL +par_sqlandworker_compress_linebuffer_tag $MYSQL : sleep .3;echo 4 d 4 d NULL NULL +par_sqlandworker_compress_linebuffer_tag $MYSQL : sleep .3;echo 4 e 4 e NULL NULL +par_sqlandworker_compress_linebuffer_tag $MYSQL : sleep .3;echo 5 a 5 a NULL NULL +par_sqlandworker_compress_linebuffer_tag $MYSQL : sleep .3;echo 5 b 5 b NULL NULL +par_sqlandworker_compress_linebuffer_tag $MYSQL : sleep .3;echo 5 c 5 c NULL NULL +par_sqlandworker_compress_linebuffer_tag $MYSQL : sleep .3;echo 5 d 5 d NULL NULL +par_sqlandworker_compress_linebuffer_tag $MYSQL : sleep .3;echo 5 e 5 e NULL NULL +p_wrapper par_sqlandworker_compress_linebuffer_tag \$PG +par_sqlandworker_compress_linebuffer_tag $PG Exit=0 +par_sqlandworker_compress_linebuffer_tag $PG Exit=0 +par_sqlandworker_compress_linebuffer_tag $PG host | command | v1 | v2 | stdout | stderr +par_sqlandworker_compress_linebuffer_tag $PG ------+-------------------+----+----+--------+-------- +par_sqlandworker_compress_linebuffer_tag $PG : | sleep .3;echo 1 a | 1 | a | | +par_sqlandworker_compress_linebuffer_tag $PG : | sleep .3;echo 1 b | 1 | b | | +par_sqlandworker_compress_linebuffer_tag $PG : | sleep .3;echo 1 c | 1 | c | | +par_sqlandworker_compress_linebuffer_tag $PG : | sleep .3;echo 1 d | 1 | d | | +par_sqlandworker_compress_linebuffer_tag $PG : | sleep .3;echo 1 e | 1 | e | | +par_sqlandworker_compress_linebuffer_tag $PG : | sleep .3;echo 2 a | 2 | a | | +par_sqlandworker_compress_linebuffer_tag $PG : | sleep .3;echo 2 b | 2 | b | | +par_sqlandworker_compress_linebuffer_tag $PG : | sleep .3;echo 2 c | 2 | c | | +par_sqlandworker_compress_linebuffer_tag $PG : | sleep .3;echo 2 d | 2 | d | | +par_sqlandworker_compress_linebuffer_tag $PG : | sleep .3;echo 2 e | 2 | e | | +par_sqlandworker_compress_linebuffer_tag $PG : | sleep .3;echo 3 a | 3 | a | | +par_sqlandworker_compress_linebuffer_tag $PG : | sleep .3;echo 3 b | 3 | b | | +par_sqlandworker_compress_linebuffer_tag $PG : | sleep .3;echo 3 c | 3 | c | | +par_sqlandworker_compress_linebuffer_tag $PG : | sleep .3;echo 3 d | 3 | d | | +par_sqlandworker_compress_linebuffer_tag $PG : | sleep .3;echo 3 e | 3 | e | | +par_sqlandworker_compress_linebuffer_tag $PG : | sleep .3;echo 4 a | 4 | a | | +par_sqlandworker_compress_linebuffer_tag $PG : | sleep .3;echo 4 b | 4 | b | | +par_sqlandworker_compress_linebuffer_tag $PG : | sleep .3;echo 4 c | 4 | c | | +par_sqlandworker_compress_linebuffer_tag $PG : | sleep .3;echo 4 d | 4 | d | | +par_sqlandworker_compress_linebuffer_tag $PG : | sleep .3;echo 4 e | 4 | e | | +par_sqlandworker_compress_linebuffer_tag $PG : | sleep .3;echo 5 a | 5 | a | | +par_sqlandworker_compress_linebuffer_tag $PG : | sleep .3;echo 5 b | 5 | b | | +par_sqlandworker_compress_linebuffer_tag $PG : | sleep .3;echo 5 c | 5 | c | | +par_sqlandworker_compress_linebuffer_tag $PG : | sleep .3;echo 5 d | 5 | d | | +par_sqlandworker_compress_linebuffer_tag $PG : | sleep .3;echo 5 e | 5 | e | | +par_sqlandworker_compress_linebuffer_tag $PG (25 rows) +par_sqlandworker_compress_linebuffer_tag $PG +p_wrapper par_sqlandworker_compress_linebuffer_tag \$SQLITE +par_sqlandworker_compress_linebuffer_tag $SQLITE Exit=0 +par_sqlandworker_compress_linebuffer_tag $SQLITE Exit=0 +par_sqlandworker_compress_linebuffer_tag $SQLITE Host|Command|V1|V2|Stdout|Stderr +par_sqlandworker_compress_linebuffer_tag $SQLITE :|sleep .3;echo 1 a|1|a|| +par_sqlandworker_compress_linebuffer_tag $SQLITE :|sleep .3;echo 1 b|1|b|| +par_sqlandworker_compress_linebuffer_tag $SQLITE :|sleep .3;echo 1 c|1|c|| +par_sqlandworker_compress_linebuffer_tag $SQLITE :|sleep .3;echo 1 d|1|d|| +par_sqlandworker_compress_linebuffer_tag $SQLITE :|sleep .3;echo 1 e|1|e|| +par_sqlandworker_compress_linebuffer_tag $SQLITE :|sleep .3;echo 2 a|2|a|| +par_sqlandworker_compress_linebuffer_tag $SQLITE :|sleep .3;echo 2 b|2|b|| +par_sqlandworker_compress_linebuffer_tag $SQLITE :|sleep .3;echo 2 c|2|c|| +par_sqlandworker_compress_linebuffer_tag $SQLITE :|sleep .3;echo 2 d|2|d|| +par_sqlandworker_compress_linebuffer_tag $SQLITE :|sleep .3;echo 2 e|2|e|| +par_sqlandworker_compress_linebuffer_tag $SQLITE :|sleep .3;echo 3 a|3|a|| +par_sqlandworker_compress_linebuffer_tag $SQLITE :|sleep .3;echo 3 b|3|b|| +par_sqlandworker_compress_linebuffer_tag $SQLITE :|sleep .3;echo 3 c|3|c|| +par_sqlandworker_compress_linebuffer_tag $SQLITE :|sleep .3;echo 3 d|3|d|| +par_sqlandworker_compress_linebuffer_tag $SQLITE :|sleep .3;echo 3 e|3|e|| +par_sqlandworker_compress_linebuffer_tag $SQLITE :|sleep .3;echo 4 a|4|a|| +par_sqlandworker_compress_linebuffer_tag $SQLITE :|sleep .3;echo 4 b|4|b|| +par_sqlandworker_compress_linebuffer_tag $SQLITE :|sleep .3;echo 4 c|4|c|| +par_sqlandworker_compress_linebuffer_tag $SQLITE :|sleep .3;echo 4 d|4|d|| +par_sqlandworker_compress_linebuffer_tag $SQLITE :|sleep .3;echo 4 e|4|e|| +par_sqlandworker_compress_linebuffer_tag $SQLITE :|sleep .3;echo 5 a|5|a|| +par_sqlandworker_compress_linebuffer_tag $SQLITE :|sleep .3;echo 5 b|5|b|| +par_sqlandworker_compress_linebuffer_tag $SQLITE :|sleep .3;echo 5 c|5|c|| +par_sqlandworker_compress_linebuffer_tag $SQLITE :|sleep .3;echo 5 d|5|d|| +par_sqlandworker_compress_linebuffer_tag $SQLITE :|sleep .3;echo 5 e|5|e|| +p_wrapper par_sqlandworker_linebuffer \$MYSQL par_sqlandworker_linebuffer $MYSQL Exit=0 par_sqlandworker_linebuffer $MYSQL Exit=0 par_sqlandworker_linebuffer $MYSQL Host Command V1 V2 Stdout Stderr -par_sqlandworker_linebuffer $MYSQL : sleep .3;echo 1 a 1 a NULL NULL -par_sqlandworker_linebuffer $MYSQL : sleep .3;echo 1 b 1 b NULL NULL -par_sqlandworker_linebuffer $MYSQL : sleep .3;echo 1 c 1 c NULL NULL -par_sqlandworker_linebuffer $MYSQL : sleep .3;echo 1 d 1 d NULL NULL -par_sqlandworker_linebuffer $MYSQL : sleep .3;echo 1 e 1 e NULL NULL -par_sqlandworker_linebuffer $MYSQL : sleep .3;echo 2 a 2 a NULL NULL -par_sqlandworker_linebuffer $MYSQL : sleep .3;echo 2 b 2 b NULL NULL -par_sqlandworker_linebuffer $MYSQL : sleep .3;echo 2 c 2 c NULL NULL -par_sqlandworker_linebuffer $MYSQL : sleep .3;echo 2 d 2 d NULL NULL -par_sqlandworker_linebuffer $MYSQL : sleep .3;echo 2 e 2 e NULL NULL -par_sqlandworker_linebuffer $MYSQL : sleep .3;echo 3 a 3 a NULL NULL -par_sqlandworker_linebuffer $MYSQL : sleep .3;echo 3 b 3 b NULL NULL -par_sqlandworker_linebuffer $MYSQL : sleep .3;echo 3 c 3 c NULL NULL -par_sqlandworker_linebuffer $MYSQL : sleep .3;echo 3 d 3 d NULL NULL -par_sqlandworker_linebuffer $MYSQL : sleep .3;echo 3 e 3 e NULL NULL -par_sqlandworker_linebuffer $MYSQL : sleep .3;echo 4 a 4 a NULL NULL -par_sqlandworker_linebuffer $MYSQL : sleep .3;echo 4 b 4 b NULL NULL -par_sqlandworker_linebuffer $MYSQL : sleep .3;echo 4 c 4 c NULL NULL -par_sqlandworker_linebuffer $MYSQL : sleep .3;echo 4 d 4 d NULL NULL -par_sqlandworker_linebuffer $MYSQL : sleep .3;echo 4 e 4 e NULL NULL -par_sqlandworker_linebuffer $MYSQL : sleep .3;echo 5 a 5 a NULL NULL -par_sqlandworker_linebuffer $MYSQL : sleep .3;echo 5 b 5 b NULL NULL -par_sqlandworker_linebuffer $MYSQL : sleep .3;echo 5 c 5 c NULL NULL -par_sqlandworker_linebuffer $MYSQL : sleep .3;echo 5 d 5 d NULL NULL -par_sqlandworker_linebuffer $MYSQL : sleep .3;echo 5 e 5 e NULL NULL -par_sqlandworker_linebuffer $PG +par_sqlandworker_linebuffer $MYSQL : sleep .3;echo 1 a 1 a 1 a\n +par_sqlandworker_linebuffer $MYSQL : sleep .3;echo 1 b 1 b 1 b\n +par_sqlandworker_linebuffer $MYSQL : sleep .3;echo 1 c 1 c 1 c\n +par_sqlandworker_linebuffer $MYSQL : sleep .3;echo 1 d 1 d 1 d\n +par_sqlandworker_linebuffer $MYSQL : sleep .3;echo 1 e 1 e 1 e\n +par_sqlandworker_linebuffer $MYSQL : sleep .3;echo 2 a 2 a 2 a\n +par_sqlandworker_linebuffer $MYSQL : sleep .3;echo 2 b 2 b 2 b\n +par_sqlandworker_linebuffer $MYSQL : sleep .3;echo 2 c 2 c 2 c\n +par_sqlandworker_linebuffer $MYSQL : sleep .3;echo 2 d 2 d 2 d\n +par_sqlandworker_linebuffer $MYSQL : sleep .3;echo 2 e 2 e 2 e\n +par_sqlandworker_linebuffer $MYSQL : sleep .3;echo 3 a 3 a 3 a\n +par_sqlandworker_linebuffer $MYSQL : sleep .3;echo 3 b 3 b 3 b\n +par_sqlandworker_linebuffer $MYSQL : sleep .3;echo 3 c 3 c 3 c\n +par_sqlandworker_linebuffer $MYSQL : sleep .3;echo 3 d 3 d 3 d\n +par_sqlandworker_linebuffer $MYSQL : sleep .3;echo 3 e 3 e 3 e\n +par_sqlandworker_linebuffer $MYSQL : sleep .3;echo 4 a 4 a 4 a\n +par_sqlandworker_linebuffer $MYSQL : sleep .3;echo 4 b 4 b 4 b\n +par_sqlandworker_linebuffer $MYSQL : sleep .3;echo 4 c 4 c 4 c\n +par_sqlandworker_linebuffer $MYSQL : sleep .3;echo 4 d 4 d 4 d\n +par_sqlandworker_linebuffer $MYSQL : sleep .3;echo 4 e 4 e 4 e\n +par_sqlandworker_linebuffer $MYSQL : sleep .3;echo 5 a 5 a 5 a\n +par_sqlandworker_linebuffer $MYSQL : sleep .3;echo 5 b 5 b 5 b\n +par_sqlandworker_linebuffer $MYSQL : sleep .3;echo 5 c 5 c 5 c\n +par_sqlandworker_linebuffer $MYSQL : sleep .3;echo 5 d 5 d 5 d\n +par_sqlandworker_linebuffer $MYSQL : sleep .3;echo 5 e 5 e 5 e\n +p_wrapper par_sqlandworker_linebuffer \$PG par_sqlandworker_linebuffer $PG Exit=0 par_sqlandworker_linebuffer $PG Exit=0 par_sqlandworker_linebuffer $PG host | command | v1 | v2 | stdout | stderr par_sqlandworker_linebuffer $PG ------+-------------------+----+----+--------+-------- -par_sqlandworker_linebuffer $PG : | sleep .3;echo 1 a | 1 | a | | -par_sqlandworker_linebuffer $PG : | sleep .3;echo 1 b | 1 | b | | -par_sqlandworker_linebuffer $PG : | sleep .3;echo 1 c | 1 | c | | -par_sqlandworker_linebuffer $PG : | sleep .3;echo 1 d | 1 | d | | -par_sqlandworker_linebuffer $PG : | sleep .3;echo 1 e | 1 | e | | -par_sqlandworker_linebuffer $PG : | sleep .3;echo 2 a | 2 | a | | -par_sqlandworker_linebuffer $PG : | sleep .3;echo 2 b | 2 | b | | -par_sqlandworker_linebuffer $PG : | sleep .3;echo 2 c | 2 | c | | -par_sqlandworker_linebuffer $PG : | sleep .3;echo 2 d | 2 | d | | -par_sqlandworker_linebuffer $PG : | sleep .3;echo 2 e | 2 | e | | -par_sqlandworker_linebuffer $PG : | sleep .3;echo 3 a | 3 | a | | -par_sqlandworker_linebuffer $PG : | sleep .3;echo 3 b | 3 | b | | -par_sqlandworker_linebuffer $PG : | sleep .3;echo 3 c | 3 | c | | -par_sqlandworker_linebuffer $PG : | sleep .3;echo 3 d | 3 | d | | -par_sqlandworker_linebuffer $PG : | sleep .3;echo 3 e | 3 | e | | -par_sqlandworker_linebuffer $PG : | sleep .3;echo 4 a | 4 | a | | -par_sqlandworker_linebuffer $PG : | sleep .3;echo 4 b | 4 | b | | -par_sqlandworker_linebuffer $PG : | sleep .3;echo 4 c | 4 | c | | -par_sqlandworker_linebuffer $PG : | sleep .3;echo 4 d | 4 | d | | -par_sqlandworker_linebuffer $PG : | sleep .3;echo 4 e | 4 | e | | -par_sqlandworker_linebuffer $PG : | sleep .3;echo 5 a | 5 | a | | -par_sqlandworker_linebuffer $PG : | sleep .3;echo 5 b | 5 | b | | -par_sqlandworker_linebuffer $PG : | sleep .3;echo 5 c | 5 | c | | -par_sqlandworker_linebuffer $PG : | sleep .3;echo 5 d | 5 | d | | -par_sqlandworker_linebuffer $PG : | sleep .3;echo 5 e | 5 | e | | +par_sqlandworker_linebuffer $PG : | sleep .3;echo 1 a | 1 | a | 1 a +| +par_sqlandworker_linebuffer $PG | | | | | +par_sqlandworker_linebuffer $PG : | sleep .3;echo 1 b | 1 | b | 1 b +| +par_sqlandworker_linebuffer $PG | | | | | +par_sqlandworker_linebuffer $PG : | sleep .3;echo 1 c | 1 | c | 1 c +| +par_sqlandworker_linebuffer $PG | | | | | +par_sqlandworker_linebuffer $PG : | sleep .3;echo 1 d | 1 | d | 1 d +| +par_sqlandworker_linebuffer $PG | | | | | +par_sqlandworker_linebuffer $PG : | sleep .3;echo 1 e | 1 | e | 1 e +| +par_sqlandworker_linebuffer $PG | | | | | +par_sqlandworker_linebuffer $PG : | sleep .3;echo 2 a | 2 | a | 2 a +| +par_sqlandworker_linebuffer $PG | | | | | +par_sqlandworker_linebuffer $PG : | sleep .3;echo 2 b | 2 | b | 2 b +| +par_sqlandworker_linebuffer $PG | | | | | +par_sqlandworker_linebuffer $PG : | sleep .3;echo 2 c | 2 | c | 2 c +| +par_sqlandworker_linebuffer $PG | | | | | +par_sqlandworker_linebuffer $PG : | sleep .3;echo 2 d | 2 | d | 2 d +| +par_sqlandworker_linebuffer $PG | | | | | +par_sqlandworker_linebuffer $PG : | sleep .3;echo 2 e | 2 | e | 2 e +| +par_sqlandworker_linebuffer $PG | | | | | +par_sqlandworker_linebuffer $PG : | sleep .3;echo 3 a | 3 | a | 3 a +| +par_sqlandworker_linebuffer $PG | | | | | +par_sqlandworker_linebuffer $PG : | sleep .3;echo 3 b | 3 | b | 3 b +| +par_sqlandworker_linebuffer $PG | | | | | +par_sqlandworker_linebuffer $PG : | sleep .3;echo 3 c | 3 | c | 3 c +| +par_sqlandworker_linebuffer $PG | | | | | +par_sqlandworker_linebuffer $PG : | sleep .3;echo 3 d | 3 | d | 3 d +| +par_sqlandworker_linebuffer $PG | | | | | +par_sqlandworker_linebuffer $PG : | sleep .3;echo 3 e | 3 | e | 3 e +| +par_sqlandworker_linebuffer $PG | | | | | +par_sqlandworker_linebuffer $PG : | sleep .3;echo 4 a | 4 | a | 4 a +| +par_sqlandworker_linebuffer $PG | | | | | +par_sqlandworker_linebuffer $PG : | sleep .3;echo 4 b | 4 | b | 4 b +| +par_sqlandworker_linebuffer $PG | | | | | +par_sqlandworker_linebuffer $PG : | sleep .3;echo 4 c | 4 | c | 4 c +| +par_sqlandworker_linebuffer $PG | | | | | +par_sqlandworker_linebuffer $PG : | sleep .3;echo 4 d | 4 | d | 4 d +| +par_sqlandworker_linebuffer $PG | | | | | +par_sqlandworker_linebuffer $PG : | sleep .3;echo 4 e | 4 | e | 4 e +| +par_sqlandworker_linebuffer $PG | | | | | +par_sqlandworker_linebuffer $PG : | sleep .3;echo 5 a | 5 | a | 5 a +| +par_sqlandworker_linebuffer $PG | | | | | +par_sqlandworker_linebuffer $PG : | sleep .3;echo 5 b | 5 | b | 5 b +| +par_sqlandworker_linebuffer $PG | | | | | +par_sqlandworker_linebuffer $PG : | sleep .3;echo 5 c | 5 | c | 5 c +| +par_sqlandworker_linebuffer $PG | | | | | +par_sqlandworker_linebuffer $PG : | sleep .3;echo 5 d | 5 | d | 5 d +| +par_sqlandworker_linebuffer $PG | | | | | +par_sqlandworker_linebuffer $PG : | sleep .3;echo 5 e | 5 | e | 5 e +| +par_sqlandworker_linebuffer $PG | | | | | par_sqlandworker_linebuffer $PG (25 rows) par_sqlandworker_linebuffer $PG -par_sqlandworker_linebuffer $SQLITE +p_wrapper par_sqlandworker_linebuffer \$SQLITE par_sqlandworker_linebuffer $SQLITE Exit=0 par_sqlandworker_linebuffer $SQLITE Exit=0 par_sqlandworker_linebuffer $SQLITE Host|Command|V1|V2|Stdout|Stderr -par_sqlandworker_linebuffer $SQLITE :|sleep .3;echo 1 a|1|a|| -par_sqlandworker_linebuffer $SQLITE :|sleep .3;echo 1 b|1|b|| -par_sqlandworker_linebuffer $SQLITE :|sleep .3;echo 1 c|1|c|| -par_sqlandworker_linebuffer $SQLITE :|sleep .3;echo 1 d|1|d|| -par_sqlandworker_linebuffer $SQLITE :|sleep .3;echo 1 e|1|e|| -par_sqlandworker_linebuffer $SQLITE :|sleep .3;echo 2 a|2|a|| -par_sqlandworker_linebuffer $SQLITE :|sleep .3;echo 2 b|2|b|| -par_sqlandworker_linebuffer $SQLITE :|sleep .3;echo 2 c|2|c|| -par_sqlandworker_linebuffer $SQLITE :|sleep .3;echo 2 d|2|d|| -par_sqlandworker_linebuffer $SQLITE :|sleep .3;echo 2 e|2|e|| -par_sqlandworker_linebuffer $SQLITE :|sleep .3;echo 3 a|3|a|| -par_sqlandworker_linebuffer $SQLITE :|sleep .3;echo 3 b|3|b|| -par_sqlandworker_linebuffer $SQLITE :|sleep .3;echo 3 c|3|c|| -par_sqlandworker_linebuffer $SQLITE :|sleep .3;echo 3 d|3|d|| -par_sqlandworker_linebuffer $SQLITE :|sleep .3;echo 3 e|3|e|| -par_sqlandworker_linebuffer $SQLITE :|sleep .3;echo 4 a|4|a|| -par_sqlandworker_linebuffer $SQLITE :|sleep .3;echo 4 b|4|b|| -par_sqlandworker_linebuffer $SQLITE :|sleep .3;echo 4 c|4|c|| -par_sqlandworker_linebuffer $SQLITE :|sleep .3;echo 4 d|4|d|| -par_sqlandworker_linebuffer $SQLITE :|sleep .3;echo 4 e|4|e|| -par_sqlandworker_linebuffer $SQLITE :|sleep .3;echo 5 a|5|a|| -par_sqlandworker_linebuffer $SQLITE :|sleep .3;echo 5 b|5|b|| -par_sqlandworker_linebuffer $SQLITE :|sleep .3;echo 5 c|5|c|| -par_sqlandworker_linebuffer $SQLITE :|sleep .3;echo 5 d|5|d|| -par_sqlandworker_linebuffer $SQLITE :|sleep .3;echo 5 e|5|e|| -par_sqlandworker_lo $MYSQL +par_sqlandworker_linebuffer $SQLITE :|sleep .3;echo 1 a|1|a|1 a +par_sqlandworker_linebuffer $SQLITE | +par_sqlandworker_linebuffer $SQLITE :|sleep .3;echo 1 b|1|b|1 b +par_sqlandworker_linebuffer $SQLITE | +par_sqlandworker_linebuffer $SQLITE :|sleep .3;echo 1 c|1|c|1 c +par_sqlandworker_linebuffer $SQLITE | +par_sqlandworker_linebuffer $SQLITE :|sleep .3;echo 1 d|1|d|1 d +par_sqlandworker_linebuffer $SQLITE | +par_sqlandworker_linebuffer $SQLITE :|sleep .3;echo 1 e|1|e|1 e +par_sqlandworker_linebuffer $SQLITE | +par_sqlandworker_linebuffer $SQLITE :|sleep .3;echo 2 a|2|a|2 a +par_sqlandworker_linebuffer $SQLITE | +par_sqlandworker_linebuffer $SQLITE :|sleep .3;echo 2 b|2|b|2 b +par_sqlandworker_linebuffer $SQLITE | +par_sqlandworker_linebuffer $SQLITE :|sleep .3;echo 2 c|2|c|2 c +par_sqlandworker_linebuffer $SQLITE | +par_sqlandworker_linebuffer $SQLITE :|sleep .3;echo 2 d|2|d|2 d +par_sqlandworker_linebuffer $SQLITE | +par_sqlandworker_linebuffer $SQLITE :|sleep .3;echo 2 e|2|e|2 e +par_sqlandworker_linebuffer $SQLITE | +par_sqlandworker_linebuffer $SQLITE :|sleep .3;echo 3 a|3|a|3 a +par_sqlandworker_linebuffer $SQLITE | +par_sqlandworker_linebuffer $SQLITE :|sleep .3;echo 3 b|3|b|3 b +par_sqlandworker_linebuffer $SQLITE | +par_sqlandworker_linebuffer $SQLITE :|sleep .3;echo 3 c|3|c|3 c +par_sqlandworker_linebuffer $SQLITE | +par_sqlandworker_linebuffer $SQLITE :|sleep .3;echo 3 d|3|d|3 d +par_sqlandworker_linebuffer $SQLITE | +par_sqlandworker_linebuffer $SQLITE :|sleep .3;echo 3 e|3|e|3 e +par_sqlandworker_linebuffer $SQLITE | +par_sqlandworker_linebuffer $SQLITE :|sleep .3;echo 4 a|4|a|4 a +par_sqlandworker_linebuffer $SQLITE | +par_sqlandworker_linebuffer $SQLITE :|sleep .3;echo 4 b|4|b|4 b +par_sqlandworker_linebuffer $SQLITE | +par_sqlandworker_linebuffer $SQLITE :|sleep .3;echo 4 c|4|c|4 c +par_sqlandworker_linebuffer $SQLITE | +par_sqlandworker_linebuffer $SQLITE :|sleep .3;echo 4 d|4|d|4 d +par_sqlandworker_linebuffer $SQLITE | +par_sqlandworker_linebuffer $SQLITE :|sleep .3;echo 4 e|4|e|4 e +par_sqlandworker_linebuffer $SQLITE | +par_sqlandworker_linebuffer $SQLITE :|sleep .3;echo 5 a|5|a|5 a +par_sqlandworker_linebuffer $SQLITE | +par_sqlandworker_linebuffer $SQLITE :|sleep .3;echo 5 b|5|b|5 b +par_sqlandworker_linebuffer $SQLITE | +par_sqlandworker_linebuffer $SQLITE :|sleep .3;echo 5 c|5|c|5 c +par_sqlandworker_linebuffer $SQLITE | +par_sqlandworker_linebuffer $SQLITE :|sleep .3;echo 5 d|5|d|5 d +par_sqlandworker_linebuffer $SQLITE | +par_sqlandworker_linebuffer $SQLITE :|sleep .3;echo 5 e|5|e|5 e +par_sqlandworker_linebuffer $SQLITE | +p_wrapper par_sqlandworker_linebuffer_tag \$MYSQL +par_sqlandworker_linebuffer_tag $MYSQL Exit=0 +par_sqlandworker_linebuffer_tag $MYSQL Exit=0 +par_sqlandworker_linebuffer_tag $MYSQL Host Command V1 V2 Stdout Stderr +par_sqlandworker_linebuffer_tag $MYSQL : sleep .3;echo 1 a 1 a 1 a\t1 a\n +par_sqlandworker_linebuffer_tag $MYSQL : sleep .3;echo 1 b 1 b 1 b\t1 b\n +par_sqlandworker_linebuffer_tag $MYSQL : sleep .3;echo 1 c 1 c 1 c\t1 c\n +par_sqlandworker_linebuffer_tag $MYSQL : sleep .3;echo 1 d 1 d 1 d\t1 d\n +par_sqlandworker_linebuffer_tag $MYSQL : sleep .3;echo 1 e 1 e 1 e\t1 e\n +par_sqlandworker_linebuffer_tag $MYSQL : sleep .3;echo 2 a 2 a 2 a\t2 a\n +par_sqlandworker_linebuffer_tag $MYSQL : sleep .3;echo 2 b 2 b 2 b\t2 b\n +par_sqlandworker_linebuffer_tag $MYSQL : sleep .3;echo 2 c 2 c 2 c\t2 c\n +par_sqlandworker_linebuffer_tag $MYSQL : sleep .3;echo 2 d 2 d 2 d\t2 d\n +par_sqlandworker_linebuffer_tag $MYSQL : sleep .3;echo 2 e 2 e 2 e\t2 e\n +par_sqlandworker_linebuffer_tag $MYSQL : sleep .3;echo 3 a 3 a 3 a\t3 a\n +par_sqlandworker_linebuffer_tag $MYSQL : sleep .3;echo 3 b 3 b 3 b\t3 b\n +par_sqlandworker_linebuffer_tag $MYSQL : sleep .3;echo 3 c 3 c 3 c\t3 c\n +par_sqlandworker_linebuffer_tag $MYSQL : sleep .3;echo 3 d 3 d 3 d\t3 d\n +par_sqlandworker_linebuffer_tag $MYSQL : sleep .3;echo 3 e 3 e 3 e\t3 e\n +par_sqlandworker_linebuffer_tag $MYSQL : sleep .3;echo 4 a 4 a 4 a\t4 a\n +par_sqlandworker_linebuffer_tag $MYSQL : sleep .3;echo 4 b 4 b 4 b\t4 b\n +par_sqlandworker_linebuffer_tag $MYSQL : sleep .3;echo 4 c 4 c 4 c\t4 c\n +par_sqlandworker_linebuffer_tag $MYSQL : sleep .3;echo 4 d 4 d 4 d\t4 d\n +par_sqlandworker_linebuffer_tag $MYSQL : sleep .3;echo 4 e 4 e 4 e\t4 e\n +par_sqlandworker_linebuffer_tag $MYSQL : sleep .3;echo 5 a 5 a 5 a\t5 a\n +par_sqlandworker_linebuffer_tag $MYSQL : sleep .3;echo 5 b 5 b 5 b\t5 b\n +par_sqlandworker_linebuffer_tag $MYSQL : sleep .3;echo 5 c 5 c 5 c\t5 c\n +par_sqlandworker_linebuffer_tag $MYSQL : sleep .3;echo 5 d 5 d 5 d\t5 d\n +par_sqlandworker_linebuffer_tag $MYSQL : sleep .3;echo 5 e 5 e 5 e\t5 e\n +p_wrapper par_sqlandworker_linebuffer_tag \$PG +par_sqlandworker_linebuffer_tag $PG Exit=0 +par_sqlandworker_linebuffer_tag $PG Exit=0 +par_sqlandworker_linebuffer_tag $PG host | command | v1 | v2 | stdout | stderr +par_sqlandworker_linebuffer_tag $PG ------+-------------------+----+----+-------------+-------- +par_sqlandworker_linebuffer_tag $PG : | sleep .3;echo 1 a | 1 | a | 1 a 1 a+| +par_sqlandworker_linebuffer_tag $PG | | | | | +par_sqlandworker_linebuffer_tag $PG : | sleep .3;echo 1 b | 1 | b | 1 b 1 b+| +par_sqlandworker_linebuffer_tag $PG | | | | | +par_sqlandworker_linebuffer_tag $PG : | sleep .3;echo 1 c | 1 | c | 1 c 1 c+| +par_sqlandworker_linebuffer_tag $PG | | | | | +par_sqlandworker_linebuffer_tag $PG : | sleep .3;echo 1 d | 1 | d | 1 d 1 d+| +par_sqlandworker_linebuffer_tag $PG | | | | | +par_sqlandworker_linebuffer_tag $PG : | sleep .3;echo 1 e | 1 | e | 1 e 1 e+| +par_sqlandworker_linebuffer_tag $PG | | | | | +par_sqlandworker_linebuffer_tag $PG : | sleep .3;echo 2 a | 2 | a | 2 a 2 a+| +par_sqlandworker_linebuffer_tag $PG | | | | | +par_sqlandworker_linebuffer_tag $PG : | sleep .3;echo 2 b | 2 | b | 2 b 2 b+| +par_sqlandworker_linebuffer_tag $PG | | | | | +par_sqlandworker_linebuffer_tag $PG : | sleep .3;echo 2 c | 2 | c | 2 c 2 c+| +par_sqlandworker_linebuffer_tag $PG | | | | | +par_sqlandworker_linebuffer_tag $PG : | sleep .3;echo 2 d | 2 | d | 2 d 2 d+| +par_sqlandworker_linebuffer_tag $PG | | | | | +par_sqlandworker_linebuffer_tag $PG : | sleep .3;echo 2 e | 2 | e | 2 e 2 e+| +par_sqlandworker_linebuffer_tag $PG | | | | | +par_sqlandworker_linebuffer_tag $PG : | sleep .3;echo 3 a | 3 | a | 3 a 3 a+| +par_sqlandworker_linebuffer_tag $PG | | | | | +par_sqlandworker_linebuffer_tag $PG : | sleep .3;echo 3 b | 3 | b | 3 b 3 b+| +par_sqlandworker_linebuffer_tag $PG | | | | | +par_sqlandworker_linebuffer_tag $PG : | sleep .3;echo 3 c | 3 | c | 3 c 3 c+| +par_sqlandworker_linebuffer_tag $PG | | | | | +par_sqlandworker_linebuffer_tag $PG : | sleep .3;echo 3 d | 3 | d | 3 d 3 d+| +par_sqlandworker_linebuffer_tag $PG | | | | | +par_sqlandworker_linebuffer_tag $PG : | sleep .3;echo 3 e | 3 | e | 3 e 3 e+| +par_sqlandworker_linebuffer_tag $PG | | | | | +par_sqlandworker_linebuffer_tag $PG : | sleep .3;echo 4 a | 4 | a | 4 a 4 a+| +par_sqlandworker_linebuffer_tag $PG | | | | | +par_sqlandworker_linebuffer_tag $PG : | sleep .3;echo 4 b | 4 | b | 4 b 4 b+| +par_sqlandworker_linebuffer_tag $PG | | | | | +par_sqlandworker_linebuffer_tag $PG : | sleep .3;echo 4 c | 4 | c | 4 c 4 c+| +par_sqlandworker_linebuffer_tag $PG | | | | | +par_sqlandworker_linebuffer_tag $PG : | sleep .3;echo 4 d | 4 | d | 4 d 4 d+| +par_sqlandworker_linebuffer_tag $PG | | | | | +par_sqlandworker_linebuffer_tag $PG : | sleep .3;echo 4 e | 4 | e | 4 e 4 e+| +par_sqlandworker_linebuffer_tag $PG | | | | | +par_sqlandworker_linebuffer_tag $PG : | sleep .3;echo 5 a | 5 | a | 5 a 5 a+| +par_sqlandworker_linebuffer_tag $PG | | | | | +par_sqlandworker_linebuffer_tag $PG : | sleep .3;echo 5 b | 5 | b | 5 b 5 b+| +par_sqlandworker_linebuffer_tag $PG | | | | | +par_sqlandworker_linebuffer_tag $PG : | sleep .3;echo 5 c | 5 | c | 5 c 5 c+| +par_sqlandworker_linebuffer_tag $PG | | | | | +par_sqlandworker_linebuffer_tag $PG : | sleep .3;echo 5 d | 5 | d | 5 d 5 d+| +par_sqlandworker_linebuffer_tag $PG | | | | | +par_sqlandworker_linebuffer_tag $PG : | sleep .3;echo 5 e | 5 | e | 5 e 5 e+| +par_sqlandworker_linebuffer_tag $PG | | | | | +par_sqlandworker_linebuffer_tag $PG (25 rows) +par_sqlandworker_linebuffer_tag $PG +p_wrapper par_sqlandworker_linebuffer_tag \$SQLITE +par_sqlandworker_linebuffer_tag $SQLITE Exit=0 +par_sqlandworker_linebuffer_tag $SQLITE Exit=0 +par_sqlandworker_linebuffer_tag $SQLITE Host|Command|V1|V2|Stdout|Stderr +par_sqlandworker_linebuffer_tag $SQLITE :|sleep .3;echo 1 a|1|a|1 a 1 a +par_sqlandworker_linebuffer_tag $SQLITE | +par_sqlandworker_linebuffer_tag $SQLITE :|sleep .3;echo 1 b|1|b|1 b 1 b +par_sqlandworker_linebuffer_tag $SQLITE | +par_sqlandworker_linebuffer_tag $SQLITE :|sleep .3;echo 1 c|1|c|1 c 1 c +par_sqlandworker_linebuffer_tag $SQLITE | +par_sqlandworker_linebuffer_tag $SQLITE :|sleep .3;echo 1 d|1|d|1 d 1 d +par_sqlandworker_linebuffer_tag $SQLITE | +par_sqlandworker_linebuffer_tag $SQLITE :|sleep .3;echo 1 e|1|e|1 e 1 e +par_sqlandworker_linebuffer_tag $SQLITE | +par_sqlandworker_linebuffer_tag $SQLITE :|sleep .3;echo 2 a|2|a|2 a 2 a +par_sqlandworker_linebuffer_tag $SQLITE | +par_sqlandworker_linebuffer_tag $SQLITE :|sleep .3;echo 2 b|2|b|2 b 2 b +par_sqlandworker_linebuffer_tag $SQLITE | +par_sqlandworker_linebuffer_tag $SQLITE :|sleep .3;echo 2 c|2|c|2 c 2 c +par_sqlandworker_linebuffer_tag $SQLITE | +par_sqlandworker_linebuffer_tag $SQLITE :|sleep .3;echo 2 d|2|d|2 d 2 d +par_sqlandworker_linebuffer_tag $SQLITE | +par_sqlandworker_linebuffer_tag $SQLITE :|sleep .3;echo 2 e|2|e|2 e 2 e +par_sqlandworker_linebuffer_tag $SQLITE | +par_sqlandworker_linebuffer_tag $SQLITE :|sleep .3;echo 3 a|3|a|3 a 3 a +par_sqlandworker_linebuffer_tag $SQLITE | +par_sqlandworker_linebuffer_tag $SQLITE :|sleep .3;echo 3 b|3|b|3 b 3 b +par_sqlandworker_linebuffer_tag $SQLITE | +par_sqlandworker_linebuffer_tag $SQLITE :|sleep .3;echo 3 c|3|c|3 c 3 c +par_sqlandworker_linebuffer_tag $SQLITE | +par_sqlandworker_linebuffer_tag $SQLITE :|sleep .3;echo 3 d|3|d|3 d 3 d +par_sqlandworker_linebuffer_tag $SQLITE | +par_sqlandworker_linebuffer_tag $SQLITE :|sleep .3;echo 3 e|3|e|3 e 3 e +par_sqlandworker_linebuffer_tag $SQLITE | +par_sqlandworker_linebuffer_tag $SQLITE :|sleep .3;echo 4 a|4|a|4 a 4 a +par_sqlandworker_linebuffer_tag $SQLITE | +par_sqlandworker_linebuffer_tag $SQLITE :|sleep .3;echo 4 b|4|b|4 b 4 b +par_sqlandworker_linebuffer_tag $SQLITE | +par_sqlandworker_linebuffer_tag $SQLITE :|sleep .3;echo 4 c|4|c|4 c 4 c +par_sqlandworker_linebuffer_tag $SQLITE | +par_sqlandworker_linebuffer_tag $SQLITE :|sleep .3;echo 4 d|4|d|4 d 4 d +par_sqlandworker_linebuffer_tag $SQLITE | +par_sqlandworker_linebuffer_tag $SQLITE :|sleep .3;echo 4 e|4|e|4 e 4 e +par_sqlandworker_linebuffer_tag $SQLITE | +par_sqlandworker_linebuffer_tag $SQLITE :|sleep .3;echo 5 a|5|a|5 a 5 a +par_sqlandworker_linebuffer_tag $SQLITE | +par_sqlandworker_linebuffer_tag $SQLITE :|sleep .3;echo 5 b|5|b|5 b 5 b +par_sqlandworker_linebuffer_tag $SQLITE | +par_sqlandworker_linebuffer_tag $SQLITE :|sleep .3;echo 5 c|5|c|5 c 5 c +par_sqlandworker_linebuffer_tag $SQLITE | +par_sqlandworker_linebuffer_tag $SQLITE :|sleep .3;echo 5 d|5|d|5 d 5 d +par_sqlandworker_linebuffer_tag $SQLITE | +par_sqlandworker_linebuffer_tag $SQLITE :|sleep .3;echo 5 e|5|e|5 e 5 e +par_sqlandworker_linebuffer_tag $SQLITE | +p_wrapper par_sqlandworker_lo \$MYSQL par_sqlandworker_lo $MYSQL Exit=0 par_sqlandworker_lo $MYSQL Exit=0 par_sqlandworker_lo $MYSQL Host Command V1 V2 Stdout Stderr @@ -257,7 +537,7 @@ par_sqlandworker_lo $MYSQL lo sleep .3;echo 5 b 5 b 5 b\n par_sqlandworker_lo $MYSQL lo sleep .3;echo 5 c 5 c 5 c\n par_sqlandworker_lo $MYSQL lo sleep .3;echo 5 d 5 d 5 d\n par_sqlandworker_lo $MYSQL lo sleep .3;echo 5 e 5 e 5 e\n -par_sqlandworker_lo $PG +p_wrapper par_sqlandworker_lo \$PG par_sqlandworker_lo $PG Exit=0 par_sqlandworker_lo $PG Exit=0 par_sqlandworker_lo $PG host | command | v1 | v2 | stdout | stderr @@ -314,7 +594,7 @@ par_sqlandworker_lo $PG lo | sleep .3;echo 5 e | 5 | e | 5 e +| par_sqlandworker_lo $PG | | | | | par_sqlandworker_lo $PG (25 rows) par_sqlandworker_lo $PG -par_sqlandworker_lo $SQLITE +p_wrapper par_sqlandworker_lo \$SQLITE par_sqlandworker_lo $SQLITE Exit=0 par_sqlandworker_lo $SQLITE Exit=0 par_sqlandworker_lo $SQLITE Host|Command|V1|V2|Stdout|Stderr @@ -368,7 +648,7 @@ par_sqlandworker_lo $SQLITE lo|sleep .3;echo 5 d|5|d|5 d par_sqlandworker_lo $SQLITE | par_sqlandworker_lo $SQLITE lo|sleep .3;echo 5 e|5|e|5 e par_sqlandworker_lo $SQLITE | -par_sqlandworker_results $MYSQL +p_wrapper par_sqlandworker_results \$MYSQL par_sqlandworker_results $MYSQL Exit=0 par_sqlandworker_results $MYSQL Exit=0 par_sqlandworker_results $MYSQL Host Command V1 V2 Stdout Stderr @@ -397,7 +677,7 @@ par_sqlandworker_results $MYSQL : sleep .3;echo 5 b 5 b /tmp/out--sql/1/5/2/b/st par_sqlandworker_results $MYSQL : sleep .3;echo 5 c 5 c /tmp/out--sql/1/5/2/c/stdout /tmp/out--sql/1/5/2/c/stderr par_sqlandworker_results $MYSQL : sleep .3;echo 5 d 5 d /tmp/out--sql/1/5/2/d/stdout /tmp/out--sql/1/5/2/d/stderr par_sqlandworker_results $MYSQL : sleep .3;echo 5 e 5 e /tmp/out--sql/1/5/2/e/stdout /tmp/out--sql/1/5/2/e/stderr -par_sqlandworker_results $PG +p_wrapper par_sqlandworker_results \$PG par_sqlandworker_results $PG Exit=0 par_sqlandworker_results $PG Exit=0 par_sqlandworker_results $PG host | command | v1 | v2 | stdout | stderr @@ -429,7 +709,7 @@ par_sqlandworker_results $PG : | sleep .3;echo 5 d | 5 | d | /tmp/out--sql par_sqlandworker_results $PG : | sleep .3;echo 5 e | 5 | e | /tmp/out--sql/1/5/2/e/stdout | /tmp/out--sql/1/5/2/e/stderr par_sqlandworker_results $PG (25 rows) par_sqlandworker_results $PG -par_sqlandworker_results $SQLITE +p_wrapper par_sqlandworker_results \$SQLITE par_sqlandworker_results $SQLITE Exit=0 par_sqlandworker_results $SQLITE Exit=0 par_sqlandworker_results $SQLITE Host|Command|V1|V2|Stdout|Stderr @@ -458,7 +738,147 @@ par_sqlandworker_results $SQLITE :|sleep .3;echo 5 b|5|b|/tmp/out--sql/1/5/2/b/s par_sqlandworker_results $SQLITE :|sleep .3;echo 5 c|5|c|/tmp/out--sql/1/5/2/c/stdout|/tmp/out--sql/1/5/2/c/stderr par_sqlandworker_results $SQLITE :|sleep .3;echo 5 d|5|d|/tmp/out--sql/1/5/2/d/stdout|/tmp/out--sql/1/5/2/d/stderr par_sqlandworker_results $SQLITE :|sleep .3;echo 5 e|5|e|/tmp/out--sql/1/5/2/e/stdout|/tmp/out--sql/1/5/2/e/stderr -par_sqlandworker_unbuffer $MYSQL +p_wrapper par_sqlandworker_tag \$MYSQL +par_sqlandworker_tag $MYSQL Exit=0 +par_sqlandworker_tag $MYSQL Exit=0 +par_sqlandworker_tag $MYSQL Host Command V1 V2 Stdout Stderr +par_sqlandworker_tag $MYSQL : sleep .3;echo 1 a 1 a 1 a\t1 a\n +par_sqlandworker_tag $MYSQL : sleep .3;echo 1 b 1 b 1 b\t1 b\n +par_sqlandworker_tag $MYSQL : sleep .3;echo 1 c 1 c 1 c\t1 c\n +par_sqlandworker_tag $MYSQL : sleep .3;echo 1 d 1 d 1 d\t1 d\n +par_sqlandworker_tag $MYSQL : sleep .3;echo 1 e 1 e 1 e\t1 e\n +par_sqlandworker_tag $MYSQL : sleep .3;echo 2 a 2 a 2 a\t2 a\n +par_sqlandworker_tag $MYSQL : sleep .3;echo 2 b 2 b 2 b\t2 b\n +par_sqlandworker_tag $MYSQL : sleep .3;echo 2 c 2 c 2 c\t2 c\n +par_sqlandworker_tag $MYSQL : sleep .3;echo 2 d 2 d 2 d\t2 d\n +par_sqlandworker_tag $MYSQL : sleep .3;echo 2 e 2 e 2 e\t2 e\n +par_sqlandworker_tag $MYSQL : sleep .3;echo 3 a 3 a 3 a\t3 a\n +par_sqlandworker_tag $MYSQL : sleep .3;echo 3 b 3 b 3 b\t3 b\n +par_sqlandworker_tag $MYSQL : sleep .3;echo 3 c 3 c 3 c\t3 c\n +par_sqlandworker_tag $MYSQL : sleep .3;echo 3 d 3 d 3 d\t3 d\n +par_sqlandworker_tag $MYSQL : sleep .3;echo 3 e 3 e 3 e\t3 e\n +par_sqlandworker_tag $MYSQL : sleep .3;echo 4 a 4 a 4 a\t4 a\n +par_sqlandworker_tag $MYSQL : sleep .3;echo 4 b 4 b 4 b\t4 b\n +par_sqlandworker_tag $MYSQL : sleep .3;echo 4 c 4 c 4 c\t4 c\n +par_sqlandworker_tag $MYSQL : sleep .3;echo 4 d 4 d 4 d\t4 d\n +par_sqlandworker_tag $MYSQL : sleep .3;echo 4 e 4 e 4 e\t4 e\n +par_sqlandworker_tag $MYSQL : sleep .3;echo 5 a 5 a 5 a\t5 a\n +par_sqlandworker_tag $MYSQL : sleep .3;echo 5 b 5 b 5 b\t5 b\n +par_sqlandworker_tag $MYSQL : sleep .3;echo 5 c 5 c 5 c\t5 c\n +par_sqlandworker_tag $MYSQL : sleep .3;echo 5 d 5 d 5 d\t5 d\n +par_sqlandworker_tag $MYSQL : sleep .3;echo 5 e 5 e 5 e\t5 e\n +p_wrapper par_sqlandworker_tag \$PG +par_sqlandworker_tag $PG Exit=0 +par_sqlandworker_tag $PG Exit=0 +par_sqlandworker_tag $PG host | command | v1 | v2 | stdout | stderr +par_sqlandworker_tag $PG ------+-------------------+----+----+-------------+-------- +par_sqlandworker_tag $PG : | sleep .3;echo 1 a | 1 | a | 1 a 1 a+| +par_sqlandworker_tag $PG | | | | | +par_sqlandworker_tag $PG : | sleep .3;echo 1 b | 1 | b | 1 b 1 b+| +par_sqlandworker_tag $PG | | | | | +par_sqlandworker_tag $PG : | sleep .3;echo 1 c | 1 | c | 1 c 1 c+| +par_sqlandworker_tag $PG | | | | | +par_sqlandworker_tag $PG : | sleep .3;echo 1 d | 1 | d | 1 d 1 d+| +par_sqlandworker_tag $PG | | | | | +par_sqlandworker_tag $PG : | sleep .3;echo 1 e | 1 | e | 1 e 1 e+| +par_sqlandworker_tag $PG | | | | | +par_sqlandworker_tag $PG : | sleep .3;echo 2 a | 2 | a | 2 a 2 a+| +par_sqlandworker_tag $PG | | | | | +par_sqlandworker_tag $PG : | sleep .3;echo 2 b | 2 | b | 2 b 2 b+| +par_sqlandworker_tag $PG | | | | | +par_sqlandworker_tag $PG : | sleep .3;echo 2 c | 2 | c | 2 c 2 c+| +par_sqlandworker_tag $PG | | | | | +par_sqlandworker_tag $PG : | sleep .3;echo 2 d | 2 | d | 2 d 2 d+| +par_sqlandworker_tag $PG | | | | | +par_sqlandworker_tag $PG : | sleep .3;echo 2 e | 2 | e | 2 e 2 e+| +par_sqlandworker_tag $PG | | | | | +par_sqlandworker_tag $PG : | sleep .3;echo 3 a | 3 | a | 3 a 3 a+| +par_sqlandworker_tag $PG | | | | | +par_sqlandworker_tag $PG : | sleep .3;echo 3 b | 3 | b | 3 b 3 b+| +par_sqlandworker_tag $PG | | | | | +par_sqlandworker_tag $PG : | sleep .3;echo 3 c | 3 | c | 3 c 3 c+| +par_sqlandworker_tag $PG | | | | | +par_sqlandworker_tag $PG : | sleep .3;echo 3 d | 3 | d | 3 d 3 d+| +par_sqlandworker_tag $PG | | | | | +par_sqlandworker_tag $PG : | sleep .3;echo 3 e | 3 | e | 3 e 3 e+| +par_sqlandworker_tag $PG | | | | | +par_sqlandworker_tag $PG : | sleep .3;echo 4 a | 4 | a | 4 a 4 a+| +par_sqlandworker_tag $PG | | | | | +par_sqlandworker_tag $PG : | sleep .3;echo 4 b | 4 | b | 4 b 4 b+| +par_sqlandworker_tag $PG | | | | | +par_sqlandworker_tag $PG : | sleep .3;echo 4 c | 4 | c | 4 c 4 c+| +par_sqlandworker_tag $PG | | | | | +par_sqlandworker_tag $PG : | sleep .3;echo 4 d | 4 | d | 4 d 4 d+| +par_sqlandworker_tag $PG | | | | | +par_sqlandworker_tag $PG : | sleep .3;echo 4 e | 4 | e | 4 e 4 e+| +par_sqlandworker_tag $PG | | | | | +par_sqlandworker_tag $PG : | sleep .3;echo 5 a | 5 | a | 5 a 5 a+| +par_sqlandworker_tag $PG | | | | | +par_sqlandworker_tag $PG : | sleep .3;echo 5 b | 5 | b | 5 b 5 b+| +par_sqlandworker_tag $PG | | | | | +par_sqlandworker_tag $PG : | sleep .3;echo 5 c | 5 | c | 5 c 5 c+| +par_sqlandworker_tag $PG | | | | | +par_sqlandworker_tag $PG : | sleep .3;echo 5 d | 5 | d | 5 d 5 d+| +par_sqlandworker_tag $PG | | | | | +par_sqlandworker_tag $PG : | sleep .3;echo 5 e | 5 | e | 5 e 5 e+| +par_sqlandworker_tag $PG | | | | | +par_sqlandworker_tag $PG (25 rows) +par_sqlandworker_tag $PG +p_wrapper par_sqlandworker_tag \$SQLITE +par_sqlandworker_tag $SQLITE Exit=0 +par_sqlandworker_tag $SQLITE Exit=0 +par_sqlandworker_tag $SQLITE Host|Command|V1|V2|Stdout|Stderr +par_sqlandworker_tag $SQLITE :|sleep .3;echo 1 a|1|a|1 a 1 a +par_sqlandworker_tag $SQLITE | +par_sqlandworker_tag $SQLITE :|sleep .3;echo 1 b|1|b|1 b 1 b +par_sqlandworker_tag $SQLITE | +par_sqlandworker_tag $SQLITE :|sleep .3;echo 1 c|1|c|1 c 1 c +par_sqlandworker_tag $SQLITE | +par_sqlandworker_tag $SQLITE :|sleep .3;echo 1 d|1|d|1 d 1 d +par_sqlandworker_tag $SQLITE | +par_sqlandworker_tag $SQLITE :|sleep .3;echo 1 e|1|e|1 e 1 e +par_sqlandworker_tag $SQLITE | +par_sqlandworker_tag $SQLITE :|sleep .3;echo 2 a|2|a|2 a 2 a +par_sqlandworker_tag $SQLITE | +par_sqlandworker_tag $SQLITE :|sleep .3;echo 2 b|2|b|2 b 2 b +par_sqlandworker_tag $SQLITE | +par_sqlandworker_tag $SQLITE :|sleep .3;echo 2 c|2|c|2 c 2 c +par_sqlandworker_tag $SQLITE | +par_sqlandworker_tag $SQLITE :|sleep .3;echo 2 d|2|d|2 d 2 d +par_sqlandworker_tag $SQLITE | +par_sqlandworker_tag $SQLITE :|sleep .3;echo 2 e|2|e|2 e 2 e +par_sqlandworker_tag $SQLITE | +par_sqlandworker_tag $SQLITE :|sleep .3;echo 3 a|3|a|3 a 3 a +par_sqlandworker_tag $SQLITE | +par_sqlandworker_tag $SQLITE :|sleep .3;echo 3 b|3|b|3 b 3 b +par_sqlandworker_tag $SQLITE | +par_sqlandworker_tag $SQLITE :|sleep .3;echo 3 c|3|c|3 c 3 c +par_sqlandworker_tag $SQLITE | +par_sqlandworker_tag $SQLITE :|sleep .3;echo 3 d|3|d|3 d 3 d +par_sqlandworker_tag $SQLITE | +par_sqlandworker_tag $SQLITE :|sleep .3;echo 3 e|3|e|3 e 3 e +par_sqlandworker_tag $SQLITE | +par_sqlandworker_tag $SQLITE :|sleep .3;echo 4 a|4|a|4 a 4 a +par_sqlandworker_tag $SQLITE | +par_sqlandworker_tag $SQLITE :|sleep .3;echo 4 b|4|b|4 b 4 b +par_sqlandworker_tag $SQLITE | +par_sqlandworker_tag $SQLITE :|sleep .3;echo 4 c|4|c|4 c 4 c +par_sqlandworker_tag $SQLITE | +par_sqlandworker_tag $SQLITE :|sleep .3;echo 4 d|4|d|4 d 4 d +par_sqlandworker_tag $SQLITE | +par_sqlandworker_tag $SQLITE :|sleep .3;echo 4 e|4|e|4 e 4 e +par_sqlandworker_tag $SQLITE | +par_sqlandworker_tag $SQLITE :|sleep .3;echo 5 a|5|a|5 a 5 a +par_sqlandworker_tag $SQLITE | +par_sqlandworker_tag $SQLITE :|sleep .3;echo 5 b|5|b|5 b 5 b +par_sqlandworker_tag $SQLITE | +par_sqlandworker_tag $SQLITE :|sleep .3;echo 5 c|5|c|5 c 5 c +par_sqlandworker_tag $SQLITE | +par_sqlandworker_tag $SQLITE :|sleep .3;echo 5 d|5|d|5 d 5 d +par_sqlandworker_tag $SQLITE | +par_sqlandworker_tag $SQLITE :|sleep .3;echo 5 e|5|e|5 e 5 e +par_sqlandworker_tag $SQLITE | +p_wrapper par_sqlandworker_unbuffer \$MYSQL par_sqlandworker_unbuffer $MYSQL Exit=0 par_sqlandworker_unbuffer $MYSQL Exit=0 par_sqlandworker_unbuffer $MYSQL Host Command V1 V2 Stdout Stderr @@ -487,7 +907,7 @@ par_sqlandworker_unbuffer $MYSQL : sleep .3;echo 5 b 5 b NULL NULL par_sqlandworker_unbuffer $MYSQL : sleep .3;echo 5 c 5 c NULL NULL par_sqlandworker_unbuffer $MYSQL : sleep .3;echo 5 d 5 d NULL NULL par_sqlandworker_unbuffer $MYSQL : sleep .3;echo 5 e 5 e NULL NULL -par_sqlandworker_unbuffer $PG +p_wrapper par_sqlandworker_unbuffer \$PG par_sqlandworker_unbuffer $PG Exit=0 par_sqlandworker_unbuffer $PG Exit=0 par_sqlandworker_unbuffer $PG host | command | v1 | v2 | stdout | stderr @@ -519,7 +939,7 @@ par_sqlandworker_unbuffer $PG : | sleep .3;echo 5 d | 5 | d | | par_sqlandworker_unbuffer $PG : | sleep .3;echo 5 e | 5 | e | | par_sqlandworker_unbuffer $PG (25 rows) par_sqlandworker_unbuffer $PG -par_sqlandworker_unbuffer $SQLITE +p_wrapper par_sqlandworker_unbuffer \$SQLITE par_sqlandworker_unbuffer $SQLITE Exit=0 par_sqlandworker_unbuffer $SQLITE Exit=0 par_sqlandworker_unbuffer $SQLITE Host|Command|V1|V2|Stdout|Stderr From 402d3960d4ea1ab16b00fbff37f8c9dfea3e3282 Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Sun, 10 Jul 2016 16:02:26 +0200 Subject: [PATCH 05/12] parallel: Fixed bug #48449: memory leak. --- src/parallel | 71 ++++++++++++++------ testsuite/tests-to-run/parallel-local-30s.sh | 60 ++++++++++++----- testsuite/wanted-results/parallel-local-30s | 57 +++++++--------- 3 files changed, 117 insertions(+), 71 deletions(-) diff --git a/src/parallel b/src/parallel index c55da1b0..4c62232e 100755 --- a/src/parallel +++ b/src/parallel @@ -191,10 +191,10 @@ reaper(); ::debug("init", "Done reaping\n"); if($opt::pipe and @opt::a) { for my $job (@Global::tee_jobs) { - unlink $job->fh(2,"name"); + ::rm($job->fh(2,"name")); $job->set_fh(2,"name",""); $job->print(); - unlink $job->fh(1,"name"); + ::rm($job->fh(1,"name")); } } ::debug("init", "Cleaning\n"); @@ -3488,6 +3488,7 @@ sub reaper { ::wait_and_exit($Global::halt_exitstatus); } } + $job->cleanup(); start_more_jobs(); if($opt::progress) { my %progress = progress(); @@ -3814,6 +3815,7 @@ sub tmpfile { sub tmpname { # Select a name that does not exist # Do not create the file as it may be used for creating a socket (by tmux) + # Remember the name in $Global::unlink to avoid hitting the same name twice my $name = shift; my($tmpname); if(not -w $ENV{'TMPDIR'}) { @@ -3832,13 +3834,21 @@ sub tmpname { } sub tmpfifo { - # Securely make a fifo by securely making a dir with a fifo in it + # Find an unused name and mkfifo on it use POSIX qw(mkfifo); my $tmpfifo = tmpname("fif",@_); mkfifo($tmpfifo,0600); return $tmpfifo; } +sub rm { + # Remove file and remove it from %Global::unlink + # Uses: + # %Global::unlink + delete @Global::unlink{@_}; + unlink @_; +} + sub size_of_block_dev { # Like -s but for block devices # Input: @@ -4452,8 +4462,8 @@ sub new { sub DESTROY { my $self = shift; # Remove temporary files if they are created. - unlink $self->{'loadavg_file'}; - unlink $self->{'swap_activity_file'}; + ::rm($self->{'loadavg_file'}); + ::rm($self->{'swap_activity_file'}); } sub string { @@ -6158,6 +6168,8 @@ sub new { # filehandle for stdin (used for --pipe) # filename for writing stdout to (used for --files) # remaining data not sent to stdin (used for --pipe) + # tmpfiles to cleanup when job is done + 'unlink' => [], # amount of data sent via stdin (used for --pipe) 'transfersize' => 0, # size of files using --transfer 'returnsize' => 0, # size of files using --return @@ -6381,6 +6393,25 @@ sub openoutputfiles { } } +sub add_rm { + # Files to remove when job is done + my $self = shift; + push $self->{'unlink'}, @_; +} + +sub get_rm { + # Files to remove when job is done + my $self = shift; + return @{$self->{'unlink'}}; +} + +sub cleanup { + # Remove files when job is done + my $self = shift; + unlink $self->get_rm(); + delete @Global::unlink{$self->get_rm()}; +} + sub grouped { my $self = shift; # Set reading FD if using --group (--ungroup does not need) @@ -6392,7 +6423,7 @@ sub grouped { ::die_bug("fdr: Cannot open ".$self->fh($fdno,'name')); $self->set_fh($fdno,'r',$fdr); # Unlink if required - $Global::debug or unlink $self->fh($fdno,"unlink"); + $Global::debug or ::rm($self->fh($fdno,"unlink")); } } @@ -7402,7 +7433,7 @@ sub sshcleanup { my $workdir = $self->workdir(); my $cleancmd = ""; - for my $file ($self->cleanup()) { + for my $file ($self->remote_cleanup()) { my @subworkdirs = parentdirs_of($file); $cleancmd .= $sshlogin->cleanup_cmd($file,$workdir).";"; } @@ -7412,7 +7443,7 @@ sub sshcleanup { return $cleancmd; } -sub cleanup { +sub remote_cleanup { # Returns: # Files to remove at cleanup my $self = shift; @@ -7602,6 +7633,7 @@ sub start { $ENV{'PARALLEL_SEQ'} = $job->seq(); $ENV{'PARALLEL_PID'} = $$; $ENV{'PARALLEL_TMP'} = ::tmpname("par"); + $job->add_rm($ENV{'PARALLEL_TMP'}); ::debug("run", $Global::total_running, " processes . Starting (", $job->seq(), "): $command\n"); if($opt::pipe) { @@ -7721,6 +7753,7 @@ sub print_dryrun_and_verbose { my $actual_command = shift; # Temporary file name. Used for fifo to communicate exit val my $tmpfifo=::tmpname("tmx"); + $self->add_rm($tmpfifo); if(length($tmpfifo) >=100) { ::error("tmux does not support sockets with path > 100."); @@ -7882,7 +7915,7 @@ sub print { if($opt::dryrun) { # Nothing was printed to this job: # cleanup tmp files if --files was set - unlink $self->fh(1,"name"); + ::rm($self->fh(1,"name")); } if($opt::pipe and $self->virgin()) { # Skip --joblog, --dryrun, --verbose @@ -7973,8 +8006,8 @@ sub files_print { # Nothing was printed to this job: # cleanup unused tmp files if --files was set for my $fdno (1,2) { - unlink $self->fh($fdno,"name"); - unlink $self->fh($fdno,"unlink"); + ::rm($self->fh($fdno,"name")); + ::rm($self->fh($fdno,"unlink")); } } elsif($fdno == 1 and $self->fh($fdno,"name")) { print $out_fd $self->tag(),$self->fh($fdno,"name"),"\n"; @@ -9256,7 +9289,7 @@ sub tmux_length { my $tmuxcmd = $ENV{'TMUX'}." -S $tmpfile new-session -d -n echo $l". ("x"x$l). " && echo $l; rm -f $tmpfile"; push @out, ::qqx($tmuxcmd); - unlink $tmpfile; + ::rm($tmpfile); } ::debug("tmux","tmux-out ",@out); chomp @out; @@ -10124,9 +10157,9 @@ sub run { eval { $rv = $sth->execute(@_) }) { last; } else { - if($@ =~ /no such table/ + if($@ =~ /no such table|Table .* doesn.t exist|relation ".*" does not exist/ or - $DBI::errstr =~ /no such table/) { + $DBI::errstr =~ /no such table|Table .* doesn.t exist|relation ".*" does not exist/) { # This is fine: # It is just a worker that reported back too late - # another worker had finished the job first @@ -10330,7 +10363,7 @@ sub remove_dead_locks { if($host eq ::hostname()) { if(not kill 0, $pid) { ::debug("sem", "Dead: $d\n"); - unlink $d; + ::rm($d); } else { ::debug("sem", "Alive: $d\n"); } @@ -10379,12 +10412,12 @@ sub acquire { sub release { my $self = shift; - unlink $self->{'pidfile'}; + ::rm($self->{'pidfile'}); if($self->nlinks() == 1) { # This is the last link, so atomic cleanup $self->lock(); if($self->nlinks() == 1) { - unlink $self->{'idfile'}; + ::rm($self->{'idfile'}); rmdir $self->{'lockdir'}; } $self->unlock(); @@ -10402,7 +10435,7 @@ sub pid_change { $self->{'pidfile'} = $self->{'lockdir'}."/".$$.'@'.::hostname(); my $retval = link $self->{'idfile'}, $self->{'pidfile'}; ::debug("sem","link($self->{'idfile'},$self->{'pidfile'})=$retval\n"); - unlink $old_pidfile; + ::rm($old_pidfile); } sub atomic_link_if_count_less_than { @@ -10500,7 +10533,7 @@ sub lock { sub unlock { my $self = shift; - unlink $self->{'lockfile'}; + ::rm($self->{'lockfile'}); close $self->{'lockfh'}; ::debug("run", "unlocked\n"); } diff --git a/testsuite/tests-to-run/parallel-local-30s.sh b/testsuite/tests-to-run/parallel-local-30s.sh index 6504d5d5..88123115 100644 --- a/testsuite/tests-to-run/parallel-local-30s.sh +++ b/testsuite/tests-to-run/parallel-local-30s.sh @@ -4,27 +4,51 @@ # Each should be taking 30-100s and be possible to run in parallel # I.e.: No race conditions, no logins -# Assume /tmp/shm is easy to fill up -export SHM=/tmp/shm/parallel -mkdir -p $SHM -sudo umount -l $SHM -sudo mount -t tmpfs -o size=10% none $SHM +par_race_condition1() { + echo '### Test race condition on 8 CPU (my laptop)' + seq 1 5000000 > /tmp/parallel_race_cond + seq 1 10 | parallel -k "cat /tmp/parallel_race_cond | parallel --pipe --recend '' -k gzip >/dev/null; echo {}" + rm /tmp/parallel_race_cond +} -cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj4 -k --joblog /tmp/jl-`basename $0` -L1 -echo '### Test race condition on 8 CPU (my laptop)'; - seq 1 5000000 > /tmp/parallel_test; - seq 1 10 | parallel -k "cat /tmp/parallel_test | parallel --pipe --recend '' -k gzip >/dev/null; echo {}"; - rm /tmp/parallel_test +par_tmp_full() { + # Assume /tmp/shm is easy to fill up + export SHM=/tmp/shm/parallel + mkdir -p $SHM + sudo umount -l $SHM + sudo mount -t tmpfs -o size=10% none $SHM -echo '**' + echo "### Test --tmpdir running full. bug #40733 was caused by this" + stdout parallel -j1 --tmpdir $SHM cat /dev/zero ::: dummy +} -echo "### Test --tmpdir running full. bug #40733 was caused by this" - stdout parallel -j1 --tmpdir $SHM cat /dev/zero ::: dummy +par_bug_48290() { + echo "### bug #48290: round-robin does not distribute data based on business" + echo "Jobslot 1 is 256 times slower than jobslot 4 and should get much less data" + yes "$(seq 1000|xargs)" | head -c 30M | + parallel --tagstring {%} --linebuffer --compress -j4 --roundrobin --pipe --block 10k \ + pv -qL '{= $_=int( $job->slot()**4/2+1) =}'0000 | + perl -ne '/^\d+/ and $s{$&}++; END { print map { "$_\n" } sort { $s{$b} <=> $s{$a} } keys %s}' +} -echo '**' +par_memory_leak() { + a_run() { + seq $1 |time -v parallel true 2>&1 | + grep 'Maximum resident' | + field 6; + } + export -f a_run + echo "### Test for memory leaks" + echo "Of 10 runs of 1 job at least one should be bigger than a 3000 job run" + small_max=$(seq 10 | parallel a_run 1 | jq -s max) + big=$(a_run 3000) + if [ $small_max -lt $big ] ; then + echo "Bad: Memleak likely." + else + echo "Good: No memleak detected." + fi +} -echo "### bug #48290: round-robin does not distribute data based on business" - echo "Jobslot 1 is 8 times slower than jobslot 8 and should get much less data" - seq 10000000 | parallel --tagstring {%} --linebuffer --compress -j8 --roundrobin --pipe --block 300k 'pv -qL {%}00000'| perl -ne '/^\d+/ and $s{$&}++; END { print map { "$_\n" } sort { $s{$a} <=> $s{$b} } keys %s}' -EOF +export -f $(compgen -A function | grep par_) +compgen -A function | grep par_ | sort | parallel -j6 --tag -k '{} 2>&1' diff --git a/testsuite/wanted-results/parallel-local-30s b/testsuite/wanted-results/parallel-local-30s index 1b5d20b5..c92e9918 100644 --- a/testsuite/wanted-results/parallel-local-30s +++ b/testsuite/wanted-results/parallel-local-30s @@ -1,34 +1,23 @@ -echo '### Test race condition on 8 CPU (my laptop)'; seq 1 5000000 > /tmp/parallel_test; seq 1 10 | parallel -k "cat /tmp/parallel_test | parallel --pipe --recend '' -k gzip >/dev/null; echo {}"; rm /tmp/parallel_test -### Test race condition on 8 CPU (my laptop) -1 -2 -3 -4 -5 -6 -7 -8 -9 -10 -echo '**' -** -echo "### Test --tmpdir running full. bug #40733 was caused by this" -### Test --tmpdir running full. bug #40733 was caused by this - stdout parallel -j1 --tmpdir $SHM cat /dev/zero ::: dummy -parallel: Error: Output is incomplete. Cannot append to buffer file in /tmp/shm/parallel. Is the disk full? -parallel: Error: Change $TMPDIR with --tmpdir or use --compress. -echo '**' -** -echo "### bug #48290: round-robin does not distribute data based on business" -### bug #48290: round-robin does not distribute data based on business - echo "Jobslot 1 is 8 times slower than jobslot 8 and should get much less data" -Jobslot 1 is 8 times slower than jobslot 8 and should get much less data - seq 10000000 | parallel --tagstring {%} --linebuffer --compress -j8 --roundrobin --pipe --block 300k 'pv -qL {%}00000'| perl -ne '/^\d+/ and $s{$&}++; END { print map { "$_\n" } sort { $s{$a} <=> $s{$b} } keys %s}' -1 -2 -3 -5 -4 -7 -6 -8 +par_bug_48290 ### bug #48290: round-robin does not distribute data based on business +par_bug_48290 Jobslot 1 is 256 times slower than jobslot 4 and should get much less data +par_bug_48290 4 +par_bug_48290 3 +par_bug_48290 2 +par_bug_48290 1 +par_memory_leak ### Test for memory leaks +par_memory_leak Of 10 runs of 1 job at least one should be bigger than a 3000 job run +par_memory_leak Good: No memleak detected. +par_race_condition1 ### Test race condition on 8 CPU (my laptop) +par_race_condition1 1 +par_race_condition1 2 +par_race_condition1 3 +par_race_condition1 4 +par_race_condition1 5 +par_race_condition1 6 +par_race_condition1 7 +par_race_condition1 8 +par_race_condition1 9 +par_race_condition1 10 +par_tmp_full ### Test --tmpdir running full. bug #40733 was caused by this +par_tmp_full parallel: Error: Output is incomplete. Cannot append to buffer file in /tmp/shm/parallel. Is the disk full? +par_tmp_full parallel: Error: Change $TMPDIR with --tmpdir or use --compress. From 1cb786c62eeb1eb9a6e18bd189c6547fdf5d924e Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Sun, 10 Jul 2016 22:57:33 +0200 Subject: [PATCH 06/12] env_parallel.tcsh: --env _ works. --- src/env_parallel.pod | 57 ++- src/env_parallel.tcsh | 57 ++- testsuite/tests-to-run/parallel-local-ssh7.sh | 423 ++++++++++-------- testsuite/wanted-results/parallel-local-ssh7 | 36 ++ 4 files changed, 362 insertions(+), 211 deletions(-) diff --git a/src/env_parallel.pod b/src/env_parallel.pod index 3b800cbd..2196e581 100644 --- a/src/env_parallel.pod +++ b/src/env_parallel.pod @@ -202,27 +202,35 @@ E.g. by doing: =item aliases - alias myecho=echo - env_parallel myecho ::: test - env_parallel -S server myecho ::: test + alias myecho='echo aliases' + env_parallel myecho ::: work + env_parallel -S server myecho ::: work + env_parallel --env myecho myecho ::: work + env_parallel --env myecho -S server myecho ::: work =item functions - myfunc() { echo $*; } - env_parallel myfunc ::: test - env_parallel -S server myfunc ::: test + myfunc() { echo functions $*; } + env_parallel myfunc ::: work + env_parallel -S server myfunc ::: work + env_parallel --env myfunc myfunc ::: work + env_parallel --env myfunc -S server myfunc ::: work =item variables - myvar=test - env_parallel echo '$myvar' ::: test - env_parallel -S server echo '$myvar' ::: test + myvar=variables + env_parallel echo '$myvar' ::: work + env_parallel -S server echo '$myvar' ::: work + env_parallel --env myvar echo '$myvar' ::: work + env_parallel --env myvar -S server echo '$myvar' ::: work =item arrays - myarray=(foo bar baz) - env_parallel echo '${myarray[{}]}' ::: 0 1 2 - env_parallel -S server echo '${myarray[{}]}' ::: 0 1 2 + myarray=(arrays work, too) + env_parallel -k echo '${myarray[{}]}' ::: 0 1 2 + env_parallel -k -S server echo '${myarray[{}]}' ::: 0 1 2 + env_parallel -k --env myarray echo '${myarray[{}]}' ::: 0 1 2 + env_parallel -k --env myarray -S server echo '${myarray[{}]}' ::: 0 1 2 =back @@ -327,9 +335,11 @@ E.g. by doing: =item aliases - alias myecho echo - env_parallel myecho ::: test - env_parallel -S server myecho ::: test + alias myecho 'echo aliases' + env_parallel myecho ::: work + env_parallel -S server myecho ::: work + env_parallel --env myecho myecho ::: work + env_parallel --env myecho -S server myecho ::: work =item functions @@ -337,16 +347,19 @@ Not supported by B. =item variables - set myvar=test - env_parallel echo "\$myvar" ::: test - env_parallel -S tcsh@server echo "\$myvar" ::: test - + set myvar=variables + env_parallel echo '$myvar' ::: work + env_parallel -S server echo '$myvar' ::: work + env_parallel --env myvar echo '$myvar' ::: work + env_parallel --env myvar -S server echo '$myvar' ::: work =item arrays with no special chars - set myarray=(foo bar baz) - env_parallel echo "\${myarray\[\{\}\]}" ::: 1 2 3 - env_parallel -S tcsh@server echo "\${myarray\[\{\}\]}" ::: 1 2 3 + set myarray=(arrays work, too) + env_parallel -k echo \$'{myarray[{}]}' ::: 1 2 3 + env_parallel -k -S server echo \$'{myarray[{}]}' ::: 1 2 3 + env_parallel -k --env myarray echo \$'{myarray[{}]}' ::: 1 2 3 + env_parallel -k --env myarray -S server echo \$'{myarray[{}]}' ::: 1 2 3 =back diff --git a/src/env_parallel.tcsh b/src/env_parallel.tcsh index 0a97d75b..f9b7389e 100755 --- a/src/env_parallel.tcsh +++ b/src/env_parallel.tcsh @@ -29,8 +29,44 @@ if ("`alias env_parallel`" == '') then # Activate alias alias env_parallel 'setenv PARALLEL "\!*"; source `which env_parallel.tcsh`' else + # Get the --env variables if set + # --env _ should be ignored + # and convert a b c to (a|b|c) + # If --env not set: Match everything (.*) + set _tMpscRIpt=`tempfile` + cat <<'EOF' > $_tMpscRIpt + for(@ARGV){ + /^_$/ and $next_is_env = 0; + $next_is_env and push @envvar, split/,/, $_; + $next_is_env = /^--env$/; + } + $vars = join "|",map { quotemeta $_ } @envvar; + print $vars ? "($vars)" : "(.*)"; +'EOF' + set _grep_REGEXP="`perl $_tMpscRIpt -- !*`" + # Deal with --env _ + cat <<'EOF' > $_tMpscRIpt + #!/usr/bin/perl + + for(@ARGV){ + $next_is_env and push @envvar, split/,/, $_; + $next_is_env=/^--env$/; + } + if(grep { /^_$/ } @envvar) { + if(not open(IN, "<", "$ENV{HOME}/.parallel/ignored_vars")) { + print STDERR "parallel: Error: ", + "Run 'parallel --record-env' in a clean environment first.\n"; + } else { + chomp(@ignored_vars = ); + $vars = join "|",map { quotemeta $_ } @ignored_vars; + print $vars ? "($vars)" : "(nOVaRs)"; + } + } +'EOF' + set _ignore_UNDERSCORE="`perl $_tMpscRIpt -- !*`" + rm $_tMpscRIpt # Get the scalar and array variable names - set _vARnAmES=(`set | awk -e '{print $1}' |grep -vE '^(_|killring|prompt2)$'`) + set _vARnAmES=(`set | awk -e '{print $1}' |grep -vE '^(_|killring|prompt2|command)$' | grep -E "^$_grep_REGEXP"\$ | grep -vE "^$_ignore_UNDERSCORE"\$`) # Make a tmpfile for the variable definitions set _tMpvARfILe=`tempfile` @@ -38,18 +74,20 @@ else # Make a tmpfile for the variable definitions + alias set _tMpaLLfILe=`tempfile` foreach _vARnAmE ($_vARnAmES); - # if $?myvar && $#myvar <= 1 echo scalar_myvar=$var - eval if'($?'$_vARnAmE' && ${#'$_vARnAmE'} <= 1) echo scalar_'$_vARnAmE'='\"\$$_vARnAmE\" >> $_tMpvARfILe; - # if $?myvar && $#myvar > 1 echo array_myvar=$var - eval if'($?'$_vARnAmE' && ${#'$_vARnAmE'} > 1) echo array_'$_vARnAmE'="$'$_vARnAmE'"' >> $_tMpvARfILe; + # if not defined: next + eval if'(! $?'$_vARnAmE') continue' + # if $#myvar <= 1 echo scalar_myvar=$var + eval if'(${#'$_vARnAmE'} <= 1) echo scalar_'$_vARnAmE'='\"\$$_vARnAmE\" >> $_tMpvARfILe; + # if $#myvar > 1 echo array_myvar=$var + eval if'(${#'$_vARnAmE'} > 1) echo array_'$_vARnAmE'="$'$_vARnAmE'"' >> $_tMpvARfILe; end - + unset _vARnAmE _vARnAmES # shell quote variables (--plain needed due to $PARALLEL abuse) # Convert 'scalar_myvar=...' to 'set myvar=...' # Convert 'array_myvar=...' to 'set array=(...)' cat $_tMpvARfILe | parallel --plain --shellquote | perl -pe 's/^scalar_(\S+).=/set $1=/ or s/^array_(\S+).=(.*)/set $1=($2)/ && s/\\ / /g;' > $_tMpaLLfILe # Cleanup - rm $_tMpvARfILe; unset _tMpvARfILe _vARnAmE _vARnAmES + rm $_tMpvARfILe; unset _tMpvARfILe # ALIAS TO EXPORT ALIASES: @@ -76,7 +114,10 @@ else # Prepend with "\nalias " # s/^/\001alias /; # Quoted: s/\^/\\001alias\ /\; - alias | perl -pe s/\\047/\\047\\042\\047\\042\\047/g\;s/\^\(\\S+\)\(\\s+\)\\\(\(.\*\)\\\)/\\1\\2\\3/\;s/\^\(\\S+\)\(\\s+\)\(.\*\)/\\1\\2\\047\\3\\047/\;s/\^/\\001alias\ /\;s/\\\!/\\\\\\\!/g >> $_tMpaLLfILe + alias | \ + grep -E "^$_grep_REGEXP" | \ + grep -vE "^$_ignore_UNDERSCORE""[^_a-zA-Z]" | \ + perl -pe s/\\047/\\047\\042\\047\\042\\047/g\;s/\^\(\\S+\)\(\\s+\)\\\(\(.\*\)\\\)/\\1\\2\\3/\;s/\^\(\\S+\)\(\\s+\)\(.\*\)/\\1\\2\\047\\3\\047/\;s/\^/\\001alias\ /\;s/\\\!/\\\\\\\!/g >> $_tMpaLLfILe setenv PARALLEL_ENV "`cat $_tMpaLLfILe; rm $_tMpaLLfILe`"; unset _tMpaLLfILe; diff --git a/testsuite/tests-to-run/parallel-local-ssh7.sh b/testsuite/tests-to-run/parallel-local-ssh7.sh index a0db5f9c..a7df1eda 100755 --- a/testsuite/tests-to-run/parallel-local-ssh7.sh +++ b/testsuite/tests-to-run/parallel-local-ssh7.sh @@ -9,30 +9,30 @@ par_bash_man() { myscript=$(cat <<'_EOF' echo "### From man env_parallel" - - alias myecho="echo aliases"; - env_parallel myecho ::: work; - env_parallel -S server myecho ::: work; - env_parallel --env myecho myecho ::: work; - env_parallel --env myecho -S server myecho ::: work - - myfunc() { echo functions $*; }; - env_parallel myfunc ::: work; - env_parallel -S server myfunc ::: work; - env_parallel --env myfunc myfunc ::: work; - env_parallel --env myfunc -S server myfunc ::: work - - myvar=variables; - env_parallel echo "\$myvar" ::: work; - env_parallel -S server echo "\$myvar" ::: work; - env_parallel --env myvar echo "\$myvar" ::: work; - env_parallel --env myvar -S server echo "\$myvar" ::: work - - myarray=(arrays work, too); - env_parallel -k echo "\${myarray[{}]}" ::: 0 1 2; - env_parallel -k -S server echo "\${myarray[{}]}" ::: 0 1 2; - env_parallel -k --env myarray echo "\${myarray[{}]}" ::: 0 1 2; - env_parallel -k --env myarray -S server echo "\${myarray[{}]}" ::: 0 1 2 + + alias myecho="echo aliases"; + env_parallel myecho ::: work; + env_parallel -S server myecho ::: work; + env_parallel --env myecho myecho ::: work; + env_parallel --env myecho -S server myecho ::: work + + myfunc() { echo functions $*; }; + env_parallel myfunc ::: work; + env_parallel -S server myfunc ::: work; + env_parallel --env myfunc myfunc ::: work; + env_parallel --env myfunc -S server myfunc ::: work + + myvar=variables; + env_parallel echo "\$myvar" ::: work; + env_parallel -S server echo "\$myvar" ::: work; + env_parallel --env myvar echo "\$myvar" ::: work; + env_parallel --env myvar -S server echo "\$myvar" ::: work + + myarray=(arrays work, too); + env_parallel -k echo "\${myarray[{}]}" ::: 0 1 2; + env_parallel -k -S server echo "\${myarray[{}]}" ::: 0 1 2; + env_parallel -k --env myarray echo "\${myarray[{}]}" ::: 0 1 2; + env_parallel -k --env myarray -S server echo "\${myarray[{}]}" ::: 0 1 2 _EOF ) ssh bash@lo "$myscript" @@ -42,30 +42,30 @@ par_zsh_man() { echo '### zsh' myscript=$(cat <<'_EOF' echo "### From man env_parallel" - - alias myecho="echo aliases"; - env_parallel myecho ::: work; - env_parallel -S server myecho ::: work; - env_parallel --env myecho myecho ::: work; - env_parallel --env myecho -S server myecho ::: work - - myfunc() { echo functions $*; }; - env_parallel myfunc ::: work; - env_parallel -S server myfunc ::: work; - env_parallel --env myfunc myfunc ::: work; - env_parallel --env myfunc -S server myfunc ::: work - - myvar=variables; - env_parallel echo "\$myvar" ::: work; - env_parallel -S server echo "\$myvar" ::: work; - env_parallel --env myvar echo "\$myvar" ::: work; - env_parallel --env myvar -S server echo "\$myvar" ::: work - - myarray=(arrays work, too); - env_parallel -k echo "\${myarray[{}]}" ::: 1 2 3; - env_parallel -k -S server echo "\${myarray[{}]}" ::: 1 2 3; - env_parallel -k --env myarray echo "\${myarray[{}]}" ::: 1 2 3; - env_parallel -k --env myarray -S server echo "\${myarray[{}]}" ::: 1 2 3 + + alias myecho="echo aliases"; + env_parallel myecho ::: work; + env_parallel -S server myecho ::: work; + env_parallel --env myecho myecho ::: work; + env_parallel --env myecho -S server myecho ::: work + + myfunc() { echo functions $*; }; + env_parallel myfunc ::: work; + env_parallel -S server myfunc ::: work; + env_parallel --env myfunc myfunc ::: work; + env_parallel --env myfunc -S server myfunc ::: work + + myvar=variables; + env_parallel echo "\$myvar" ::: work; + env_parallel -S server echo "\$myvar" ::: work; + env_parallel --env myvar echo "\$myvar" ::: work; + env_parallel --env myvar -S server echo "\$myvar" ::: work + + myarray=(arrays work, too); + env_parallel -k echo "\${myarray[{}]}" ::: 1 2 3; + env_parallel -k -S server echo "\${myarray[{}]}" ::: 1 2 3; + env_parallel -k --env myarray echo "\${myarray[{}]}" ::: 1 2 3; + env_parallel -k --env myarray -S server echo "\${myarray[{}]}" ::: 1 2 3 _EOF ) ssh zsh@lo "$myscript" @@ -75,72 +75,134 @@ par_ksh_man() { echo '### ksh' myscript=$(cat <<'_EOF' echo "### From man env_parallel" - - . `which env_parallel.ksh`; - alias myecho="echo aliases"; - env_parallel myecho ::: work; - env_parallel -S server myecho ::: work; - env_parallel --env myecho myecho ::: work; - env_parallel --env myecho -S server myecho ::: work - - . `which env_parallel.ksh`; - myfunc() { echo functions $*; }; - env_parallel myfunc ::: work; - env_parallel -S server myfunc ::: work; - env_parallel --env myfunc myfunc ::: work; - env_parallel --env myfunc -S server myfunc ::: work - - . `which env_parallel.ksh`; - myvar=variables; - env_parallel echo "\$myvar" ::: work; - env_parallel -S server echo "\$myvar" ::: work; - env_parallel --env myvar echo "\$myvar" ::: work; - env_parallel --env myvar -S server echo "\$myvar" ::: work - - . `which env_parallel.ksh`; - myarray=(arrays work, too); - env_parallel -k echo "\${myarray[{}]}" ::: 0 1 2; - env_parallel -k -S server echo "\${myarray[{}]}" ::: 0 1 2; - env_parallel -k --env myarray echo "\${myarray[{}]}" ::: 0 1 2; - env_parallel -k --env myarray -S server echo "\${myarray[{}]}" ::: 0 1 2 + + . `which env_parallel.ksh`; + alias myecho="echo aliases"; + env_parallel myecho ::: work; + env_parallel -S server myecho ::: work; + env_parallel --env myecho myecho ::: work; + env_parallel --env myecho -S server myecho ::: work + + . `which env_parallel.ksh`; + myfunc() { echo functions $*; }; + env_parallel myfunc ::: work; + env_parallel -S server myfunc ::: work; + env_parallel --env myfunc myfunc ::: work; + env_parallel --env myfunc -S server myfunc ::: work + + . `which env_parallel.ksh`; + myvar=variables; + env_parallel echo "\$myvar" ::: work; + env_parallel -S server echo "\$myvar" ::: work; + env_parallel --env myvar echo "\$myvar" ::: work; + env_parallel --env myvar -S server echo "\$myvar" ::: work + + . `which env_parallel.ksh`; + myarray=(arrays work, too); + env_parallel -k echo "\${myarray[{}]}" ::: 0 1 2; + env_parallel -k -S server echo "\${myarray[{}]}" ::: 0 1 2; + env_parallel -k --env myarray echo "\${myarray[{}]}" ::: 0 1 2; + env_parallel -k --env myarray -S server echo "\${myarray[{}]}" ::: 0 1 2 _EOF ) ssh ksh@lo "$myscript" } +par_tcsh_man() { + echo '### tcsh' + myscript=$(cat <<'_EOF' + echo "### From man env_parallel" + + alias myecho 'echo aliases' + env_parallel myecho ::: work + env_parallel -S server myecho ::: work + env_parallel --env myecho myecho ::: work + env_parallel --env myecho -S server myecho ::: work + + echo Functions not supported + + set myvar=variables + env_parallel echo '$myvar' ::: work + env_parallel -S server echo '$myvar' ::: work + env_parallel --env myvar echo '$myvar' ::: work + env_parallel --env myvar -S server echo '$myvar' ::: work + + set myarray=(arrays work, too) + env_parallel -k echo \$'{myarray[{}]}' ::: 1 2 3 + env_parallel -k -S server echo \$'{myarray[{}]}' ::: 1 2 3 + env_parallel -k --env myarray echo \$'{myarray[{}]}' ::: 1 2 3 + env_parallel -k --env myarray -S server echo \$'{myarray[{}]}' ::: 1 2 3 + +_EOF + ) + ssh -tt tcsh@lo "$myscript" +} + +par_csh_man() { + echo '### csh' + myscript=$(cat <<'_EOF' + echo "### From man env_parallel" + + source `which env_parallel.csh`; + + alias myecho 'echo aliases' + env_parallel myecho ::: work + env_parallel -S server myecho ::: work + #env_parallel --env myecho myecho ::: work + #env_parallel --env myecho -S server myecho ::: work + + # Functions not supported + + set myvar=variables + env_parallel echo '$myvar' ::: work + env_parallel -S server echo '$myvar' ::: work + #env_parallel --env myvar echo '$myvar' ::: work + #env_parallel --env myvar -S server echo '$myvar' ::: work + + set myarray=(arrays work, too) + env_parallel -k echo \$'{myarray[{}]}' ::: 1 2 3 + env_parallel -k -S server echo \$'{myarray[{}]}' ::: 1 2 3 + #env_parallel -k --env myarray echo \$'{myarray[{}]}' ::: 1 2 3 + #env_parallel -k --env myarray -S server echo \$'{myarray[{}]}' ::: 1 2 3 + +_EOF + ) + ssh csh@lo "$myscript" +} + par_bash_underscore() { echo '### bash' myscript=$(cat <<'_EOF' echo "### Testing of --env _" - - env_parallel --record-env; - alias myecho="echo \$myvar aliases in"; - myfunc() { myecho ${myarray[@]} functions $*; }; - myvar="variables in"; - myarray=(and arrays in); - env_parallel myfunc ::: work; - env_parallel -S server myfunc ::: work; - env_parallel --env myfunc,myvar,myarray,myecho myfunc ::: work; - env_parallel --env myfunc,myvar,myarray,myecho -S server myfunc ::: work; - env_parallel --env _ myfunc ::: work; - env_parallel --env _ -S server myfunc ::: work; - - echo myvar >> ~/.parallel/ignored_vars; - env_parallel --env _ myfunc ::: work; - env_parallel --env _ -S server myfunc ::: work; - echo myarray >> ~/.parallel/ignored_vars; - env_parallel --env _ myfunc ::: work; - env_parallel --env _ -S server myfunc ::: work; - echo myecho >> ~/.parallel/ignored_vars; - env_parallel --env _ myfunc ::: work; - echo "OK if no myecho ^^^^^^^^^^^^^^^^^^^^^^^^^" >&2; - env_parallel --env _ -S server myfunc ::: work; - echo "OK if no myecho ^^^^^^^^^^^^^^^^^^^^^^^^^" >&2; - echo myfunc >> ~/.parallel/ignored_vars; - env_parallel --env _ myfunc ::: work; - echo "OK if no myfunc ^^^^^^^^^^^^^^^^^^^^^^^^^" >&2; - env_parallel --env _ -S server myfunc ::: work; - echo "OK if no myfunc ^^^^^^^^^^^^^^^^^^^^^^^^^" >&2; + + env_parallel --record-env; + alias myecho="echo \$myvar aliases in"; + myfunc() { myecho ${myarray[@]} functions $*; }; + myvar="variables in"; + myarray=(and arrays in); + env_parallel myfunc ::: work; + env_parallel -S server myfunc ::: work; + env_parallel --env myfunc,myvar,myarray,myecho myfunc ::: work; + env_parallel --env myfunc,myvar,myarray,myecho -S server myfunc ::: work; + env_parallel --env _ myfunc ::: work; + env_parallel --env _ -S server myfunc ::: work; + + echo myvar >> ~/.parallel/ignored_vars; + env_parallel --env _ myfunc ::: work; + env_parallel --env _ -S server myfunc ::: work; + echo myarray >> ~/.parallel/ignored_vars; + env_parallel --env _ myfunc ::: work; + env_parallel --env _ -S server myfunc ::: work; + echo myecho >> ~/.parallel/ignored_vars; + env_parallel --env _ myfunc ::: work; + echo "OK if no myecho ^^^^^^^^^^^^^^^^^^^^^^^^^" >&2; + env_parallel --env _ -S server myfunc ::: work; + echo "OK if no myecho ^^^^^^^^^^^^^^^^^^^^^^^^^" >&2; + echo myfunc >> ~/.parallel/ignored_vars; + env_parallel --env _ myfunc ::: work; + echo "OK if no myfunc ^^^^^^^^^^^^^^^^^^^^^^^^^" >&2; + env_parallel --env _ -S server myfunc ::: work; + echo "OK if no myfunc ^^^^^^^^^^^^^^^^^^^^^^^^^" >&2; _EOF ) ssh bash@lo "$myscript" @@ -150,39 +212,39 @@ par_zsh_underscore() { echo '### zsh' myscript=$(cat <<'_EOF' echo "### Testing of --env _" - - . `which env_parallel.zsh`; - env_parallel --record-env; - alias myecho="echo \$myvar aliases in"; - eval `cat <<"_EOS"; - myfunc() { myecho ${myarray[@]} functions $*; }; - myvar="variables in"; - myarray=(and arrays in); - env_parallel myfunc ::: work; - env_parallel -S server myfunc ::: work; - env_parallel --env myfunc,myvar,myarray,myecho myfunc ::: work; - env_parallel --env myfunc,myvar,myarray,myecho -S server myfunc ::: work; - env_parallel --env _ myfunc ::: work; - env_parallel --env _ -S server myfunc ::: work; - - echo myvar >> ~/.parallel/ignored_vars; - env_parallel --env _ myfunc ::: work; - env_parallel --env _ -S server myfunc ::: work; - echo myarray >> ~/.parallel/ignored_vars; - env_parallel --env _ myfunc ::: work; - env_parallel --env _ -S server myfunc ::: work; - echo myecho >> ~/.parallel/ignored_vars; - : Not using the function, because aliases are expanded in functions; - env_parallel --env _ myecho ::: work; - echo OK if no .^^^^^^^^^^^^^^^^^^^^^^^^^ myecho >&2; - env_parallel --env _ -S server myecho ::: work; - echo OK if no .^^^^^^^^^^^^^^^^^^^^^^^^^ myecho >&2; - echo myfunc >> ~/.parallel/ignored_vars; - env_parallel --env _ myfunc ::: work; - echo OK if no .^^^^^^^^^^^^^^^^^^^^^^^^^ myfunc >&2; - env_parallel --env _ -S server myfunc ::: work; - echo OK if no .^^^^^^^^^^^^^^^^^^^^^^^^^ myfunc >&2; -_EOS` + + . `which env_parallel.zsh`; + env_parallel --record-env; + alias myecho="echo \$myvar aliases in"; + eval `cat <<"_EOS"; + myfunc() { myecho ${myarray[@]} functions $*; }; + myvar="variables in"; + myarray=(and arrays in); + env_parallel myfunc ::: work; + env_parallel -S server myfunc ::: work; + env_parallel --env myfunc,myvar,myarray,myecho myfunc ::: work; + env_parallel --env myfunc,myvar,myarray,myecho -S server myfunc ::: work; + env_parallel --env _ myfunc ::: work; + env_parallel --env _ -S server myfunc ::: work; + + echo myvar >> ~/.parallel/ignored_vars; + env_parallel --env _ myfunc ::: work; + env_parallel --env _ -S server myfunc ::: work; + echo myarray >> ~/.parallel/ignored_vars; + env_parallel --env _ myfunc ::: work; + env_parallel --env _ -S server myfunc ::: work; + echo myecho >> ~/.parallel/ignored_vars; + : Not using the function, because aliases are expanded in functions; + env_parallel --env _ myecho ::: work; + echo OK if no .^^^^^^^^^^^^^^^^^^^^^^^^^ myecho >&2; + env_parallel --env _ -S server myecho ::: work; + echo OK if no .^^^^^^^^^^^^^^^^^^^^^^^^^ myecho >&2; + echo myfunc >> ~/.parallel/ignored_vars; + env_parallel --env _ myfunc ::: work; + echo OK if no .^^^^^^^^^^^^^^^^^^^^^^^^^ myfunc >&2; + env_parallel --env _ -S server myfunc ::: work; + echo OK if no .^^^^^^^^^^^^^^^^^^^^^^^^^ myfunc >&2; +_EOS` _EOF ) ssh zsh@lo "$myscript" @@ -192,36 +254,36 @@ par_ksh_underscore() { echo '### ksh' myscript=$(cat <<'_EOF' echo "### Testing of --env _" - - . `which env_parallel.ksh`; - env_parallel --record-env; - alias myecho="echo \$myvar aliases in"; - myfunc() { myecho ${myarray[@]} functions $*; }; - myvar="variables in"; - myarray=(and arrays in); - env_parallel myfunc ::: work; - env_parallel -S server myfunc ::: work; - env_parallel --env myfunc,myvar,myarray,myecho myfunc ::: work; - env_parallel --env myfunc,myvar,myarray,myecho -S server myfunc ::: work; - env_parallel --env _ myfunc ::: work; - env_parallel --env _ -S server myfunc ::: work; - - echo myvar >> ~/.parallel/ignored_vars; - env_parallel --env _ myfunc ::: work; - env_parallel --env _ -S server myfunc ::: work; - echo myarray >> ~/.parallel/ignored_vars; - env_parallel --env _ myfunc ::: work; - env_parallel --env _ -S server myfunc ::: work; - echo myecho >> ~/.parallel/ignored_vars; - env_parallel --env _ myfunc ::: work; - echo "OK if no myecho ^^^^^^^^^^^^^^^^^" >&2; - env_parallel --env _ -S server myfunc ::: work; - echo "OK if no myecho ^^^^^^^^^^^^^^^^^" >&2; - echo myfunc >> ~/.parallel/ignored_vars; - env_parallel --env _ myfunc ::: work; - echo "OK if no myfunc ^^^^^^^^^^^^^^^^^" >&2; - env_parallel --env _ -S server myfunc ::: work; - echo "OK if no myfunc ^^^^^^^^^^^^^^^^^" >&2; + + . `which env_parallel.ksh`; + env_parallel --record-env; + alias myecho="echo \$myvar aliases in"; + myfunc() { myecho ${myarray[@]} functions $*; }; + myvar="variables in"; + myarray=(and arrays in); + env_parallel myfunc ::: work; + env_parallel -S server myfunc ::: work; + env_parallel --env myfunc,myvar,myarray,myecho myfunc ::: work; + env_parallel --env myfunc,myvar,myarray,myecho -S server myfunc ::: work; + env_parallel --env _ myfunc ::: work; + env_parallel --env _ -S server myfunc ::: work; + + echo myvar >> ~/.parallel/ignored_vars; + env_parallel --env _ myfunc ::: work; + env_parallel --env _ -S server myfunc ::: work; + echo myarray >> ~/.parallel/ignored_vars; + env_parallel --env _ myfunc ::: work; + env_parallel --env _ -S server myfunc ::: work; + echo myecho >> ~/.parallel/ignored_vars; + env_parallel --env _ myfunc ::: work; + echo "OK if no myecho ^^^^^^^^^^^^^^^^^" >&2; + env_parallel --env _ -S server myfunc ::: work; + echo "OK if no myecho ^^^^^^^^^^^^^^^^^" >&2; + echo myfunc >> ~/.parallel/ignored_vars; + env_parallel --env _ myfunc ::: work; + echo "OK if no myfunc ^^^^^^^^^^^^^^^^^" >&2; + env_parallel --env _ -S server myfunc ::: work; + echo "OK if no myfunc ^^^^^^^^^^^^^^^^^" >&2; _EOF ) ssh ksh@lo "$myscript" @@ -276,7 +338,6 @@ par_zsh_funky() { echo ${assocarr[a]} echo Funky-"$funky"-funky } - env_parallel alias_echo ::: alias_works env_parallel func_echo ::: function_works env_parallel -S zsh@lo alias_echo ::: alias_works_over_ssh @@ -290,8 +351,8 @@ _EOF par_ksh_funky() { myscript=$(cat <<'_EOF' - . `which env_parallel.ksh`; - + . `which env_parallel.ksh`; + myvar="myvar works" funky=$(perl -e "print pack \"c*\", 1..255") myarray=("" array_val2 3 "" 5 " space 6 ") @@ -299,7 +360,7 @@ par_ksh_funky() { assocarr[a]=assoc_val_a assocarr[b]=assoc_val_b alias alias_echo="echo 3 arg"; - + func_echo() { echo $*; echo "$myvar" @@ -307,7 +368,7 @@ par_ksh_funky() { echo ${assocarr[a]} echo Funky-"$funky"-funky } - + env_parallel alias_echo ::: alias_works env_parallel func_echo ::: function_works env_parallel -S ksh@lo alias_echo ::: alias_works_over_ssh @@ -323,18 +384,18 @@ par_fish_funky() { myscript=$(cat <<'_EOF' set myvar "myvar works" setenv myenvvar "myenvvar works" - + set funky (perl -e "print pack \"c*\", 1..255") setenv funkyenv (perl -e "print pack \"c*\", 1..255") - + set myarray "" array_val2 3 "" 5 " space 6 " - + # Assoc arrays do not exist #typeset -A assocarr #assocarr[a]=assoc_val_a #assocarr[b]=assoc_val_b alias alias_echo="echo 3 arg"; - + function func_echo echo $argv; echo "$myvar" @@ -351,12 +412,12 @@ par_fish_funky() { echo echo end - + env_parallel alias_echo ::: alias_works env_parallel func_echo ::: function_works env_parallel -S fish@lo alias_echo ::: alias_works_over_ssh env_parallel -S fish@lo func_echo ::: function_works_over_ssh - echo + echo echo "$funky" | parallel --shellquote _EOF ) @@ -373,7 +434,7 @@ par_csh_funky() { #assocarr[b]=assoc_val_b alias alias_echo echo 3 arg; alias alias_echo_var 'echo $argv; echo "$myvar"; echo "${myarray[4]} special chars problem"; echo Funky-"$funky"-funky' - + #function func_echo # echo $argv; # echo $myvar; @@ -381,7 +442,7 @@ par_csh_funky() { # #echo ${assocarr[a]} # echo Funky-"$funky"-funky #end - + env_parallel alias_echo ::: alias_works env_parallel alias_echo_var ::: alias_var_works env_parallel func_echo ::: function_does_not_work diff --git a/testsuite/wanted-results/parallel-local-ssh7 b/testsuite/wanted-results/parallel-local-ssh7 index 9d1001a6..6b952dbf 100644 --- a/testsuite/wanted-results/parallel-local-ssh7 +++ b/testsuite/wanted-results/parallel-local-ssh7 @@ -81,6 +81,18 @@ par_csh_funky space special chars problem par_csh_funky Funky-  !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~-funky alias_var_works_over_ssh par_csh_funky func_echo: Command not found. par_csh_funky \\\\\\\\ \ \ \ \ \\\\\\\\\\\\\\ \!\"\#\$%\&\'\(\)\*+,-./0123456789:\;\<\=\>\?@ABCDEFGHIJKLMNOPQRSTUVWXYZ\[\\\]\^_\`abcdefghijklmnopqrstuvwxyz\{\|\}\~\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\ +par_csh_man ### csh +par_csh_man ### From man env_parallel +par_csh_man aliases work +par_csh_man aliases work +par_csh_man variables work +par_csh_man variables work +par_csh_man arrays +par_csh_man work, +par_csh_man too +par_csh_man arrays +par_csh_man work, +par_csh_man too par_fish_funky env_parallel: Warning: ASCII value 1 in variables is not supported par_fish_funky env_parallel: Warning: ASCII value 1 in variables is not supported par_fish_funky 3 arg alias_works @@ -180,6 +192,30 @@ par_ksh_underscore /usr/bin/ksh: line 100: myfunc: not found par_ksh_underscore OK if no myfunc ^^^^^^^^^^^^^^^^^ par_ksh_underscore /usr/bin/ksh: line 100: myfunc: not found par_ksh_underscore OK if no myfunc ^^^^^^^^^^^^^^^^^ +par_tcsh_man ### tcsh +par_tcsh_man ### From man env_parallel +par_tcsh_man aliases work +par_tcsh_man aliases work +par_tcsh_man aliases work +par_tcsh_man aliases work +par_tcsh_man Functions not supported +par_tcsh_man variables work +par_tcsh_man variables work +par_tcsh_man variables work +par_tcsh_man variables work +par_tcsh_man arrays +par_tcsh_man work, +par_tcsh_man too +par_tcsh_man arrays +par_tcsh_man work, +par_tcsh_man too +par_tcsh_man arrays +par_tcsh_man work, +par_tcsh_man too +par_tcsh_man arrays +par_tcsh_man work, +par_tcsh_man too +par_tcsh_man Connection to lo closed. par_zsh_funky 3 arg alias_works par_zsh_funky function_works par_zsh_funky myvar works From 29a7113d7ffefe650e5c9503b093ba89f21dfb9d Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Sun, 10 Jul 2016 23:00:20 +0200 Subject: [PATCH 07/12] testsuite: Faster test of whether --linebuffer matters. --- testsuite/tests-to-run/parallel-local-100s.sh | 54 ++++++------------- testsuite/tests-to-run/parallel-local-30s.sh | 43 +++++++++++++++ testsuite/wanted-results/parallel-local-100s | 49 +++++++---------- testsuite/wanted-results/parallel-local-30s | 4 ++ 4 files changed, 82 insertions(+), 68 deletions(-) diff --git a/testsuite/tests-to-run/parallel-local-100s.sh b/testsuite/tests-to-run/parallel-local-100s.sh index d6cb104d..e8984d0b 100644 --- a/testsuite/tests-to-run/parallel-local-100s.sh +++ b/testsuite/tests-to-run/parallel-local-100s.sh @@ -8,45 +8,23 @@ TMP5G=${TMP5G:-/dev/shm} export TMP5G -cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj0 -k --joblog /tmp/jl-`basename $0` -L1 -echo '### Test of --retries on unreachable host' +par_retries_unreachable() { + echo '### Test of --retries on unreachable host' seq 2 | stdout parallel -k --retries 2 -v -S 4.3.2.1,: echo +} -echo '**' +par_outside_file_handle_limit() { + echo "### Test Force outside the file handle limit, 2009-02-17 Gave fork error" + (echo echo Start; seq 1 20000 | perl -pe 's/^/true /'; echo echo end) | + stdout parallel -uj 0 | egrep -v 'processes took|adjusting' +} -echo "### Test Force outside the file handle limit, 2009-02-17 Gave fork error" - (echo echo Start; seq 1 20000 | perl -pe 's/^/true /'; echo echo end) | stdout parallel -uj 0 | egrep -v 'processes took|adjusting' +par_over_4GB() { + echo '### Test if we can deal with output > 4 GB' + echo | + nice parallel --tmpdir $TMP5G -q perl -e '$a="x"x1000000;for(0..4300){print $a}' | + nice md5sum +} -echo '**' - -echo '### Test if we can deal with output > 4 GB' - echo | nice parallel --tmpdir $TMP5G -q perl -e '$a="x"x1000000;for(0..4300){print $a}' | nice md5sum - -echo '**' - -echo 'bug #41613: --compress --line-buffer no --tagstring'; - diff - <(nice perl -e 'for("x011".."x110"){print "$_\t", ("\n", map { rand } (1..100000)) }'| - nice parallel -N10 -L1 --pipe -j6 --block 20M --compress - pv -qL 1000000 | perl -pe 's/(....).*/$1/') - <(nice perl -e 'for("x011".."x110"){print "$_\t", ("\n", map { rand } (1..100000)) }'| - nice parallel -N10 -L1 --pipe -j6 --block 20M --compress --line-buffer - pv -qL 1000000 | perl -pe 's/(....).*/$1/') - >/dev/null - || (echo 'Good: --line-buffer matters'; false) && echo 'Bad: --line-buffer not working' - -echo 'bug #41613: --compress --line-buffer with --tagstring'; - diff - <(nice perl -e 'for("x011".."x110"){print "$_\t", ("\n", map { rand } (1..100000)) }'| - nice parallel -N10 -L1 --pipe -j6 --block 20M --compress --tagstring {#} - pv -qL 1000000 | perl -pe 's/(....).*/$1/') - <(nice perl -e 'for("x011".."x110"){print "$_\t", ("\n", map { rand } (1..100000)) }'| - nice parallel -N10 -L1 --pipe -j6 --block 20M --compress --tagstring {#} --line-buffer - pv -qL 1000000 | perl -pe 's/(....).*/$1/') - >/dev/null - || (echo 'Good: --line-buffer matters'; false) && echo 'Bad: --line-buffer not working' - -echo '**' - - -EOF +export -f $(compgen -A function | grep par_) +compgen -A function | grep par_ | sort | parallel -vj0 -k --tag --joblog /tmp/jl-`basename $0` '{} 2>&1' diff --git a/testsuite/tests-to-run/parallel-local-30s.sh b/testsuite/tests-to-run/parallel-local-30s.sh index 88123115..747d7142 100644 --- a/testsuite/tests-to-run/parallel-local-30s.sh +++ b/testsuite/tests-to-run/parallel-local-30s.sh @@ -49,6 +49,49 @@ par_memory_leak() { fi } +par_linebuffer_matters_compress_tag() { + echo "### (--linebuffer) --compress --tag should give different output" + random_data_with_id_prepended() { + perl -pe 's/^/'$1'/' /dev/urandom | + pv -qL 300000 | head -c 1000000 + } + export -f random_data_with_id_prepended + + nolb=$(seq 10 | + parallel -j0 --compress --tag random_data_with_id_prepended {#} | + field 1 | uniq) + lb=$(seq 10 | + parallel -j0 --linebuffer --compress --tag random_data_with_id_prepended {#} | + field 1 | uniq) + if [ "$lb" == "$nolb" ] ; then + echo "BAD: --linebuffer makes no difference" + else + echo "OK: --linebuffer makes a difference" + fi +} + +par_linebuffer_matters_compress() { + echo "### (--linebuffer) --compress --tag should give different output" + random_data_with_id_prepended() { + perl -pe 's/^/'$1'/' /dev/urandom | + pv -qL 300000 | head -c 1000000 + } + export -f random_data_with_id_prepended + + nolb=$(seq 10 | + parallel -j0 --compress random_data_with_id_prepended {#} | + field 1 | uniq) + lb=$(seq 10 | + parallel -j0 --linebuffer --compress random_data_with_id_prepended {#} | + field 1 | uniq) + if [ "$lb" == "$nolb" ] ; then + echo "BAD: --linebuffer makes no difference" + else + echo "OK: --linebuffer makes a difference" + fi +} + + export -f $(compgen -A function | grep par_) compgen -A function | grep par_ | sort | parallel -j6 --tag -k '{} 2>&1' diff --git a/testsuite/wanted-results/parallel-local-100s b/testsuite/wanted-results/parallel-local-100s index 9fe1bb28..05bbcede 100644 --- a/testsuite/wanted-results/parallel-local-100s +++ b/testsuite/wanted-results/parallel-local-100s @@ -1,34 +1,23 @@ -echo '### Test of --retries on unreachable host' -### Test of --retries on unreachable host - seq 2 | stdout parallel -k --retries 2 -v -S 4.3.2.1,: echo -parallel: Warning: Could not figure out number of cpus on 4.3.2.1 (). Using 1. -echo 1 -1 -echo 2 -2 -echo '**' -** -echo "### Test Force outside the file handle limit, 2009-02-17 Gave fork error" -### Test Force outside the file handle limit, 2009-02-17 Gave fork error - (echo echo Start; seq 1 20000 | perl -pe 's/^/true /'; echo echo end) | stdout parallel -uj 0 | egrep -v 'processes took|adjusting' -parallel: Warning: Only enough file handles to run 252 jobs in parallel. -parallel: Warning: Running 'parallel -j0 -N 252 --pipe parallel -j0' or -parallel: Warning: raising ulimit -n or /etc/security/limits.conf may help. -Start -end -echo '**' -** -echo '### Test if we can deal with output > 4 GB' -### Test if we can deal with output > 4 GB - echo | nice parallel --tmpdir $TMP5G -q perl -e '$a="x"x1000000;for(0..4300){print $a}' | nice md5sum -46a318993dfc8e2afd71ff2bc6f605f1 - -echo '**' -** -echo 'bug #41613: --compress --line-buffer no --tagstring'; diff <(nice perl -e 'for("x011".."x110"){print "$_\t", ("\n", map { rand } (1..100000)) }'| nice parallel -N10 -L1 --pipe -j6 --block 20M --compress pv -qL 1000000 | perl -pe 's/(....).*/$1/') <(nice perl -e 'for("x011".."x110"){print "$_\t", ("\n", map { rand } (1..100000)) }'| nice parallel -N10 -L1 --pipe -j6 --block 20M --compress --line-buffer pv -qL 1000000 | perl -pe 's/(....).*/$1/') >/dev/null || (echo 'Good: --line-buffer matters'; false) && echo 'Bad: --line-buffer not working' +par_compress_linebuffer 2>&1 bug #41613: --compress --line-buffer no --tagstring Good: --line-buffer matters -echo 'bug #41613: --compress --line-buffer with --tagstring'; diff <(nice perl -e 'for("x011".."x110"){print "$_\t", ("\n", map { rand } (1..100000)) }'| nice parallel -N10 -L1 --pipe -j6 --block 20M --compress --tagstring {#} pv -qL 1000000 | perl -pe 's/(....).*/$1/') <(nice perl -e 'for("x011".."x110"){print "$_\t", ("\n", map { rand } (1..100000)) }'| nice parallel -N10 -L1 --pipe -j6 --block 20M --compress --tagstring {#} --line-buffer pv -qL 1000000 | perl -pe 's/(....).*/$1/') >/dev/null || (echo 'Good: --line-buffer matters'; false) && echo 'Bad: --line-buffer not working' +par_compress_linebuffer_tagstring 2>&1 bug #41613: --compress --line-buffer with --tagstring Good: --line-buffer matters -echo '**' -** +par_outside_file_handle_limit 2>&1 +par_outside_file_handle_limit ### Test Force outside the file handle limit, 2009-02-17 Gave fork error +par_outside_file_handle_limit parallel: Warning: Only enough file handles to run 252 jobs in parallel. +par_outside_file_handle_limit parallel: Warning: Running 'parallel -j0 -N 252 --pipe parallel -j0' or +par_outside_file_handle_limit parallel: Warning: raising ulimit -n or /etc/security/limits.conf may help. +par_outside_file_handle_limit Start +par_outside_file_handle_limit end +par_over_4GB 2>&1 +par_over_4GB ### Test if we can deal with output > 4 GB +par_over_4GB 46a318993dfc8e2afd71ff2bc6f605f1 - +par_retries_unreachable 2>&1 +par_retries_unreachable ### Test of --retries on unreachable host +par_retries_unreachable parallel: Warning: Could not figure out number of cpus on 4.3.2.1 (). Using 1. +par_retries_unreachable echo 1 +par_retries_unreachable 1 +par_retries_unreachable echo 2 +par_retries_unreachable 2 diff --git a/testsuite/wanted-results/parallel-local-30s b/testsuite/wanted-results/parallel-local-30s index c92e9918..dbf72423 100644 --- a/testsuite/wanted-results/parallel-local-30s +++ b/testsuite/wanted-results/parallel-local-30s @@ -4,6 +4,10 @@ par_bug_48290 4 par_bug_48290 3 par_bug_48290 2 par_bug_48290 1 +par_linebuffer_matters_compress ### (--linebuffer) --compress --tag should give different output +par_linebuffer_matters_compress OK: --linebuffer makes a difference +par_linebuffer_matters_compress_tag ### (--linebuffer) --compress --tag should give different output +par_linebuffer_matters_compress_tag OK: --linebuffer makes a difference par_memory_leak ### Test for memory leaks par_memory_leak Of 10 runs of 1 job at least one should be bigger than a 3000 job run par_memory_leak Good: No memleak detected. From 1117140adb06790ed8b0beb8b680fc6207d461a0 Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Sun, 10 Jul 2016 23:01:53 +0200 Subject: [PATCH 08/12] parallel: Fixed SQL racing error. If two workers both work on the last job, the master will drop the table when the first returns. The second should ignore the dropped table. --- src/parallel | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/parallel b/src/parallel index 4c62232e..a0f80a54 100755 --- a/src/parallel +++ b/src/parallel @@ -10194,7 +10194,8 @@ sub get { my $self = shift; my $sth = $self->run(@_); my @retval; - while(1) { + # If $sth = 0 it means the table was dropped by another process + while($sth) { my @row = $sth->fetchrow_array(); @row or last; push @retval, \@row; From bec3d7924b662c1fdf7286ae866abda78b61ba62 Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Mon, 11 Jul 2016 03:30:18 +0200 Subject: [PATCH 09/12] env_parallel.tcsh: Testing of --env _ for tcsh works. --- src/env_parallel.tcsh | 2 +- testsuite/tests-to-run/parallel-local-ssh7.sh | 34 +++++++++++++++++++ testsuite/wanted-results/parallel-local-100s | 6 ---- testsuite/wanted-results/parallel-local-ssh7 | 18 ++++++++++ 4 files changed, 53 insertions(+), 7 deletions(-) diff --git a/src/env_parallel.tcsh b/src/env_parallel.tcsh index f9b7389e..8524167c 100755 --- a/src/env_parallel.tcsh +++ b/src/env_parallel.tcsh @@ -66,7 +66,7 @@ else set _ignore_UNDERSCORE="`perl $_tMpscRIpt -- !*`" rm $_tMpscRIpt # Get the scalar and array variable names - set _vARnAmES=(`set | awk -e '{print $1}' |grep -vE '^(_|killring|prompt2|command)$' | grep -E "^$_grep_REGEXP"\$ | grep -vE "^$_ignore_UNDERSCORE"\$`) + set _vARnAmES=(`set | awk -e '{print $1}' |grep -vE '^(#|_|killring|prompt2|command)$' | grep -E "^$_grep_REGEXP"\$ | grep -vE "^$_ignore_UNDERSCORE"\$`) # Make a tmpfile for the variable definitions set _tMpvARfILe=`tempfile` diff --git a/testsuite/tests-to-run/parallel-local-ssh7.sh b/testsuite/tests-to-run/parallel-local-ssh7.sh index a7df1eda..f69582b8 100755 --- a/testsuite/tests-to-run/parallel-local-ssh7.sh +++ b/testsuite/tests-to-run/parallel-local-ssh7.sh @@ -289,6 +289,40 @@ _EOF ssh ksh@lo "$myscript" } +par_tcsh_underscore() { + echo '### tcsh' + myscript=$(cat <<'_EOF' + echo "### Testing of --env _" + +# . `which env_parallel.tcsh`; + env_parallel --record-env; + alias myecho "echo "\$"myvar "\$'myarray'" aliases"; + set myvar="variables"; + set myarray=(and arrays in); + env_parallel myecho ::: work; + env_parallel -S server myecho ::: work; + env_parallel --env myvar,myarray,myecho myecho ::: work; + env_parallel --env myvar,myarray,myecho -S server myecho ::: work; + env_parallel --env _ myecho ::: work; + env_parallel --env _ -S server myecho ::: work; + + echo myvar >> ~/.parallel/ignored_vars; + env_parallel --env _ myecho ::: work; + env_parallel --env _ -S server myecho ::: work; + alias myecho "echo "\$'myarray'" aliases"; + echo myarray >> ~/.parallel/ignored_vars; + env_parallel --env _ myecho ::: work; + env_parallel --env _ -S server myecho ::: work; + echo myecho >> ~/.parallel/ignored_vars; + env_parallel --env _ myecho ::: work; + echo "OK ^^^^^^^^^^^^^^^^^ if no myecho" >/dev/stderr; + env_parallel --env _ -S server myecho ::: work; + echo "OK ^^^^^^^^^^^^^^^^^ if no myecho" >/dev/stderr; +_EOF + ) + ssh -tt tcsh@lo "$myscript" +} + # Test env_parallel: # + for each shell # + remote, locally diff --git a/testsuite/wanted-results/parallel-local-100s b/testsuite/wanted-results/parallel-local-100s index 05bbcede..eca8b35d 100644 --- a/testsuite/wanted-results/parallel-local-100s +++ b/testsuite/wanted-results/parallel-local-100s @@ -1,9 +1,3 @@ -par_compress_linebuffer 2>&1 -bug #41613: --compress --line-buffer no --tagstring -Good: --line-buffer matters -par_compress_linebuffer_tagstring 2>&1 -bug #41613: --compress --line-buffer with --tagstring -Good: --line-buffer matters par_outside_file_handle_limit 2>&1 par_outside_file_handle_limit ### Test Force outside the file handle limit, 2009-02-17 Gave fork error par_outside_file_handle_limit parallel: Warning: Only enough file handles to run 252 jobs in parallel. diff --git a/testsuite/wanted-results/parallel-local-ssh7 b/testsuite/wanted-results/parallel-local-ssh7 index 6b952dbf..eb9aa1ad 100644 --- a/testsuite/wanted-results/parallel-local-ssh7 +++ b/testsuite/wanted-results/parallel-local-ssh7 @@ -216,6 +216,24 @@ par_tcsh_man arrays par_tcsh_man work, par_tcsh_man too par_tcsh_man Connection to lo closed. +par_tcsh_underscore ### tcsh +par_tcsh_underscore ### Testing of --env _ +par_tcsh_underscore #: Command not found. +par_tcsh_underscore variables and arrays in aliases work +par_tcsh_underscore variables and arrays in aliases work +par_tcsh_underscore variables and arrays in aliases work +par_tcsh_underscore variables and arrays in aliases work +par_tcsh_underscore variables and arrays in aliases work +par_tcsh_underscore variables and arrays in aliases work +par_tcsh_underscore myvar: Undefined variable. +par_tcsh_underscore myvar: Undefined variable. +par_tcsh_underscore myarray: Undefined variable. +par_tcsh_underscore myarray: Undefined variable. +par_tcsh_underscore myecho: Command not found. +par_tcsh_underscore OK ^^^^^^^^^^^^^^^^^ if no myecho +par_tcsh_underscore myecho: Command not found. +par_tcsh_underscore OK ^^^^^^^^^^^^^^^^^ if no myecho +par_tcsh_underscore Connection to lo closed. par_zsh_funky 3 arg alias_works par_zsh_funky function_works par_zsh_funky myvar works From 9feeaefc564bc6f27aac53f58ebd5e9f222d2ba0 Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Mon, 11 Jul 2016 03:45:38 +0200 Subject: [PATCH 10/12] env_parallel.csh: Testing of --env _ for csh works. --- testsuite/tests-to-run/parallel-local-ssh7.sh | 34 +++++++++++++++++++ testsuite/wanted-results/parallel-local-ssh7 | 18 ++++++++++ 2 files changed, 52 insertions(+) diff --git a/testsuite/tests-to-run/parallel-local-ssh7.sh b/testsuite/tests-to-run/parallel-local-ssh7.sh index f69582b8..5d6e3504 100755 --- a/testsuite/tests-to-run/parallel-local-ssh7.sh +++ b/testsuite/tests-to-run/parallel-local-ssh7.sh @@ -323,6 +323,40 @@ _EOF ssh -tt tcsh@lo "$myscript" } +par_csh_underscore() { + echo '### csh' + myscript=$(cat <<'_EOF' + echo "### Testing of --env _" + +# . `which env_parallel.tcsh`; + env_parallel --record-env; + alias myecho "echo "\$"myvar "\$'myarray'" aliases"; + set myvar="variables"; + set myarray=(and arrays in); + env_parallel myecho ::: work; + env_parallel -S server myecho ::: work; + env_parallel --env myvar,myarray,myecho myecho ::: work; + env_parallel --env myvar,myarray,myecho -S server myecho ::: work; + env_parallel --env _ myecho ::: work; + env_parallel --env _ -S server myecho ::: work; + + echo myvar >> ~/.parallel/ignored_vars; + env_parallel --env _ myecho ::: work; + env_parallel --env _ -S server myecho ::: work; + alias myecho "echo "\$'myarray'" aliases"; + echo myarray >> ~/.parallel/ignored_vars; + env_parallel --env _ myecho ::: work; + env_parallel --env _ -S server myecho ::: work; + echo myecho >> ~/.parallel/ignored_vars; + env_parallel --env _ myecho ::: work; + echo "OK ^^^^^^^^^^^^^^^^^ if no myecho" >/dev/stderr; + env_parallel --env _ -S server myecho ::: work; + echo "OK ^^^^^^^^^^^^^^^^^ if no myecho" >/dev/stderr; +_EOF + ) + ssh -tt csh@lo "$myscript" +} + # Test env_parallel: # + for each shell # + remote, locally diff --git a/testsuite/wanted-results/parallel-local-ssh7 b/testsuite/wanted-results/parallel-local-ssh7 index eb9aa1ad..262ddad2 100644 --- a/testsuite/wanted-results/parallel-local-ssh7 +++ b/testsuite/wanted-results/parallel-local-ssh7 @@ -93,6 +93,24 @@ par_csh_man too par_csh_man arrays par_csh_man work, par_csh_man too +par_csh_underscore ### csh +par_csh_underscore ### Testing of --env _ +par_csh_underscore #: Command not found. +par_csh_underscore variables and arrays in aliases work +par_csh_underscore variables and arrays in aliases work +par_csh_underscore variables and arrays in aliases work +par_csh_underscore variables and arrays in aliases work +par_csh_underscore variables and arrays in aliases work +par_csh_underscore variables and arrays in aliases work +par_csh_underscore variables and arrays in aliases work +par_csh_underscore variables and arrays in aliases work +par_csh_underscore and arrays in aliases work +par_csh_underscore and arrays in aliases work +par_csh_underscore and arrays in aliases work +par_csh_underscore OK ^^^^^^^^^^^^^^^^^ if no myecho +par_csh_underscore and arrays in aliases work +par_csh_underscore OK ^^^^^^^^^^^^^^^^^ if no myecho +par_csh_underscore Connection to lo closed. par_fish_funky env_parallel: Warning: ASCII value 1 in variables is not supported par_fish_funky env_parallel: Warning: ASCII value 1 in variables is not supported par_fish_funky 3 arg alias_works From 3ba7a4646d3c8d0f1da7cead60e611fffaa7f47b Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Mon, 11 Jul 2016 03:50:52 +0200 Subject: [PATCH 11/12] testsuite: --env for csh. --- src/env_parallel.pod | 6 ++++++ testsuite/tests-to-run/parallel-local-ssh7.sh | 12 ++++++------ testsuite/wanted-results/parallel-local-ssh7 | 10 ++++++++++ 3 files changed, 22 insertions(+), 6 deletions(-) diff --git a/src/env_parallel.pod b/src/env_parallel.pod index 2196e581..e7447625 100644 --- a/src/env_parallel.pod +++ b/src/env_parallel.pod @@ -296,6 +296,8 @@ E.g. by doing: alias myecho echo env_parallel myecho ::: test env_parallel -S server myecho ::: test + env_parallel --env myecho myecho ::: work + env_parallel --env myecho -S server myecho ::: work =item functions @@ -306,6 +308,8 @@ Not supported by B. set myvar=test env_parallel echo "\$myvar" ::: test env_parallel -S csh@server echo "\$myvar" ::: test + env_parallel --env myvar echo '$myvar' ::: work + env_parallel --env myvar -S server echo '$myvar' ::: work =item arrays with no special chars @@ -313,6 +317,8 @@ Not supported by B. set myarray=(foo bar baz) env_parallel echo "\${myarray\[\{\}\]}" ::: 1 2 3 env_parallel -S csh@server echo "\${myarray\[\{\}\]}" ::: 1 2 3 + env_parallel -k --env myarray echo \$'{myarray[{}]}' ::: 1 2 3 + env_parallel -k --env myarray -S server echo \$'{myarray[{}]}' ::: 1 2 3 =back diff --git a/testsuite/tests-to-run/parallel-local-ssh7.sh b/testsuite/tests-to-run/parallel-local-ssh7.sh index 5d6e3504..87cf9066 100755 --- a/testsuite/tests-to-run/parallel-local-ssh7.sh +++ b/testsuite/tests-to-run/parallel-local-ssh7.sh @@ -148,22 +148,22 @@ par_csh_man() { alias myecho 'echo aliases' env_parallel myecho ::: work env_parallel -S server myecho ::: work - #env_parallel --env myecho myecho ::: work - #env_parallel --env myecho -S server myecho ::: work + env_parallel --env myecho myecho ::: work + env_parallel --env myecho -S server myecho ::: work # Functions not supported set myvar=variables env_parallel echo '$myvar' ::: work env_parallel -S server echo '$myvar' ::: work - #env_parallel --env myvar echo '$myvar' ::: work - #env_parallel --env myvar -S server echo '$myvar' ::: work + env_parallel --env myvar echo '$myvar' ::: work + env_parallel --env myvar -S server echo '$myvar' ::: work set myarray=(arrays work, too) env_parallel -k echo \$'{myarray[{}]}' ::: 1 2 3 env_parallel -k -S server echo \$'{myarray[{}]}' ::: 1 2 3 - #env_parallel -k --env myarray echo \$'{myarray[{}]}' ::: 1 2 3 - #env_parallel -k --env myarray -S server echo \$'{myarray[{}]}' ::: 1 2 3 + env_parallel -k --env myarray echo \$'{myarray[{}]}' ::: 1 2 3 + env_parallel -k --env myarray -S server echo \$'{myarray[{}]}' ::: 1 2 3 _EOF ) diff --git a/testsuite/wanted-results/parallel-local-ssh7 b/testsuite/wanted-results/parallel-local-ssh7 index 262ddad2..e0dd9250 100644 --- a/testsuite/wanted-results/parallel-local-ssh7 +++ b/testsuite/wanted-results/parallel-local-ssh7 @@ -85,8 +85,18 @@ par_csh_man ### csh par_csh_man ### From man env_parallel par_csh_man aliases work par_csh_man aliases work +par_csh_man aliases work +par_csh_man aliases work par_csh_man variables work par_csh_man variables work +par_csh_man variables work +par_csh_man variables work +par_csh_man arrays +par_csh_man work, +par_csh_man too +par_csh_man arrays +par_csh_man work, +par_csh_man too par_csh_man arrays par_csh_man work, par_csh_man too From 68a48146ea3ba72ba150e567d59a5ed5117451d2 Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Thu, 14 Jul 2016 09:35:39 +0200 Subject: [PATCH 12/12] env_parallel: --env _ supported in (t)csh and pdksh. --- src/env_parallel.bash | 38 ++- src/env_parallel.csh | 66 ++++- src/env_parallel.fish | 40 ++- src/env_parallel.ksh | 35 +-- src/env_parallel.pdksh | 91 ++++++- src/env_parallel.pod | 64 +++-- src/env_parallel.tcsh | 10 +- src/env_parallel.zsh | 35 ++- testsuite/tests-to-run/parallel-local-ssh7.sh | 227 +++++++++++++++++- testsuite/wanted-results/parallel-local-ssh7 | 175 ++++++++++++-- 10 files changed, 653 insertions(+), 128 deletions(-) diff --git a/src/env_parallel.bash b/src/env_parallel.bash index 615738b4..5e251674 100755 --- a/src/env_parallel.bash +++ b/src/env_parallel.bash @@ -33,7 +33,8 @@ env_parallel() { # and convert a b c to (a|b|c) # If --env not set: Match everything (.*) local _grep_REGEXP="$( - perl -e 'for(@ARGV){ + perl -e ' + for(@ARGV){ /^_$/ and $next_is_env = 0; $next_is_env and push @envvar, split/,/, $_; $next_is_env = /^--env$/; @@ -44,18 +45,27 @@ env_parallel() { )" # Deal with --env _ local _ignore_UNDERSCORE="$( - perl -e 'for(@ARGV){ + perl -e ' + for(@ARGV){ $next_is_env and push @envvar, split/,/, $_; $next_is_env=/^--env$/; } - $underscore = grep { /^_$/ } @envvar; - print $underscore ? "grep -vf $ENV{HOME}/.parallel/ignored_vars" : "cat"; + if(grep { /^_$/ } @envvar) { + if(not open(IN, "<", "$ENV{HOME}/.parallel/ignored_vars")) { + print STDERR "parallel: Error: ", + "Run \"parallel --record-env\" in a clean environment first.\n"; + } else { + chomp(@ignored_vars = ); + $vars = join "|",map { quotemeta $_ } @ignored_vars; + print $vars ? "($vars)" : "(,,nO,,VaRs,,)"; + } + } ' -- "$@" )" # Grep alias names local _alias_NAMES="$(compgen -a | - egrep "^${_grep_REGEXP}\$" | $_ignore_UNDERSCORE)" + grep -E "^$_grep_REGEXP"\$ | grep -vE "^$_ignore_UNDERSCORE"\$ )" local _list_alias_BODIES="alias $_alias_NAMES" if [[ "$_alias_NAMES" = "" ]] ; then # no aliases selected @@ -65,7 +75,7 @@ env_parallel() { # Grep function names local _function_NAMES="$(compgen -A function | - egrep "^${_grep_REGEXP}\$" | $_ignore_UNDERSCORE)" + grep -E "^$_grep_REGEXP"\$ | grep -vE "^$_ignore_UNDERSCORE"\$ )" local _list_function_BODIES="typeset -f $_function_NAMES" if [[ "$_function_NAMES" = "" ]] ; then # no functions selected @@ -75,7 +85,7 @@ env_parallel() { # Grep variable names local _variable_NAMES="$(compgen -A variable | - egrep "^${_grep_REGEXP}\$" | $_ignore_UNDERSCORE | + grep -E "^$_grep_REGEXP"\$ | grep -vE "^$_ignore_UNDERSCORE"\$ | grep -vFf <(readonly) | egrep -v '^(BASHOPTS|BASHPID|EUID|GROUPS|FUNCNAME|DIRSTACK|_|PIPESTATUS|PPID|SHELLOPTS|UID|USERNAME|BASH_[A-Z_]+)$')" local _list_variable_VALUES="typeset -p $_variable_NAMES" @@ -101,17 +111,3 @@ env_parallel() { `which parallel` "$@"; unset PARALLEL_ENV; } - -# Supports env of 127375 bytes -# -# _env_parallel() { -# # Saving to a tempfile -# export PARALLEL_ENV=`tempfile`; -# (echo "shopt -s expand_aliases 2>/dev/null"; alias;typeset -p | -# grep -vFf <(readonly) | -# grep -v 'declare .. (GROUPS|FUNCNAME|DIRSTACK|_|PIPESTATUS|USERNAME|BASH_[A-Z_]+) '; -# typeset -f) > $PARALLEL_ENV -# `which parallel` "$@"; -# rm "$PARALLEL_ENV" -# unset PARALLEL_ENV; -# } diff --git a/src/env_parallel.csh b/src/env_parallel.csh index 1e555b0b..d8fd3bda 100755 --- a/src/env_parallel.csh +++ b/src/env_parallel.csh @@ -29,28 +29,69 @@ if ("`alias env_parallel`" == '') then # Activate alias alias env_parallel 'setenv PARALLEL "\!*"; source `which env_parallel.csh`' else + # Get the --env variables if set + # --env _ should be ignored + # and convert a b c to (a|b|c) + # If --env not set: Match everything (.*) + set _tMpscRIpt=`tempfile` + cat <<'EOF' > $_tMpscRIpt + #!/usr/bin/perl + + for(@ARGV){ + /^_$/ and $next_is_env = 0; + $next_is_env and push @envvar, split/,/, $_; + $next_is_env = /^--env$/; + } + $vars = join "|",map { quotemeta $_ } @envvar; + print $vars ? "($vars)" : "(.*)"; +'EOF' + set _grep_REGEXP="`perl $_tMpscRIpt -- $PARALLEL`" + + # Deal with --env _ + cat <<'EOF' > $_tMpscRIpt + #!/usr/bin/perl + + for(@ARGV){ + $next_is_env and push @envvar, split/,/, $_; + $next_is_env=/^--env$/; + } + if(grep { /^_$/ } @envvar) { + if(not open(IN, "<", "$ENV{HOME}/.parallel/ignored_vars")) { + print STDERR "parallel: Error: ", + "Run 'parallel --record-env' in a clean environment first.\n"; + } else { + chomp(@ignored_vars = ); + $vars = join "|",map { quotemeta $_ } @ignored_vars; + print $vars ? "($vars)" : "(,,nO,,VaRs,,)"; + } + } +'EOF' + set _ignore_UNDERSCORE="`perl $_tMpscRIpt -- $PARALLEL`" + rm $_tMpscRIpt + # Get the scalar and array variable names - set _vARnAmES=(`set | awk -e '{print $1}' |grep -v prompt2`) - + set _vARnAmES=(`set | awk -e '{print $1}' |grep -vE '^(#|_|killring|prompt2|command)$' | grep -E "^$_grep_REGEXP"\$ | grep -vE "^$_ignore_UNDERSCORE"\$`) + # Make a tmpfile for the variable definitions set _tMpvARfILe=`tempfile` # Make a tmpfile for the variable definitions + alias set _tMpaLLfILe=`tempfile` foreach _vARnAmE ($_vARnAmES); - # if $?myvar && $#myvar <= 1 echo scalar_myvar=$var - eval if'($?'$_vARnAmE' && ${#'$_vARnAmE'} <= 1) echo scalar_'$_vARnAmE'="$'$_vARnAmE'"' >> $_tMpvARfILe; - # if $?myvar && $#myvar > 1 echo array_myvar=$var - eval if'($?'$_vARnAmE' && ${#'$_vARnAmE'} > 1) echo array_'$_vARnAmE'="$'$_vARnAmE'"' >> $_tMpvARfILe; + # if not defined: next + eval if'(! $?'$_vARnAmE') continue' + # if $#myvar <= 1 echo scalar_myvar=$var + eval if'(${#'$_vARnAmE'} <= 1) echo scalar_'$_vARnAmE'='\"\$$_vARnAmE\" >> $_tMpvARfILe; + # if $#myvar > 1 echo array_myvar=$var + eval if'(${#'$_vARnAmE'} > 1) echo array_'$_vARnAmE'="$'$_vARnAmE'"' >> $_tMpvARfILe; end - + unset _vARnAmE _vARnAmES # shell quote variables (--plain needed due to $PARALLEL abuse) # Convert 'scalar_myvar=...' to 'set myvar=...' # Convert 'array_myvar=...' to 'set array=(...)' cat $_tMpvARfILe | parallel --plain --shellquote | perl -pe 's/^scalar_(\S+).=/set $1=/ or s/^array_(\S+).=(.*)/set $1=($2)/ && s/\\ / /g;' > $_tMpaLLfILe - # Cleanup - rm $_tMpvARfILe; unset _tMpvARfILe _vARnAmE _vARnAmES + rm $_tMpvARfILe; unset _tMpvARfILe # ALIAS TO EXPORT ALIASES: @@ -61,8 +102,6 @@ else # Quoted: s/\\047/\\047\\042\\047\\042\\047/g\; # Remove () from second column -# s/^(\S+)(\s+)\((.*)\)/\1\2\3/ -# \047 => ' # s/^(\S+)(\s+)\((.*)\)/\1\2\3/; # Quoted: s/\^\(\\S+\)\(\\s+\)\\\(\(.\*\)\\\)/\\1\\2\\3/\; @@ -79,7 +118,10 @@ else # Prepend with "\nalias " # s/^/\001alias /; # Quoted: s/\^/\\001alias\ /\; - alias | perl -pe s/\\047/\\047\\042\\047\\042\\047/g\;s/\^\(\\S+\)\(\\s+\)\\\(\(.\*\)\\\)/\\1\\2\\3/\;s/\^\(\\S+\)\(\\s+\)\(.\*\)/\\1\\2\\047\\3\\047/\;s/\^/\\001alias\ /\;s/\\\!/\\\\\\\!/g >> $_tMpaLLfILe + alias | \ + grep -E "^$_grep_REGEXP" | \ + grep -vE "^$_ignore_UNDERSCORE""[^_a-zA-Z]" | \ + perl -pe s/\\047/\\047\\042\\047\\042\\047/g\;s/\^\(\\S+\)\(\\s+\)\\\(\(.\*\)\\\)/\\1\\2\\3/\;s/\^\(\\S+\)\(\\s+\)\(.\*\)/\\1\\2\\047\\3\\047/\;s/\^/\\001alias\ /\;s/\\\!/\\\\\\\!/g >> $_tMpaLLfILe setenv PARALLEL_ENV "`cat $_tMpaLLfILe; rm $_tMpaLLfILe`"; unset _tMpaLLfILe; diff --git a/src/env_parallel.fish b/src/env_parallel.fish index f881cc49..078501d8 100755 --- a/src/env_parallel.fish +++ b/src/env_parallel.fish @@ -44,10 +44,44 @@ function env_parallel # env_parallel.fish setenv PARALLEL_ENV ( begin; + set _grep_REGEXP ( + begin; + perl -e 'for(@ARGV){ + /^_$/ and $next_is_env = 0; + $next_is_env and push @envvar, split/,/, $_; + $next_is_env = /^--env$/; + } + $vars = join "|",map { quotemeta $_ } @envvar; + print $vars ? "($vars)" : "(.*)"; + ' -- $argv; + end; + ) + # Deal with --env _ + set _ignore_UNDERSCORE ( + begin; + perl -e ' + for(@ARGV){ + $next_is_env and push @envvar, split/,/, $_; + $next_is_env=/^--env$/; + } + if(grep { /^_$/ } @envvar) { + if(not open(IN, "<", "$ENV{HOME}/.parallel/ignored_vars")) { + print STDERR "parallel: Error: ", + "Run \"parallel --record-env\" in a clean environment first.\n"; + } else { + chomp(@ignored_vars = ); + $vars = join "|",map { quotemeta $_ } @ignored_vars; + print $vars ? "($vars)" : "(nO,VaRs)"; + } + } + ' -- $argv; + end; + ) + # Export function definitions - functions -n | perl -pe 's/,/\n/g' | while read d; functions $d; end; + functions -n | perl -pe 's/,/\n/g' | grep -E "^$_grep_REGEXP"\$ | grep -vE "^$_ignore_UNDERSCORE"\$ | while read d; functions $d; end; # Convert scalar vars to fish \XX quoting - eval (set -L | perl -ne 'chomp; + eval (set -L | grep -E "^$_grep_REGEXP " | grep -vE "^$_ignore_UNDERSCORE " | perl -ne 'chomp; ($name,$val)=split(/ /,$_,2); $name=~/^(HOME|USER|COLUMNS|FISH_VERSION|LINES|PWD|SHLVL|_|history|status|version)$/ and next; if($val=~/^'"'"'/) { next; } @@ -55,7 +89,7 @@ function env_parallel ') # Generate commands to set scalar variables begin; - for v in (set -n); + for v in (set -n | grep -E "^$_grep_REGEXP\$" | grep -vE "^$_ignore_UNDERSCORE\$"); # Separate variables with the string: \000 eval "for i in \$$v; echo -n $v \$i; diff --git a/src/env_parallel.ksh b/src/env_parallel.ksh index db238ecf..3d78dbf2 100755 --- a/src/env_parallel.ksh +++ b/src/env_parallel.ksh @@ -25,8 +25,6 @@ # or write to the Free Software Foundation, Inc., 51 Franklin St, # Fifth Floor, Boston, MA 02110-1301 USA -# Supports env of 127426 bytes - env_parallel() { # env_parallel.ksh @@ -35,10 +33,11 @@ env_parallel() { # and convert a b c to (a|b|c) # If --env not set: Match everything (.*) _grep_REGEXP="$( - perl -e 'for(@ARGV){ + perl -e ' + for(@ARGV){ /^_$/ and $next_is_env = 0; $next_is_env and push @envvar, split/,/, $_; - $next_is_env=/^--env$/; + $next_is_env = /^--env$/; } $vars = join "|",map { quotemeta $_ } @envvar; print $vars ? "($vars)" : "(.*)"; @@ -46,18 +45,27 @@ env_parallel() { )" # Deal with --env _ _ignore_UNDERSCORE="$( - perl -e 'for(@ARGV){ + perl -e ' + for(@ARGV){ $next_is_env and push @envvar, split/,/, $_; $next_is_env=/^--env$/; } - $underscore = grep { /^_$/ } @envvar; - print $underscore ? "grep -vf $ENV{HOME}/.parallel/ignored_vars" : "cat"; + if(grep { /^_$/ } @envvar) { + if(not open(IN, "<", "$ENV{HOME}/.parallel/ignored_vars")) { + print STDERR "parallel: Error: ", + "Run \"parallel --record-env\" in a clean environment first.\n"; + } else { + chomp(@ignored_vars = ); + $vars = join "|",map { quotemeta $_ } @ignored_vars; + print $vars ? "($vars)" : "(,,nO,,VaRs,,)"; + } + } ' -- "$@" )" # Grep alias names _alias_NAMES="$(alias | perl -pe 's/=.*//' | - egrep "^${_grep_REGEXP}\$" | $_ignore_UNDERSCORE)" + grep -E "^$_grep_REGEXP"\$ | grep -vE "^$_ignore_UNDERSCORE"\$ )" _list_alias_BODIES="alias $_alias_NAMES | perl -pe 's/^/alias /'" if [[ "$_alias_NAMES" = "" ]] ; then # no aliases selected @@ -67,7 +75,7 @@ env_parallel() { # Grep function names _function_NAMES="$(typeset +p -f | perl -pe 's/\(\).*//' | - egrep "^${_grep_REGEXP}\$" | $_ignore_UNDERSCORE)" + grep -E "^$_grep_REGEXP"\$ | grep -vE "^$_ignore_UNDERSCORE"\$ )" _list_function_BODIES="typeset -f $_function_NAMES" if [[ "$_function_NAMES" = "" ]] ; then # no functions selected @@ -77,7 +85,7 @@ env_parallel() { # Grep variable names _variable_NAMES="$(typeset +p | perl -pe 's/^typeset .. //' | - egrep "^${_grep_REGEXP}\$" | $_ignore_UNDERSCORE | + grep -E "^$_grep_REGEXP"\$ | grep -vE "^$_ignore_UNDERSCORE"\$ | egrep -v '^(PIPESTATUS)$')" _list_variable_VALUES="typeset -p $_variable_NAMES" if [[ "$_variable_NAMES" = "" ]] ; then @@ -97,10 +105,3 @@ env_parallel() { `which parallel` "$@"; unset PARALLEL_ENV; } - -# _env_parallel() { -# # env_parallel.ksh -# export PARALLEL_ENV="$(alias | perl -pe 's/^/alias /';typeset -p|egrep -v 'typeset( -i)? -r|PIPESTATUS';typeset -f)"; -# `which parallel` "$@"; -# unset PARALLEL_ENV; -# } diff --git a/src/env_parallel.pdksh b/src/env_parallel.pdksh index c82f1978..4a92ef8e 100755 --- a/src/env_parallel.pdksh +++ b/src/env_parallel.pdksh @@ -1,8 +1,8 @@ #!/bin/pdksh -# This file must be sourced in ksh: +# This file must be sourced in pdksh: # -# source `which env_parallel.ksh` +# source `which env_parallel.pdksh` # # after which 'env_parallel' works # @@ -26,8 +26,87 @@ # Fifth Floor, Boston, MA 02110-1301 USA env_parallel() { - # env_parallel.pdksh - export PARALLEL_ENV="$(alias | perl -pe 's/^/alias /';typeset -p|egrep -v 'typeset( -i)? -r|PIPESTATUS';typeset -f)"; - `which parallel` "$@"; - unset PARALLEL_ENV; + # env_parallel.pdksh + + # Get the --env variables if set + # --env _ should be ignored + # and convert a b c to (a|b|c) + # If --env not set: Match everything (.*) + _grep_REGEXP="$( + perl -e ' + for(@ARGV){ + /^_$/ and $next_is_env = 0; + $next_is_env and push @envvar, split/,/, $_; + $next_is_env = /^--env$/; + } + $vars = join "|",map { quotemeta $_ } @envvar; + print $vars ? "($vars)" : "(.*)"; + ' -- "$@" + )" + # Deal with --env _ + _ignore_UNDERSCORE="$( + perl -e ' + for(@ARGV){ + $next_is_env and push @envvar, split/,/, $_; + $next_is_env=/^--env$/; + } + if(grep { /^_$/ } @envvar) { + if(not open(IN, "<", "$ENV{HOME}/.parallel/ignored_vars")) { + print STDERR "parallel: Error: ", + "Run \"parallel --record-env\" in a clean environment first.\n"; + } else { + chomp(@ignored_vars = ); + $vars = join "|",map { quotemeta $_ } @ignored_vars; + print $vars ? "($vars)" : "(,,nO,,VaRs,,)"; + } + } + ' -- "$@" + )" + + # Grep alias names + _alias_NAMES="$(alias | perl -pe 's/=.*//' | + grep -E "^$_grep_REGEXP"\$ | grep -vE "^$_ignore_UNDERSCORE"\$ )" + _list_alias_BODIES="alias $_alias_NAMES | perl -pe 's/^/alias /'" + if [[ "$_alias_NAMES" = "" ]] ; then + # no aliases selected + _list_alias_BODIES="true" + fi + unset _alias_NAMES + + # Grep function names + _function_NAMES=$(typeset +p -f | perl -pe 's/\(\).*//' | + grep -E "^$_grep_REGEXP"\$ | grep -vE "^$_ignore_UNDERSCORE"\$ ) + _list_function_BODIES="typeset -f $_function_NAMES" + if [[ "$_function_NAMES" = "" ]] ; then + # no functions selected + _list_function_BODIES="true" + fi + unset _function_NAMES + + # Grep variable names + _tmp_READONLY="$(mktemp)" + readonly > "$_tmp_READONLY" + _variable_NAMES="$(typeset | perl -pe 's/^(type)?set( -.)* //' | + grep -E "^$_grep_REGEXP"\$ | grep -vE "^$_ignore_UNDERSCORE"\$ | + grep -vFf $_tmp_READONLY | + grep -Ev '^(PIPESTATUS)')" + rm $_tmp_READONLY + unset _tmp_READONLY + _list_variable_VALUES="typeset -p $_variable_NAMES" + if [[ "$_variable_NAMES" = "" ]] ; then + # no variables selected + _list_variable_VALUES="true" + fi + unset _variable_NAMES + + # eval is needed for aliases - cannot explain why + export PARALLEL_ENV="$( + eval $_list_alias_BODIES; + $_list_variable_VALUES; + $_list_function_BODIES)"; + unset _list_alias_BODIES + unset _list_variable_VALUES + unset _list_function_BODIES + `which parallel` "$@"; + unset PARALLEL_ENV; } diff --git a/src/env_parallel.pod b/src/env_parallel.pod index e7447625..3985ef95 100644 --- a/src/env_parallel.pod +++ b/src/env_parallel.pod @@ -160,29 +160,37 @@ E.g. by doing: =item aliases - alias myecho=echo - env_parallel myecho ::: test - env_parallel -S server myecho ::: test + alias myecho 'echo aliases' + env_parallel myecho ::: work + env_parallel -S server myecho ::: work + env_parallel --env myecho myecho ::: work + env_parallel --env myecho -S server myecho ::: work =item functions function myfunc - echo $argv + echo functions $argv end - env_parallel myfunc ::: test - env_parallel -S server myfunc ::: test + env_parallel myfunc ::: work + env_parallel -S server myfunc ::: work + env_parallel --env myfunc myfunc ::: work + env_parallel --env myfunc -S server myfunc ::: work =item variables - set myvar test - env_parallel echo '$myvar' ::: test - env_parallel -S server echo '$myvar' ::: test + set myvar variables + env_parallel echo '$myvar' ::: work + env_parallel -S server echo '$myvar' ::: work + env_parallel --env myvar echo '$myvar' ::: work + env_parallel --env myvar -S server echo '$myvar' ::: work =item arrays - set myarray foo bar baz - env_parallel echo '$myarray[{}]' ::: 1 2 3 - env_parallel -S server echo '$myarray[{}]' ::: 1 2 3 + set myarray arrays work, too + env_parallel -k echo '$myarray[{}]' ::: 1 2 3 + env_parallel -k -S server echo '$myarray[{}]' ::: 1 2 3 + env_parallel -k --env myarray echo '$myarray[{}]' ::: 1 2 3 + env_parallel -k --env myarray -S server echo '$myarray[{}]' ::: 1 2 3 =back @@ -250,27 +258,35 @@ E.g. by doing: =item aliases - alias myecho=echo - env_parallel myecho ::: test - env_parallel -S server myecho ::: test + alias myecho="echo aliases"; + env_parallel myecho ::: work; + env_parallel -S server myecho ::: work; + env_parallel --env myecho myecho ::: work; + env_parallel --env myecho -S server myecho ::: work =item functions - myfunc() { echo $*; } - env_parallel myfunc ::: test - env_parallel -S server myfunc ::: test + myfunc() { echo functions $*; }; + env_parallel myfunc ::: work; + env_parallel -S server myfunc ::: work; + env_parallel --env myfunc myfunc ::: work; + env_parallel --env myfunc -S server myfunc ::: work =item variables - myvar=test - env_parallel echo '$myvar' ::: test - env_parallel -S server echo '$myvar' ::: test + myvar=variables; + env_parallel echo "\$myvar" ::: work; + env_parallel -S server echo "\$myvar" ::: work; + env_parallel --env myvar echo "\$myvar" ::: work; + env_parallel --env myvar -S server echo "\$myvar" ::: work =item arrays - myarray=(foo bar baz) - env_parallel echo '${myarray[{}]}' ::: 0 1 2 - env_parallel -S server echo '${myarray[{}]}' ::: 0 1 2 + myarray=(arrays work, too); + env_parallel -k echo "\${myarray[{}]}" ::: 0 1 2; + env_parallel -k -S server echo "\${myarray[{}]}" ::: 0 1 2; + env_parallel -k --env myarray echo "\${myarray[{}]}" ::: 0 1 2; + env_parallel -k --env myarray -S server echo "\${myarray[{}]}" ::: 0 1 2 =back diff --git a/src/env_parallel.tcsh b/src/env_parallel.tcsh index 8524167c..e19a79af 100755 --- a/src/env_parallel.tcsh +++ b/src/env_parallel.tcsh @@ -35,6 +35,8 @@ else # If --env not set: Match everything (.*) set _tMpscRIpt=`tempfile` cat <<'EOF' > $_tMpscRIpt + #!/usr/bin/perl + for(@ARGV){ /^_$/ and $next_is_env = 0; $next_is_env and push @envvar, split/,/, $_; @@ -43,7 +45,8 @@ else $vars = join "|",map { quotemeta $_ } @envvar; print $vars ? "($vars)" : "(.*)"; 'EOF' - set _grep_REGEXP="`perl $_tMpscRIpt -- !*`" + set _grep_REGEXP="`perl $_tMpscRIpt -- $PARALLEL`" + # Deal with --env _ cat <<'EOF' > $_tMpscRIpt #!/usr/bin/perl @@ -59,12 +62,13 @@ else } else { chomp(@ignored_vars = ); $vars = join "|",map { quotemeta $_ } @ignored_vars; - print $vars ? "($vars)" : "(nOVaRs)"; + print $vars ? "($vars)" : "(,,nO,,VaRs,,)"; } } 'EOF' - set _ignore_UNDERSCORE="`perl $_tMpscRIpt -- !*`" + set _ignore_UNDERSCORE="`perl $_tMpscRIpt -- $PARALLEL`" rm $_tMpscRIpt + # Get the scalar and array variable names set _vARnAmES=(`set | awk -e '{print $1}' |grep -vE '^(#|_|killring|prompt2|command)$' | grep -E "^$_grep_REGEXP"\$ | grep -vE "^$_ignore_UNDERSCORE"\$`) diff --git a/src/env_parallel.zsh b/src/env_parallel.zsh index bf16529e..aa8f0087 100755 --- a/src/env_parallel.zsh +++ b/src/env_parallel.zsh @@ -33,7 +33,8 @@ env_parallel() { # and convert a b c to (a|b|c) # If --env not set: Match everything (.*) _grep_REGEXP="$( - perl -e 'for(@ARGV){ + perl -e ' + for(@ARGV){ /^_$/ and $next_is_env = 0; $next_is_env and push @envvar, split/,/, $_; $next_is_env = /^--env$/; @@ -44,18 +45,27 @@ env_parallel() { )" # Deal with --env _ local _ignore_UNDERSCORE="$( - perl -e 'for(@ARGV){ + perl -e ' + for(@ARGV){ $next_is_env and push @envvar, split/,/, $_; $next_is_env=/^--env$/; } - $underscore = grep { /^_$/ } @envvar; - print $underscore ? "grep -vf $ENV{HOME}/.parallel/ignored_vars" : "cat"; + if(grep { /^_$/ } @envvar) { + if(not open(IN, "<", "$ENV{HOME}/.parallel/ignored_vars")) { + print STDERR "parallel: Error: ", + "Run \"parallel --record-env\" in a clean environment first.\n"; + } else { + chomp(@ignored_vars = ); + $vars = join "|",map { quotemeta $_ } @ignored_vars; + print $vars ? "($vars)" : "(,,nO,,VaRs,,)"; + } + } ' -- "$@" )" # Grep alias names _alias_NAMES="$(print -l ${(k)aliases} | - egrep "^${_grep_REGEXP}\$" | eval $_ignore_UNDERSCORE )" + grep -E "^$_grep_REGEXP"\$ | grep -vE "^$_ignore_UNDERSCORE"\$ )" _list_alias_BODIES="alias "$(echo $_alias_NAMES|xargs)" | perl -pe 's/^/alias /'" if [[ "$_alias_NAMES" = "" ]] ; then # no aliases selected @@ -65,7 +75,7 @@ env_parallel() { # Grep function names _function_NAMES="$(print -l ${(k)functions} | - egrep "^${_grep_REGEXP}\$" | eval $_ignore_UNDERSCORE | + grep -E "^$_grep_REGEXP\$" | grep -vE "^$_ignore_UNDERSCORE\$" | grep -v '=' )" _list_function_BODIES="typeset -f "$(echo $_function_NAMES|xargs) @@ -78,7 +88,7 @@ env_parallel() { # Grep variable names # The egrep -v is crap and should be better _variable_NAMES="$(print -l ${(k)parameters} | - egrep "^${_grep_REGEXP}\$" | eval $_ignore_UNDERSCORE | + grep -E "^$_grep_REGEXP"\$ | grep -vE "^$_ignore_UNDERSCORE"\$ | egrep -v '^([-?#!$*@_0]|zsh_eval_context|ZSH_EVAL_CONTEXT|LINENO|IFS|commands|functions|options|aliases|EUID|EGID|UID|GID)$' | egrep -v 'terminfo|funcstack|galiases|keymaps|parameters|jobdirs|dirstack|functrace|funcsourcetrace|zsh_scheduled_events|dis_aliases|dis_reswords|dis_saliases|modules|reswords|saliases|widgets|userdirs|historywords|nameddirs|termcap|dis_builtins|dis_functions|jobtexts|funcfiletrace|dis_galiases|builtins|history|jobstates' )" @@ -103,14 +113,3 @@ env_parallel() { `which parallel` "$@"; unset PARALLEL_ENV; } - -_old_env_parallel() { - # env_parallel.zsh - export PARALLEL_ENV="$(alias | perl -pe 's/^/alias /'; typeset -p | - grep -aFvf <(typeset -pr) | - egrep -iav 'ZSH_EVAL_CONTEXT|LINENO=| _=|aliases|^typeset [a-z_]+$'| - egrep -av '^(typeset -A (commands|functions|options)|typeset IFS=|..$)|cyan'; - typeset -f)"; - parallel "$@"; - unset PARALLEL_ENV; -} diff --git a/testsuite/tests-to-run/parallel-local-ssh7.sh b/testsuite/tests-to-run/parallel-local-ssh7.sh index 87cf9066..faba0297 100755 --- a/testsuite/tests-to-run/parallel-local-ssh7.sh +++ b/testsuite/tests-to-run/parallel-local-ssh7.sh @@ -108,6 +108,43 @@ _EOF ssh ksh@lo "$myscript" } +par_pdksh_man() { + echo '### pdksh' + myscript=$(cat <<'_EOF' + echo "### From man env_parallel" + + . `which env_parallel.pdksh`; + alias myecho="echo aliases"; + env_parallel myecho ::: work; + env_parallel -S server myecho ::: work; + env_parallel --env myecho myecho ::: work; + env_parallel --env myecho -S server myecho ::: work + + . `which env_parallel.pdksh`; + myfunc() { echo functions $*; }; + env_parallel myfunc ::: work; + env_parallel -S server myfunc ::: work; + env_parallel --env myfunc myfunc ::: work; + env_parallel --env myfunc -S server myfunc ::: work + + . `which env_parallel.pdksh`; + myvar=variables; + env_parallel echo "\$myvar" ::: work; + env_parallel -S server echo "\$myvar" ::: work; + env_parallel --env myvar echo "\$myvar" ::: work; + env_parallel --env myvar -S server echo "\$myvar" ::: work + + . `which env_parallel.pdksh`; + myarray=(arrays work, too); + env_parallel -k echo "\${myarray[{}]}" ::: 0 1 2; + env_parallel -k -S server echo "\${myarray[{}]}" ::: 0 1 2; + env_parallel -k --env myarray echo "\${myarray[{}]}" ::: 0 1 2; + env_parallel -k --env myarray -S server echo "\${myarray[{}]}" ::: 0 1 2 +_EOF + ) + ssh pdksh@lo "$myscript" +} + par_tcsh_man() { echo '### tcsh' myscript=$(cat <<'_EOF' @@ -143,7 +180,7 @@ par_csh_man() { myscript=$(cat <<'_EOF' echo "### From man env_parallel" - source `which env_parallel.csh`; +# source `which env_parallel.csh`; alias myecho 'echo aliases' env_parallel myecho ::: work @@ -170,6 +207,43 @@ _EOF ssh csh@lo "$myscript" } +par_fish_man() { + echo '### fish' + myscript=$(cat <<'_EOF' + echo "### From man env_parallel" + + alias myecho 'echo aliases' + env_parallel myecho ::: work + env_parallel -S server myecho ::: work + env_parallel --env myecho myecho ::: work + env_parallel --env myecho -S server myecho ::: work + + function myfunc + echo functions $argv + end + env_parallel myfunc ::: work + env_parallel -S server myfunc ::: work + env_parallel --env myfunc myfunc ::: work + env_parallel --env myfunc -S server myfunc ::: work + + set myvar variables + env_parallel echo '$myvar' ::: work + env_parallel -S server echo '$myvar' ::: work + env_parallel --env myvar echo '$myvar' ::: work + env_parallel --env myvar -S server echo '$myvar' ::: work + + set myarray arrays work, too + env_parallel -k echo '$myarray[{}]' ::: 1 2 3 + env_parallel -k -S server echo '$myarray[{}]' ::: 1 2 3 + env_parallel -k --env myarray echo '$myarray[{}]' ::: 1 2 3 + env_parallel -k --env myarray -S server echo '$myarray[{}]' ::: 1 2 3 + +_EOF + ) + ssh fish@lo "$myscript" +} + + par_bash_underscore() { echo '### bash' myscript=$(cat <<'_EOF' @@ -195,14 +269,14 @@ par_bash_underscore() { env_parallel --env _ -S server myfunc ::: work; echo myecho >> ~/.parallel/ignored_vars; env_parallel --env _ myfunc ::: work; - echo "OK if no myecho ^^^^^^^^^^^^^^^^^^^^^^^^^" >&2; + echo "OK if no myecho ^^^^^^^^^^^^^^^^^^^^^^^^^" >&2; env_parallel --env _ -S server myfunc ::: work; - echo "OK if no myecho ^^^^^^^^^^^^^^^^^^^^^^^^^" >&2; + echo "OK if no myecho ^^^^^^^^^^^^^^^^^^^^^^^^^" >&2; echo myfunc >> ~/.parallel/ignored_vars; env_parallel --env _ myfunc ::: work; - echo "OK if no myfunc ^^^^^^^^^^^^^^^^^^^^^^^^^" >&2; + echo "OK if no myfunc ^^^^^^^^^^^^^^^^^^^^^^^^^" >&2; env_parallel --env _ -S server myfunc ::: work; - echo "OK if no myfunc ^^^^^^^^^^^^^^^^^^^^^^^^^" >&2; + echo "OK if no myfunc ^^^^^^^^^^^^^^^^^^^^^^^^^" >&2; _EOF ) ssh bash@lo "$myscript" @@ -289,6 +363,45 @@ _EOF ssh ksh@lo "$myscript" } +par_pdksh_underscore() { + echo '### pdksh' + myscript=$(cat <<'_EOF' + echo "### Testing of --env _" + + . `which env_parallel.pdksh`; + env_parallel --record-env; + alias myecho="echo \$myvar aliases in"; + myfunc() { myecho ${myarray[@]} functions $*; }; + myvar="variables in"; + myarray=(and arrays in); + env_parallel myfunc ::: work; + env_parallel -S server myfunc ::: work; + env_parallel --env myfunc,myvar,myarray,myecho myfunc ::: work; + env_parallel --env myfunc,myvar,myarray,myecho -S server myfunc ::: work; + env_parallel --env _ myfunc ::: work; + env_parallel --env _ -S server myfunc ::: work; + + echo myvar >> ~/.parallel/ignored_vars; + env_parallel --env _ myfunc ::: work; + env_parallel --env _ -S server myfunc ::: work; + echo myarray >> ~/.parallel/ignored_vars; + env_parallel --env _ myfunc ::: work; + env_parallel --env _ -S server myfunc ::: work; + echo myecho >> ~/.parallel/ignored_vars; + env_parallel --env _ myfunc ::: work; + echo "OK if no myecho ^^^^^^^^^^^^^^^^^" >&2; + env_parallel --env _ -S server myfunc ::: work; + echo "OK if no myecho ^^^^^^^^^^^^^^^^^" >&2; + echo myfunc >> ~/.parallel/ignored_vars; + env_parallel --env _ myfunc ::: work; + echo "OK if no myfunc ^^^^^^^^^^^^^^^^^" >&2; + env_parallel --env _ -S server myfunc ::: work; + echo "OK if no myfunc ^^^^^^^^^^^^^^^^^" >&2; +_EOF + ) + ssh pdksh@lo "$myscript" +} + par_tcsh_underscore() { echo '### tcsh' myscript=$(cat <<'_EOF' @@ -328,7 +441,7 @@ par_csh_underscore() { myscript=$(cat <<'_EOF' echo "### Testing of --env _" -# . `which env_parallel.tcsh`; +# . `which env_parallel.csh`; env_parallel --record-env; alias myecho "echo "\$"myvar "\$'myarray'" aliases"; set myvar="variables"; @@ -357,6 +470,47 @@ _EOF ssh -tt csh@lo "$myscript" } +par_fish_underscore() { + echo '### fish' + myscript=$(cat <<'_EOF' + echo "### Testing of --env _" + +# . `which env_parallel.fish`; + env_parallel --record-env; + alias myecho="echo \$myvar aliases"; + function myfunc + myecho $myarray functions $argv + end + set myvar "variables in"; + set myarray and arrays in; + env_parallel myfunc ::: work; + env_parallel -S server myfunc ::: work; + env_parallel --env myfunc,myvar,myarray,myecho myfunc ::: work; + env_parallel --env myfunc,myvar,myarray,myecho -S server myfunc ::: work; + env_parallel --env _ myfunc ::: work; + env_parallel --env _ -S server myfunc ::: work; + + echo myvar >> ~/.parallel/ignored_vars; + env_parallel --env _ myfunc ::: work; + env_parallel --env _ -S server myfunc ::: work; + echo myarray >> ~/.parallel/ignored_vars; + env_parallel --env _ myfunc ::: work; + env_parallel --env _ -S server myfunc ::: work; + echo myecho >> ~/.parallel/ignored_vars; + env_parallel --env _ myfunc ::: work; + echo "OK if ^^^^^^^^^^^^^^^^^ no myecho" >&2; + env_parallel --env _ -S server myfunc ::: work; + echo "OK if ^^^^^^^^^^^^^^^^^ no myecho" >&2; + echo myfunc >> ~/.parallel/ignored_vars; + env_parallel --env _ myfunc ::: work; + echo "OK if ^^^^^^^^^^^^^^^^^ no myfunc" >&2; + env_parallel --env _ -S server myfunc ::: work; + echo "OK if ^^^^^^^^^^^^^^^^^ no myfunc" >&2; +_EOF + ) + ssh fish@lo "$myscript" +} + # Test env_parallel: # + for each shell # + remote, locally @@ -448,6 +602,35 @@ _EOF ssh ksh@lo "$myscript" } +par_pdksh_funky() { + myscript=$(cat <<'_EOF' + . `which env_parallel.pdksh`; + + myvar="myvar works" + funky=$(perl -e "print pack \"c*\", 1..255") + myarray=("" array_val2 3 "" 5 " space 6 ") + assocarr[a]=assoc_val_a + assocarr[b]=assoc_val_b + alias alias_echo="echo 3 arg"; + func_echo() { + echo $*; + echo "$myvar" + echo "${myarray[5]}" + echo ${assocarr[a]} + echo Funky-"$funky"-funky + } + + env_parallel alias_echo ::: alias_works + env_parallel func_echo ::: function_works + env_parallel -S lo alias_echo ::: alias_works_over_ssh + env_parallel -S lo func_echo ::: function_works_over_ssh + echo + echo "$funky" | parallel --shellquote +_EOF + ) + ssh pdksh@lo "$myscript" +} + par_fish_funky() { myscript=$(cat <<'_EOF' set myvar "myvar works" @@ -524,6 +707,38 @@ _EOF ssh csh@lo "$myscript" } +par_tcsh_funky() { + myscript=$(cat <<'_EOF' + set myvar = "myvar works" + set funky = "`perl -e 'print pack q(c*), 2..255'`" + set myarray = ('' 'array_val2' '3' '' '5' ' space 6 ') + # declare -A assocarr + # assocarr[a]=assoc_val_a + # assocarr[b]=assoc_val_b + alias alias_echo echo 3 arg; + alias alias_echo_var 'echo $argv; echo "$myvar"; echo "${myarray[4]} special chars problem"; echo Funky-"$funky"-funky' + + # function func_echo + # echo $argv; + # echo $myvar; + # echo ${myarray[2]} + # #echo ${assocarr[a]} + # echo Funky-"$funky"-funky + # end + + env_parallel alias_echo ::: alias_works + env_parallel alias_echo_var ::: alias_var_works + env_parallel func_echo ::: function_does_not_work + env_parallel -S tcsh@lo alias_echo ::: alias_works_over_ssh + env_parallel -S tcsh@lo alias_echo_var ::: alias_var_works_over_ssh + env_parallel -S tcsh@lo func_echo ::: function_does_not_work_over_ssh + echo + echo "$funky" | parallel --shellquote +_EOF + ) + ssh tcsh@lo "$myscript" +} + export -f $(compgen -A function | grep par_) # Tested with -j1..8 diff --git a/testsuite/wanted-results/parallel-local-ssh7 b/testsuite/wanted-results/parallel-local-ssh7 index e0dd9250..b00442e6 100644 --- a/testsuite/wanted-results/parallel-local-ssh7 +++ b/testsuite/wanted-results/parallel-local-ssh7 @@ -62,14 +62,14 @@ par_bash_underscore aliases in and arrays in functions work par_bash_underscore aliases in and arrays in functions work par_bash_underscore aliases in functions work par_bash_underscore aliases in functions work -par_bash_underscore /bin/bash: line 64: myecho: command not found -par_bash_underscore OK if no myecho ^^^^^^^^^^^^^^^^^^^^^^^^^ -par_bash_underscore /bin/bash: line 64: myecho: command not found -par_bash_underscore OK if no myecho ^^^^^^^^^^^^^^^^^^^^^^^^^ -par_bash_underscore /bin/bash: line 62: myfunc: command not found -par_bash_underscore OK if no myfunc ^^^^^^^^^^^^^^^^^^^^^^^^^ -par_bash_underscore /bin/bash: line 62: myfunc: command not found -par_bash_underscore OK if no myfunc ^^^^^^^^^^^^^^^^^^^^^^^^^ +par_bash_underscore /bin/bash: line 139: myecho: command not found +par_bash_underscore OK if no myecho ^^^^^^^^^^^^^^^^^^^^^^^^^ +par_bash_underscore /bin/bash: line 139: myecho: command not found +par_bash_underscore OK if no myecho ^^^^^^^^^^^^^^^^^^^^^^^^^ +par_bash_underscore /bin/bash: line 136: myfunc: command not found +par_bash_underscore OK if no myfunc ^^^^^^^^^^^^^^^^^^^^^^^^^ +par_bash_underscore /bin/bash: line 136: myfunc: command not found +par_bash_underscore OK if no myfunc ^^^^^^^^^^^^^^^^^^^^^^^^^ par_csh_funky 3 arg alias_works par_csh_funky myvar works par_csh_funky space special chars problem @@ -112,13 +112,13 @@ par_csh_underscore variables and arrays in aliases work par_csh_underscore variables and arrays in aliases work par_csh_underscore variables and arrays in aliases work par_csh_underscore variables and arrays in aliases work -par_csh_underscore variables and arrays in aliases work -par_csh_underscore variables and arrays in aliases work -par_csh_underscore and arrays in aliases work -par_csh_underscore and arrays in aliases work -par_csh_underscore and arrays in aliases work +par_csh_underscore myvar: Undefined variable. +par_csh_underscore myvar: Undefined variable. +par_csh_underscore myarray: Undefined variable. +par_csh_underscore myarray: Undefined variable. +par_csh_underscore myecho: Command not found. par_csh_underscore OK ^^^^^^^^^^^^^^^^^ if no myecho -par_csh_underscore and arrays in aliases work +par_csh_underscore myecho: Command not found. par_csh_underscore OK ^^^^^^^^^^^^^^^^^ if no myecho par_csh_underscore Connection to lo closed. par_fish_funky env_parallel: Warning: ASCII value 1 in variables is not supported @@ -157,6 +157,68 @@ par_fish_funky par_fish_funky par_fish_funky par_fish_funky \\\\\\\\ \ \ \ \ \\\\\\\\\\\\\\ \ \!\"\#\$%\&\'\(\)\*+,-./0123456789:\;\<\=\>\?@ABCDEFGHIJKLMNOPQRSTUVWXYZ\[\\\]\^_\`abcdefghijklmnopqrstuvwxyz\{\|\}\~\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\ +par_fish_man ### fish +par_fish_man ### From man env_parallel +par_fish_man aliases work +par_fish_man aliases work +par_fish_man aliases work +par_fish_man aliases work +par_fish_man functions work +par_fish_man functions work +par_fish_man functions work +par_fish_man functions work +par_fish_man variables work +par_fish_man variables work +par_fish_man variables work +par_fish_man variables work +par_fish_man arrays +par_fish_man work, +par_fish_man too +par_fish_man arrays +par_fish_man work, +par_fish_man too +par_fish_man arrays +par_fish_man work, +par_fish_man too +par_fish_man arrays +par_fish_man work, +par_fish_man too +par_fish_underscore ### fish +par_fish_underscore ### Testing of --env _ +par_fish_underscore variables in aliases and arrays in functions work +par_fish_underscore variables in aliases and arrays in functions work +par_fish_underscore variables in aliases and arrays in functions work +par_fish_underscore variables in aliases and arrays in functions work +par_fish_underscore variables in aliases and arrays in functions work +par_fish_underscore variables in aliases and arrays in functions work +par_fish_underscore aliases and arrays in functions work +par_fish_underscore aliases and arrays in functions work +par_fish_underscore aliases functions work +par_fish_underscore aliases functions work +par_fish_underscore fish: Unknown command “myecho” +par_fish_underscore Standard input: myecho $myarray functions $argv +par_fish_underscore ^ +par_fish_underscore in function “myfunc”, +par_fish_underscore called on standard input, +par_fish_underscore with parameter list “work” +par_fish_underscore +par_fish_underscore OK if ^^^^^^^^^^^^^^^^^ no myecho +par_fish_underscore fish: Unknown command “myecho” +par_fish_underscore Standard input: myecho $myarray functions $argv +par_fish_underscore ^ +par_fish_underscore in function “myfunc”, +par_fish_underscore called on standard input, +par_fish_underscore with parameter list “work” +par_fish_underscore +par_fish_underscore OK if ^^^^^^^^^^^^^^^^^ no myecho +par_fish_underscore fish: Unknown command “myfunc” +par_fish_underscore Standard input: myfunc work +par_fish_underscore ^ +par_fish_underscore OK if ^^^^^^^^^^^^^^^^^ no myfunc +par_fish_underscore fish: Unknown command “myfunc” +par_fish_underscore Standard input: myfunc work +par_fish_underscore ^ +par_fish_underscore OK if ^^^^^^^^^^^^^^^^^ no myfunc par_ksh_funky 3 arg alias_works par_ksh_funky function_works par_ksh_funky myvar works @@ -212,14 +274,91 @@ par_ksh_underscore aliases in and arrays in functions work par_ksh_underscore aliases in and arrays in functions work par_ksh_underscore aliases in functions work par_ksh_underscore aliases in functions work -par_ksh_underscore /usr/bin/ksh[100]: myecho: not found [No such file or directory] +par_ksh_underscore /usr/bin/ksh[193]: myecho: not found [No such file or directory] par_ksh_underscore OK if no myecho ^^^^^^^^^^^^^^^^^ -par_ksh_underscore /usr/bin/ksh[100]: myecho: not found [No such file or directory] +par_ksh_underscore /usr/bin/ksh[193]: myecho: not found [No such file or directory] par_ksh_underscore OK if no myecho ^^^^^^^^^^^^^^^^^ -par_ksh_underscore /usr/bin/ksh: line 100: myfunc: not found +par_ksh_underscore /usr/bin/ksh: line 193: myfunc: not found par_ksh_underscore OK if no myfunc ^^^^^^^^^^^^^^^^^ -par_ksh_underscore /usr/bin/ksh: line 100: myfunc: not found +par_ksh_underscore /usr/bin/ksh: line 193: myfunc: not found par_ksh_underscore OK if no myfunc ^^^^^^^^^^^^^^^^^ +par_pdksh_funky 3 arg alias_works +par_pdksh_funky function_works +par_pdksh_funky myvar works +par_pdksh_funky space 6 +par_pdksh_funky assoc_val_b +par_pdksh_funky Funky- +par_pdksh_funky  !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~-funky +par_pdksh_funky 3 arg alias_works_over_ssh +par_pdksh_funky function_works_over_ssh +par_pdksh_funky myvar works +par_pdksh_funky space 6 +par_pdksh_funky assoc_val_b +par_pdksh_funky Funky- +par_pdksh_funky  !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~-funky +par_pdksh_funky +par_pdksh_funky \\\\\\\\ +par_pdksh_funky \ \ \ \\\\\\\\\\\\\\ \!\"\#\$%\&\'\(\)\*+,-./0123456789:\;\<\=\>\?@ABCDEFGHIJKLMNOPQRSTUVWXYZ\[\\\]\^_\`abcdefghijklmnopqrstuvwxyz\{\|\}\~\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\ +par_pdksh_man ### pdksh +par_pdksh_man ### From man env_parallel +par_pdksh_man aliases work +par_pdksh_man aliases work +par_pdksh_man aliases work +par_pdksh_man aliases work +par_pdksh_man functions work +par_pdksh_man functions work +par_pdksh_man functions work +par_pdksh_man functions work +par_pdksh_man variables work +par_pdksh_man variables work +par_pdksh_man variables work +par_pdksh_man variables work +par_pdksh_man arrays +par_pdksh_man work, +par_pdksh_man too +par_pdksh_man arrays +par_pdksh_man work, +par_pdksh_man too +par_pdksh_man arrays +par_pdksh_man work, +par_pdksh_man too +par_pdksh_man arrays +par_pdksh_man work, +par_pdksh_man too +par_pdksh_underscore ### pdksh +par_pdksh_underscore ### Testing of --env _ +par_pdksh_underscore variables in aliases in and arrays in functions work +par_pdksh_underscore variables in aliases in and arrays in functions work +par_pdksh_underscore variables in aliases in and arrays in functions work +par_pdksh_underscore variables in aliases in and arrays in functions work +par_pdksh_underscore variables in aliases in and arrays in functions work +par_pdksh_underscore variables in aliases in and arrays in functions work +par_pdksh_underscore aliases in and arrays in functions work +par_pdksh_underscore aliases in and arrays in functions work +par_pdksh_underscore aliases in functions work +par_pdksh_underscore aliases in functions work +par_pdksh_underscore aliases in functions work +par_pdksh_underscore OK if no myecho ^^^^^^^^^^^^^^^^^ +par_pdksh_underscore aliases in functions work +par_pdksh_underscore OK if no myecho ^^^^^^^^^^^^^^^^^ +par_pdksh_underscore /bin/pdksh: myfunc: not found +par_pdksh_underscore OK if no myfunc ^^^^^^^^^^^^^^^^^ +par_pdksh_underscore /bin/pdksh: myfunc: not found +par_pdksh_underscore OK if no myfunc ^^^^^^^^^^^^^^^^^ +par_tcsh_funky 3 arg alias_works +par_tcsh_funky +par_tcsh_funky myvar works +par_tcsh_funky space special chars problem +par_tcsh_funky Funky-  !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~-funky alias_var_works +par_tcsh_funky func_echo: Command not found. +par_tcsh_funky 3 arg alias_works_over_ssh +par_tcsh_funky +par_tcsh_funky myvar works +par_tcsh_funky space special chars problem +par_tcsh_funky Funky-  !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~-funky alias_var_works_over_ssh +par_tcsh_funky func_echo: Command not found. +par_tcsh_funky +par_tcsh_funky \\\\\\\\ \ \ \ \ \\\\\\\\\\\\\\ \!\"\#\$%\&\'\(\)\*+,-./0123456789:\;\<\=\>\?@ABCDEFGHIJKLMNOPQRSTUVWXYZ\[\\\]\^_\`abcdefghijklmnopqrstuvwxyz\{\|\}\~\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\ par_tcsh_man ### tcsh par_tcsh_man ### From man env_parallel par_tcsh_man aliases work