parallel/testsuite/tests-to-run/parallel-local-ssh4.sh

#!/bin/bash

unset run_test

# SSH only allowed to localhost/lo
# --retries if ssh dies
cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | parallel -vj1 --retries 2 -k --joblog /tmp/jl-`basename $0` -L1
echo '### --env _'
  fUbAr="OK FUBAR" parallel -S parallel@lo --env _ echo '$fUbAr $DEBEMAIL' ::: test
  fUbAr="OK FUBAR" parallel -S csh@lo --env _ echo '$fUbAr $DEBEMAIL' ::: test

echo '### --env _ with explicit mentioning of normally ignored var $DEBEMAIL'
  fUbAr="OK FUBAR" parallel -S parallel@lo --env DEBEMAIL,_ echo '$fUbAr $DEBEMAIL' ::: test
  fUbAr="OK FUBAR" parallel -S csh@lo --env DEBEMAIL,_ echo '$fUbAr $DEBEMAIL' ::: test

echo 'bug #40137: SHELL not bash: Warning when exporting funcs'
  . <(printf 'myfunc() {\necho $1\n}'); export -f myfunc; parallel --env myfunc -S lo myfunc ::: no_warning
  . <(printf 'myfunc() {\necho $1\n}'); export -f myfunc; SHELL=/bin/sh parallel --env myfunc -S lo myfunc ::: warning

echo 'env_parallel from man page - transfer non-exported var'
  env_parallel() { 
    export parallel_bash_environment="$(echo "shopt -s expand_aliases 2>/dev/null"; alias;typeset -p | grep -vFf <(readonly; echo GROUPS; echo FUNCNAME; echo DIRSTACK; echo _; echo PIPESTATUS; echo USERNAME) | grep -v BASH_;typeset -f)"; 
    `which parallel` "$@"; 
    unset parallel_bash_environment; 
  }; 
  var=nonexported env_parallel -S parallel@lo echo '$var' ::: variable

echo 'compared to parallel - no transfer non-exported var'
  var=nonexported parallel -S parallel@lo echo '$var' ::: variable

echo '### zsh'

echo 'env in zsh'
  echo 'Normal variable export'
  export B=\'; 
  PARALLEL_SHELL=/usr/bin/zsh parallel --env B echo '$B' ::: a

  echo 'Function export as variable'
  export myfuncvar="() { echo myfuncvar \$*; }"; 
  PARALLEL_SHELL=/usr/bin/zsh parallel --env myfuncvar myfuncvar ::: a

  echo 'Function export as function'
  myfunc() { echo myfunc $*; }; 
  export -f myfunc; 
  PARALLEL_SHELL=/usr/bin/zsh parallel --env myfunc myfunc ::: a


  ssh zsh@lo 'fun="() { echo function from zsh to zsh \$*; }"; 
              export fun; 
              parallel --env fun fun ::: OK'

  ssh zsh@lo 'fun="() { echo function from zsh to bash \$*; }"; 
              export fun; 
              parallel -S parallel@lo --env fun fun ::: OK'

echo '### csh'
  echo "3 big vars run remotely - length(base64) > 1000"
  stdout ssh csh@lo 'setenv A `seq 200|xargs`; 
                     setenv B `seq 200 -1 1|xargs`; 
                     setenv C `seq 300 -2 1|xargs`; 
                     parallel -Scsh@lo --env A,B,C -k echo \$\{\}\|wc ::: A B C'
echo '### csh2'
  echo "3 big vars run locally"
  stdout ssh csh@lo 'setenv A `seq 200|xargs`; 
                     setenv B `seq 200 -1 1|xargs`; 
                     setenv C `seq 300 -2 1|xargs`; 
                     parallel --env A,B,C -k echo \$\{\}\|wc ::: A B C'

echo '### Test tmux works on different shells'
  (stdout parallel -Scsh@lo,tcsh@lo,parallel@lo,zsh@lo --tmux echo ::: 1 2 3 4; echo $?) | grep -v 'See output';
  (stdout parallel -Scsh@lo,tcsh@lo,parallel@lo,zsh@lo --tmux false ::: 1 2 3 4; echo $?) | grep -v 'See output';

  export PARTMUX='parallel -Scsh@lo,tcsh@lo,parallel@lo,zsh@lo --tmux '; 
  stdout ssh zsh@lo      "$PARTMUX" 'true  ::: 1 2 3 4; echo $status' | grep -v 'See output'; 
  stdout ssh zsh@lo      "$PARTMUX" 'false ::: 1 2 3 4; echo $status' | grep -v 'See output'; 
  stdout ssh parallel@lo "$PARTMUX" 'true  ::: 1 2 3 4; echo $?'      | grep -v 'See output'; 
  stdout ssh parallel@lo "$PARTMUX" 'false ::: 1 2 3 4; echo $?'      | grep -v 'See output'; 
  stdout ssh tcsh@lo     "$PARTMUX" 'true  ::: 1 2 3 4; echo $status' | grep -v 'See output'; 
  stdout ssh tcsh@lo     "$PARTMUX" 'false ::: 1 2 3 4; echo $status' | grep -v 'See output'; 
  stdout ssh csh@lo      "$PARTMUX" 'true  ::: 1 2 3 4; echo $status' | grep -v 'See output'; 
  stdout ssh csh@lo      "$PARTMUX" 'false ::: 1 2 3 4; echo $status' | grep -v 'See output'

echo '### works'
  stdout parallel -Sparallel@lo --tmux echo ::: \\\\\\\"\\\\\\\"\\\;\@ | grep -v 'See output'
  stdout parallel -Sparallel@lo --tmux echo ::: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx | grep -v 'See output'

echo '### These blocked due to length'
  stdout parallel -Slo      --tmux echo ::: \\\\\\\"\\\\\\\"\\\;\@ | grep -v 'See output'
  stdout parallel -Scsh@lo  --tmux echo ::: \\\\\\\"\\\\\\\"\\\;\@ | grep -v 'See output'
  stdout parallel -Stcsh@lo --tmux echo ::: \\\\\\\"\\\\\\\"\\\;\@ | grep -v 'See output'
  stdout parallel -Szsh@lo  --tmux echo ::: \\\\\\\"\\\\\\\"\\\;\@ | grep -v 'See output'
  stdout parallel -Scsh@lo  --tmux echo ::: 111111111111111111111111111111111111111111111111111111111 | grep -v 'See output'

echo '### bug #43746: --transfer and --return of multiple inputs {1} and {2}'
echo '### and:'
echo '### bug #44371: --trc with csh complains'
  cd /tmp; echo 1 > file1; echo 2 > file2; 
  parallel -Scsh@lo --trc {1}.a --trc {2}.b 'echo A {1} > {1}.a; echo B {2} > {2}.b' ::: file1 ::: file2; 
  cat file1.a file2.b; 
  rm /tmp/file1 /tmp/file2 /tmp/file1.a /tmp/file2.b

echo '### bug #44143: csh and nice'
  parallel --nice 1 -S csh@lo setenv B {}\; echo '$B' ::: OK

echo '### bug #45575: -m and multiple hosts repeats first args'
  seq 1 3 | parallel -X -S 2/lo,2/: -k echo 

EOF

echo
echo Test env_parallel:
echo + for each shell
echo + remote, locally
echo + variables, variables with funky content, arrays, assoc array, functions, aliases
echo
echo "### Bash environment"
#stdout ssh -t lo <<'EOS'
myvar="myvar  works"
funky=$(perl -e 'print pack "c*", 1..255')
myarray=('' array_val2 3 '' 5)
declare -A assocarr
assocarr[a]=assoc_val_a
assocarr[b]=assoc_val_b
alias alias_echo="echo 3 arg";
func_echo() {
  echo $*;
  echo "$myvar"
  echo ${myarray[1]}
  echo ${assocarr[a]}
  echo Funky-"$funky"-funky
}

env_parallel() {
  export PARALLEL_ENV="$(echo "shopt -s expand_aliases 2>/dev/null"; alias;typeset -p |
    grep -vFf <(readonly) |
    grep -v 'declare .. (GROUPS|FUNCNAME|DIRSTACK|_|PIPESTATUS|USERNAME|BASH_[A-Z_]+) ';
    typeset -f)";
  `which parallel` "$@";
  unset PARALLEL_ENV;
}

env_parallel alias_echo ::: alias_works
env_parallel func_echo ::: function_works
env_parallel -S lo alias_echo ::: alias_works_over_ssh
env_parallel -S lo func_echo ::: function_works_over_ssh
echo
echo "$funky" | parallel --shellquote
#EOS


echo
echo "### Zsh environment"
stdout ssh -q zsh@lo <<'EOS' | egrep -v 'Welcome to |packages can be updated.'
myvar="myvar  works"
funky=$(perl -e 'print pack "c*", 1..255')
myarray=('' array_val2 3 '' 5)
declare -A assocarr
assocarr[a]=assoc_val_a
assocarr[b]=assoc_val_b
alias alias_echo="echo 3 arg";
func_echo() {
  echo $*;
  echo "$myvar"
  echo $myarray[2]
  echo ${assocarr[a]}
  echo Funky-"$funky"-funky
}

env_parallel() {
  export PARALLEL_ENV="$(alias | perl -pe 's/^/alias /';typeset -p |
    grep -aFvf <(typeset -pr)|egrep -iav 'ZSH_EVAL_CONTEXT|LINENO=| _=|aliases|^typeset [a-z_]+$'|
    egrep -av '^(typeset IFS=|..$)|cyan';
    typeset -f)";
  parallel "$@";
  unset PARALLEL_ENV;
}

# alias does not work: http://unix.stackexchange.com/questions/223534/defining-an-alias-and-immediately-use-it
env_parallel alias_echo ::: alias_does_not_work
env_parallel func_echo ::: function_works
env_parallel -S zsh@lo alias_echo ::: alias_does_not_work_over_ssh
env_parallel -S zsh@lo func_echo ::: function_works_over_ssh
echo
echo "$funky" | parallel --shellquote
EOS

echo
echo "### Ksh environment"
stdout ssh -q ksh@lo <<'EOS' | egrep -v 'Welcome to |packages can be updated.'
myvar="myvar  works"
funky=$(perl -e 'print pack "c*", 1..255')
myarray=('' array_val2 3 '' 5)
typeset -A assocarr
assocarr[a]=assoc_val_a
assocarr[b]=assoc_val_b
alias alias_echo="echo 3 arg";

func_echo() {
  echo $*;
  echo "$myvar"
  echo ${myarray[1]}
  echo ${assocarr[a]}
  echo Funky-"$funky"-funky
}

env_parallel() {
  export PARALLEL_ENV="$(alias | perl -pe 's/^/alias /';typeset -p|egrep -v 'typeset( -i)? -r|PIPESTATUS';typeset -f)";
  `which parallel` "$@";
  unset PARALLEL_ENV;
}
env_parallel alias_echo ::: alias_works
env_parallel func_echo ::: function_works
env_parallel -S ksh@lo alias_echo ::: alias_works_over_ssh
env_parallel -S ksh@lo func_echo ::: function_works_over_ssh
echo
echo "$funky" | parallel --shellquote
EOS

echo
echo "### Fish environment"
stdout ssh -q fish@lo <<'EOS' | egrep -v 'Welcome to |packages can be updated.'
# All variables cannot reliably be exported
# perl -e 'print map { "setenv///$_///$ENV{$_}\n"} grep !/^(PWD|SHLVL|PATH)$/, keys %ENV'| sh -c 'parallel --shellquote' | perl -pe 's:///: :g' |fish

set myvar "myvar  works"
setenv myenvvar "myenvvar  works"

set funky (perl -e 'print pack "c*", 1..255')
setenv funkyenv (perl -e 'print pack "c*", 1..255')

set myarray '' array_val2 3 '' 5
# Assoc arrays do not exist
#typeset -A assocarr
#assocarr[a]=assoc_val_a
#assocarr[b]=assoc_val_b
alias alias_echo="echo 3 arg";

function func_echo
  echo $argv;
  echo "$myvar"
  echo "$myenvvar"
# Arrays are not exported
#  echo $myarray[2]
# Assoc arrays do not exist in fish
#  echo ${assocarr[a]}
  echo
  echo
  echo
  echo Funky-"$funky"-funky
  echo Funky-"$funkyenv"-funky
  echo
  echo
  echo
end

function env_parallel
  setenv PARALLEL_ENV (begin; functions -n | perl -pe 's/,/\n/g' | while read d; functions $d; end; perl -e 'print map { "$_///$ENV{$_}\n"} grep !/^(PWD|SHLVL|PATH)$/, keys %ENV'| sh -c 'parallel --shellquote' | perl -pe 's:^([^/]+)///:setenv $1 :'; end |perl -pe 's/\001/\\cb/g;s/\n/\001/')
  parallel $argv;
  set -e PARALLEL_ENV
end

env_parallel alias_echo ::: alias_works
env_parallel func_echo ::: function_works
env_parallel -S fish@lo alias_echo ::: alias_works_over_ssh
env_parallel -S fish@lo func_echo ::: function_works_over_ssh
echo 
echo "$funky" | parallel --shellquote
EOS

echo 
echo "### csh environment"
# http://hyperpolyglot.org/unix-shells
# makealias:
#   alias quote     "/bin/sed -e 's/\\!/\\\\\!/g' -e 's/'\\\''/'\\\'\\\\\\\'\\\''/g' -e 's/^/'\''/' -e 's/"\$"/'\''/'"
#   alias makealias "quote | /bin/sed 's/^/alias \!:1 /' \!:2*"
#
#   makealias_with_newline
#   perl -e '$/=undef;$_=<>;s/\n/\\\n/g;s/\047/\047\042\047\042\047/g;print'

stdout ssh -q csh@lo <<'EOS' | egrep -v 'Welcome to |packages can be updated.'
set myvar = "myvar  works"
set funky = "`perl -e 'print pack q(c*), 1..255'`"
set myarray = ('' 'array_val2' '3' '' '5')
#declare -A assocarr
#assocarr[a]=assoc_val_a
#assocarr[b]=assoc_val_b
alias alias_echo echo 3 arg;
alias alias_echo_var 'echo $argv; echo $myvar; echo ${myarray[2]}; echo Funky-"$funky"-funky'

#function func_echo
#  echo $argv;
#  echo $myvar;
#  echo ${myarray[2]}
#  #echo ${assocarr[a]}
#  echo Funky-"$funky"-funky
#end

# ALIAS TO EXPORT ALIASES:

#   Quote ' by putting it inside "
#   s/'/'"'"'/g;
#   ' => \047 " => \042
#   s/\047/\047\042\047\042\047/g;
#   Quoted: s/\\047/\\047\\042\\047\\042\\047/g\;

#   Remove () from second column
#   s/^(\S+)(\s+)\((.*)\)/\1\2\3/
#   \047 => '
#   s/^(\S+)(\s+)\((.*)\)/\1\2\3/;
#   Quoted: s/\^\(\\S+\)\(\\s+\)\\\(\(.\*\)\\\)/\\1\\2\\3/\;

#   Add ' around second column
#   s/^(\S+)(\s+)(.*)/\1\2'\3'/
#   \047 => '
#   s/^(\S+)(\s+)(.*)/\1\2\047\3\047/;
#   Quoted: s/\^\(\\S+\)\(\\s+\)\(.\*\)/\\1\\2\\047\\3\\047/\;

#   Quote ! as \!
#   s/\!/\\\!/g;
#   Quoted: s/\\\!/\\\\\\\!/g;

#   Prepend with "\nalias "
#   s/^/\001alias /;
#   Quoted: s/\^/\\001alias\ /\;

alias env_parallel 'setenv PARALLEL_ENV "`alias | perl -pe s/\\047/\\047\\042\\047\\042\\047/g\;s/\^\(\\S+\)\(\\s+\)\\\(\(.\*\)\\\)/\\1\\2\\3/\;s/\^\(\\S+\)\(\\s+\)\(.\*\)/\\1\\2\\047\\3\\047/\;s/\^/\\001alias\ /\;s/\\\!/\\\\\\\!/g;`";parallel \!*; setenv PARALLEL_ENV'


##  set tmpfile=`tempfile`
##  foreach v (`set | awk -e '{print $1}' |grep -v prompt2`);
##  eval if'($?'$v' && ${#'$v'} <= 1) echo scalar'$v'="$'$v'"' >> $tmpfile;
##  eval if'($?'$v' && ${#'$v'} > 1) echo array'$v'="$'$v'"' >> $tmpfile;
##  end
##  cat $tmpfile | parallel --shellquote | perl -pe 's/^scalar(\S+).=/set $1=/ or s/^array(\S+).=(.*)/set $1=($2)/ && s/\\ / /g;'; rm $tmpfile
##  
##  set tmpfile=`tempfile`
##  foreach _vARnAmE (`set | awk -e '{print $1}' |grep -v prompt2`);
##  eval if'($?'$_vARnAmE' && ${#'$_vARnAmE'} <= 1) echo scalar'$_vARnAmE'="$'$_vARnAmE'"' >> $tmpfile; eval if'($?'$_vARnAmE' && ${#'$_vARnAmE'} > 1) echo array'$_vARnAmE'="$'$_vARnAmE'"' >> $tmpfile;
##  end
##  cat $tmpfile | parallel --shellquote | perl -pe 's/^scalar(\S+).=/set $1=/ or s/^array(\S+).=(.*)/set $1=($2)/ && s/\\ / /g;'; rm $tmpfile; unset tmpfile
##  
##  #!/bin/csh
##  
##  set _tmpfile=`tempfile`;
##  foreach _vARnAmE (`set | awk -e '{print $1}' |grep -Ev 'prompt2|_tmpfile'`);
##  eval if'($?'$_vARnAmE' && ${#'$_vARnAmE'} <= 1) echo scalar'$_vARnAmE'="$'$_vARnAmE'"' >> $_tmpfile;
##  eval if'($?'$_vARnAmE' && ${#'$_vARnAmE'} > 1) echo array'$_vARnAmE'="$'$_vARnAmE'"' >> $_tmpfile;
##  end 
##  setenv PARALLEL_ENV `cat $_tmpfile | parallel --shellquote | perl -pe 's/^scalar(\S+).=/set $1=/ or s/^array(\S+).=(.*)/set $1=($2)/ && s/\\ / /g; s/$/\001/';`
##  rm $_tmpfile;
##  unset _tmpfile
##  
##  setenv PARALLEL_ENV "$PARALLEL_ENV`alias | perl -pe s/\\047/\\047\\042\\047\\042\\047/g\;s/\^\(\\S+\)\(\\s+\)\\\(\(.\*\)\\\)/\\1\\2\\3/\;s/\^\(\\S+\)\(\\s+\)\(.\*\)/\\1\\2\\047\\3\\047/\;s/\^/\\001alias\ /\;s/\\\!/\\\\\\\!/g;`"
##  parallel \!*
##  setenv PARALLEL_ENV
##  
##  
##  perl -e '$/=undef;$_=<>;s/\n/\\\\\n/g;s/\047/\047\042\047\042\047/g;print "eval \047$_\047"'
##  
##  foreach g (h i j)
##  echo $g
##  end
##  
##  


env_parallel alias_echo ::: alias_works
env_parallel alias_echo_var ::: alias_var_does_not_work
env_parallel func_echo ::: function_does_not_work
env_parallel -S csh@lo alias_echo ::: alias_works_over_ssh
env_parallel -S csh@lo alias_echo_var ::: alias_var_does_not_work
env_parallel -S csh@lo func_echo ::: function_does_not_work_over_ssh
echo
echo "$funky" | parallel --shellquote
EOS