env_parallel.zsh now supports aliases.

This commit is contained in:
Ole Tange 2016-07-02 18:00:51 +02:00
parent 8637d30bca
commit dd5ade4fbe
7 changed files with 254 additions and 59 deletions

View file

@ -29,36 +29,19 @@
env_parallel() {
# env_parallel.bash
local argv_ARRAY=()
local grep_ARRAY=()
local _par_i
local _par_ARR
# Get the --env variables if set
while test $# -gt 0; do
key="$1"
case $key in
--env)
argv_ARRAY+=("$1")
shift
# split --env on ,
IFS=',' read -ra _par_ARR <<< "$1"
for _par_i in "${_par_ARR[@]}"; do
grep_ARRAY+=("$_par_i")
done
;;
esac
argv_ARRAY+=("$1")
shift # past argument or value
done
# This converts a b c to (a|b|c)
local grep_REGEXP="$(perl -e 'print "(".(join "|",map { quotemeta $_ } @ARGV).")"' "${grep_ARRAY[@]}")"
if [[ "$grep_REGEXP" = "()" ]] ; then
# --env not set: Match everything
grep_REGEXP="(.*)"
fi
# and convert a b c to (a|b|c)
# If --env not set: Match everything (.*)
local grep_REGEXP="$(
perl -e 'for(@ARGV){
$next_is_env and push @envvar, split/,/, $_;
$next_is_env=/^--env$/;
}
$vars = join "|",map { quotemeta $_ } @envvar;
print $vars ? "($vars)" : "(.*)";
' -- "$@"
)"
# Grep alias names
local _alias_NAMES="$(compgen -a |
@ -90,20 +73,22 @@ env_parallel() {
fi
# Copy shopt (so e.g. extended globbing works)
# But force expand_aliases as aliases otherwise do not work
export PARALLEL_ENV="$(
shopt 2>/dev/null |
perl -pe 's:\s+off:;: and s/^/shopt -u /;
s:\s+on:;: and s/^/shopt -s /;';
echo 'shopt -s expand_aliases 2>/dev/null';
$_list_alias_BODIES;
$_list_variable_VALUES;
$_list_function_BODIES)";
`which parallel` "${argv_ARRAY[@]}";
`which parallel` "$@";
unset PARALLEL_ENV;
}
# Supports env of 127375 bytes
#
# env_parallel() {
# _env_parallel() {
# # Saving to a tempfile
# export PARALLEL_ENV=`tempfile`;
# (echo "shopt -s expand_aliases 2>/dev/null"; alias;typeset -p |

View file

@ -95,6 +95,9 @@ E.g. by doing:
=head2 Zsh
B<--env> is supported to export only the variable, alias, function, or
array with the given name. Multiple B<--env>s can be given.
Installation
Put this in $HOME/.zshrc:
@ -109,26 +112,35 @@ E.g. by doing:
=item aliases
Not supported - Zsh does not support aliases defined in the same
parsing as they are used.
alias myecho='echo aliases'
env_parallel myecho ::: work
env_parallel -S server myecho ::: work
env_parallel --env myecho myecho ::: work
env_parallel --env myecho -S server myecho ::: work
=item functions
myfunc() { echo $*; }
env_parallel myfunc ::: test
env_parallel -S server myfunc ::: test
myfunc() { echo functions $*; }
env_parallel myfunc ::: work
env_parallel -S server myfunc ::: work
env_parallel --env myfunc myfunc ::: work
env_parallel --env myfunc -S server myfunc ::: work
=item variables
myvar=test
env_parallel echo '$myvar' ::: test
env_parallel -S server echo '$myvar' ::: test
myvar=variables
env_parallel echo '$myvar' ::: work
env_parallel -S server echo '$myvar' ::: work
env_parallel --env myvar echo '$myvar' ::: work
env_parallel --env myvar -S server echo '$myvar' ::: work
=item arrays
myarray=(foo bar baz)
env_parallel echo '${myarray[{}]}' ::: 1 2 3
env_parallel -S server echo '${myarray[{}]}' ::: 1 2 3
myarray=(arrays work, too)
env_parallel -k echo '${myarray[{}]}' ::: 0 1 2
env_parallel -k -S server echo '${myarray[{}]}' ::: 0 1 2
env_parallel -k --env myarray echo '${myarray[{}]}' ::: 0 1 2
env_parallel -k --env myarray -S server echo '${myarray[{}]}' ::: 0 1 2
=back

View file

@ -26,6 +26,67 @@
# Fifth Floor, Boston, MA 02110-1301 USA
env_parallel() {
# env_parallel.zsh
# Get the --env variables if set
# and convert a b c to (a|b|c)
# If --env not set: Match everything (.*)
grep_REGEXP="$(
perl -e 'for(@ARGV){
$next_is_env and push @envvar, split/,/, $_;
$next_is_env=/^--env$/;
}
$vars = join "|",map { quotemeta $_ } @envvar;
print $vars ? "($vars)" : "(.*)";
' -- "$@"
)"
# Grep alias names
_alias_NAMES="$(print -l ${(k)aliases} |
egrep "^${grep_REGEXP}\$")"
_list_alias_BODIES="alias "$(echo $_alias_NAMES|xargs)" | perl -pe 's/^/alias /'"
if [[ "$_alias_NAMES" = "" ]] ; then
# no aliases selected
_list_alias_BODIES="true"
fi
# Grep function names
_function_NAMES="$(print -l ${(k)functions} |
egrep "^${grep_REGEXP}\$" |
grep -v '='
)"
_list_function_BODIES="typeset -f "$(echo $_function_NAMES|xargs)
if [[ "$_function_NAMES" = "" ]] ; then
# no functions selected
_list_function_BODIES="true"
fi
# Grep variable names
# The egrep -v is crap and should be better
_variable_NAMES="$(print -l ${(k)parameters} |
egrep "^${grep_REGEXP}\$" |
egrep -v '^([-?#!$*@_0]|zsh_eval_context|ZSH_EVAL_CONTEXT|LINENO|IFS|commands|functions|options|aliases|EUID|EGID|UID|GID)$' |
egrep -v 'terminfo|funcstack|galiases|keymaps|parameters|jobdirs|dirstack|functrace|funcsourcetrace|zsh_scheduled_events|dis_aliases|dis_reswords|dis_saliases|modules|reswords|saliases|widgets|userdirs|historywords|nameddirs|termcap|dis_builtins|dis_functions|jobtexts|funcfiletrace|dis_galiases|builtins|history|jobstates'
)"
_list_variable_VALUES="typeset -p "$(echo $_variable_NAMES|xargs)" |
grep -aFvf <(typeset -pr)
"
if [[ "$_variable_NAMES" = "" ]] ; then
# no variables selected
_list_variable_VALUES="true"
fi
export PARALLEL_ENV="$(
eval $_list_alias_BODIES;
eval $_list_function_BODIES;
eval $_list_variable_VALUES;
)";
`which parallel` "$@";
unset PARALLEL_ENV;
}
_old_env_parallel() {
# env_parallel.zsh
export PARALLEL_ENV="$(alias | perl -pe 's/^/alias /'; typeset -p |
grep -aFvf <(typeset -pr) |

View file

@ -6816,10 +6816,16 @@ sub wrapped {
close $parallel_env;
}
# If $PARALLEL_ENV set, put that in front of the command
# Used for importing functions for fish
# Used for env_parallel.*
# Map \001 to \n to make it easer to quote \n in $PARALLEL_ENV
$ENV{'PARALLEL_ENV'} =~ s/\001/\n/g;
$command = $ENV{'PARALLEL_ENV'}."\n".$command;
if($Global::shell =~ /zsh/) {
# The extra 'eval' will make aliases work, too
$command = $ENV{'PARALLEL_ENV'}."\n".
"eval ".::shell_quote_scalar($command);
} else {
$command = $ENV{'PARALLEL_ENV'}."\n".$command;
}
}
# Wrap with ssh + tranferring of files
$command = $self->sshlogin_wrap($command);
@ -6887,6 +6893,7 @@ sub string_zip_base64 {
# Pipe string through 'bzip2 -9' and base64 encode it into 1000
# byte blocks.
# 1000 bytes is the largest word size csh supports
# Zipping will make exporting big environments work, too
# Input:
# @strings = to be encoded
# Returns:
@ -6958,8 +6965,8 @@ sub base64_wrap {
my $eval_string = shift;
return
"perl -e ".
::shell_quote_scalar(base64_eval())." ".
join" ",::shell_quote(string_base64($eval_string));
::shell_quote_scalar(base64_zip_eval())." ".
join" ",::shell_quote(string_zip_base64($eval_string));
}
sub base64_eval {
@ -7184,8 +7191,8 @@ sub sshlogin_wrap {
# csh does not deal well with $ENV with \n
$quoted_remote_command =
"perl -e ".
::shell_quote_scalar(::shell_quote_scalar(base64_eval()))." ".
join" ",::shell_quote(::shell_quote(string_base64($remote_command)));
::shell_quote_scalar(::shell_quote_scalar(base64_zip_eval()))." ".
join" ",::shell_quote(::shell_quote(string_zip_base64($remote_command)));
} else {
$quoted_remote_command = $dq_remote_command;
}

View file

@ -1,6 +1,6 @@
#!/bin/bash
mysqlrootpass=${mysqlrootpass:-FoOo}
mysqlrootpass=${mysqlrootpass:+FoOo}
# The testsuite depends on this:
@ -12,20 +12,22 @@ sudo aptitude install dpkg-dev build-essential debhelper
# DATABASES
sudo aptitude install postgresql mysql-server
echo '# Create PostgreSQL'
sudo su - postgres -c 'createdb '`whoami`
sudo su - postgres -c 'createuser '`whoami`
sudo su - postgres -c "sql pg:/// \"ALTER USER \\\"`whoami`\\\" WITH PASSWORD '`whoami`';\""
echo '# Create MySQL'
sudo su - mysql mysqladmin create `whoami`
sql mysql://root:$mysqlrootpass@/mysql "CREATE DATABASE `whoami`;CREATE USER '`whoami`'@'localhost' IDENTIFIED BY '`whoami`'; GRANT ALL ON `whoami`.* TO '`whoami`'@'localhost';"
sql mysql://root:"$mysqlrootpass"@/mysql "DROP DATABASE `whoami`;DROP USER '`whoami`'@'localhost';"
sql mysql://root:"$mysqlrootpass"@/mysql "CREATE DATABASE `whoami`;CREATE USER '`whoami`'@'localhost' IDENTIFIED BY '`whoami`'; GRANT ALL ON `whoami`.* TO '`whoami`'@'localhost';"
# SHELLS
sudo aptitude install ash csh fdclone fish fizsh ksh mksh pdksh posh rush sash tcsh yash zsh
SSHPASS=`goodpasswd`
export SSHPASS
shells="sh csh ash tcsh zsh ksh fish fizsh mksh pdksh posh rc sash yash nopathbash nopathcsh"
shells="bash sh csh ash tcsh zsh ksh fish fizsh mksh pdksh posh rc sash yash nopathbash nopathcsh"
create_shell_user() {
shell="$1"
sudo deluser $shell && sudo mv /home/$shell /tmp/$shell.$RANDOM
@ -79,7 +81,7 @@ sudo aptitude install lsh-client
cd
mkdir .lsh
lsh-make-seed -o ".lsh/yarrow-seed-file"
lsh --sloppy-host-authentication --capture-to ~/.lsh/host-acls lo
lsh --sloppy-host-authentication --capture-to ~/.lsh/host-acls lo echo Added host-auth
lsh-keygen | lsh-writekey -c none
lsh-export-key --openssh < ~/.lsh/identity.pub | lsh localhost 'cat >>.ssh/authorized_keys'
lsh-export-key --openssh < ~/.lsh/identity.pub | ssh csh@localhost 'cat >>.ssh/authorized_keys'

View file

@ -6,7 +6,75 @@ unset run_test
# SSH only allowed to localhost/lo
# --retries if ssh dies
cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | parallel -vj1 --retries 2 -k --joblog /tmp/jl-`basename $0` -L1
cat <<'EOF' | sed -e s/\$SERVER1/$SERVER1/\;s/\$SERVER2/$SERVER2/ | parallel -vj4 --retries 2 -k --joblog /tmp/jl-`basename $0` -L1
echo '### --env from man env_parallel'
echo '### bash'
ssh bash@lo '
alias myecho="echo aliases";
env_parallel myecho ::: work;
env_parallel -S server myecho ::: work;
env_parallel --env myecho myecho ::: work;
env_parallel --env myecho -S server myecho ::: work
'
ssh bash@lo '
myfunc() { echo functions $*; };
env_parallel myfunc ::: work;
env_parallel -S server myfunc ::: work;
env_parallel --env myfunc myfunc ::: work;
env_parallel --env myfunc -S server myfunc ::: work
'
ssh bash@lo '
myvar=variables;
env_parallel echo "\$myvar" ::: work;
env_parallel -S server echo "\$myvar" ::: work;
env_parallel --env myvar echo "\$myvar" ::: work;
env_parallel --env myvar -S server echo "\$myvar" ::: work
'
ssh bash@lo '
myarray=(arrays work, too);
env_parallel -k echo "\${myarray[{}]}" ::: 0 1 2;
env_parallel -k -S server echo "\${myarray[{}]}" ::: 0 1 2;
env_parallel -k --env myarray echo "\${myarray[{}]}" ::: 0 1 2;
env_parallel -k --env myarray -S server echo "\${myarray[{}]}" ::: 0 1 2
'
echo '### zsh'
ssh zsh@lo '
alias myecho="echo aliases";
env_parallel myecho ::: work;
env_parallel -S server myecho ::: work;
env_parallel --env myecho myecho ::: work;
env_parallel --env myecho -S server myecho ::: work
'
ssh zsh@lo '
myfunc() { echo functions $*; };
env_parallel myfunc ::: work;
env_parallel -S server myfunc ::: work;
env_parallel --env myfunc myfunc ::: work;
env_parallel --env myfunc -S server myfunc ::: work
'
ssh zsh@lo '
myvar=variables;
env_parallel echo "\$myvar" ::: work;
env_parallel -S server echo "\$myvar" ::: work;
env_parallel --env myvar echo "\$myvar" ::: work;
env_parallel --env myvar -S server echo "\$myvar" ::: work
'
ssh zsh@lo '
myarray=(arrays work, too);
env_parallel -k echo "\${myarray[{}]}" ::: 1 2 3;
env_parallel -k -S server echo "\${myarray[{}]}" ::: 1 2 3;
env_parallel -k --env myarray echo "\${myarray[{}]}" ::: 1 2 3;
env_parallel -k --env myarray -S server echo "\${myarray[{}]}" ::: 1 2 3
'
echo '### --env _'
fUbAr="OK FUBAR" parallel -S parallel@lo --env _ echo '$fUbAr $DEBEMAIL' ::: test
fUbAr="OK FUBAR" parallel -S csh@lo --env _ echo '$fUbAr $DEBEMAIL' ::: test
@ -172,11 +240,9 @@ func_echo() {
echo Funky-"$funky"-funky
}
# alias does not work:
# http://unix.stackexchange.com/questions/223534/defining-an-alias-and-immediately-use-it
env_parallel alias_echo ::: alias_does_not_work
env_parallel alias_echo ::: alias_works
env_parallel func_echo ::: function_works
env_parallel -S zsh@lo alias_echo ::: alias_does_not_work_over_ssh
env_parallel -S zsh@lo alias_echo ::: alias_works_over_ssh
env_parallel -S zsh@lo func_echo ::: function_works_over_ssh
echo
echo "$funky" | parallel --shellquote

View file

@ -1,3 +1,65 @@
echo '### --env from man env_parallel'
### --env from man env_parallel
echo '### bash'
### bash
ssh bash@lo ' alias myecho="echo aliases"; env_parallel myecho ::: work; env_parallel -S server myecho ::: work; env_parallel --env myecho myecho ::: work; env_parallel --env myecho -S server myecho ::: work '
aliases work
aliases work
aliases work
aliases work
ssh bash@lo ' myfunc() { echo functions $*; }; env_parallel myfunc ::: work; env_parallel -S server myfunc ::: work; env_parallel --env myfunc myfunc ::: work; env_parallel --env myfunc -S server myfunc ::: work '
functions work
functions work
functions work
functions work
ssh bash@lo ' myvar=variables; env_parallel echo "\$myvar" ::: work; env_parallel -S server echo "\$myvar" ::: work; env_parallel --env myvar echo "\$myvar" ::: work; env_parallel --env myvar -S server echo "\$myvar" ::: work '
variables work
variables work
variables work
variables work
ssh bash@lo ' myarray=(arrays work, too); env_parallel -k echo "\${myarray[{}]}" ::: 0 1 2; env_parallel -k -S server echo "\${myarray[{}]}" ::: 0 1 2; env_parallel -k --env myarray echo "\${myarray[{}]}" ::: 0 1 2; env_parallel -k --env myarray -S server echo "\${myarray[{}]}" ::: 0 1 2 '
arrays
work,
too
arrays
work,
too
arrays
work,
too
arrays
work,
too
echo '### zsh'
### zsh
ssh zsh@lo ' alias myecho="echo aliases"; env_parallel myecho ::: work; env_parallel -S server myecho ::: work; env_parallel --env myecho myecho ::: work; env_parallel --env myecho -S server myecho ::: work '
aliases work
aliases work
aliases work
aliases work
ssh zsh@lo ' myfunc() { echo functions $*; }; env_parallel myfunc ::: work; env_parallel -S server myfunc ::: work; env_parallel --env myfunc myfunc ::: work; env_parallel --env myfunc -S server myfunc ::: work '
functions work
functions work
functions work
functions work
ssh zsh@lo ' myvar=variables; env_parallel echo "\$myvar" ::: work; env_parallel -S server echo "\$myvar" ::: work; env_parallel --env myvar echo "\$myvar" ::: work; env_parallel --env myvar -S server echo "\$myvar" ::: work '
variables work
variables work
variables work
variables work
ssh zsh@lo ' myarray=(arrays work, too); env_parallel -k echo "\${myarray[{}]}" ::: 1 2 3; env_parallel -k -S server echo "\${myarray[{}]}" ::: 1 2 3; env_parallel -k --env myarray echo "\${myarray[{}]}" ::: 1 2 3; env_parallel -k --env myarray -S server echo "\${myarray[{}]}" ::: 1 2 3 '
arrays
work,
too
arrays
work,
too
arrays
work,
too
arrays
work,
too
echo '### --env _'
### --env _
fUbAr="OK FUBAR" parallel -S parallel@lo --env _ echo '$fUbAr $DEBEMAIL' ::: test
@ -166,7 +228,7 @@ Funky-
### Zsh environment
* Documentation: http://www.linuxmint.com
zsh:130: command not found: alias_echo
3 arg alias_works
function_works
myvar works
space 6
@ -174,7 +236,7 @@ assoc_val_a
Funky-

 !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~€亗儎厗噲墛媽崕彁憭摂晼棙櫄洔潪煚、¥ウЖ┆<D096><E29486><EFBFBD>辈炒刀犯购患骄坷谅媚牌侨墒颂臀闲岩釉罩棕仝圮蒉哙徕沅彐玷殛腱眍镳耱篝貊鼬<E8B28A><E9BCAC><EFBFBD><EFBFBD>-funky
zsh:130: command not found: alias_echo
3 arg alias_works_over_ssh
function_works_over_ssh
myvar works
space 6