diff --git a/doc/release_new_version b/doc/release_new_version index b23995fc..67213906 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -266,6 +266,8 @@ for Big Data Applications https://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumb * FaceCrop uses GNU Parallel: https://github.com/EderSantana/FaceCrop +* GNU parallel 應用範例 http://staypython.blogspot.dk/2016/04/gnu-parallel.html + * Bug fixes and man page updates. GNU Parallel - For people who live life in the parallel lane. diff --git a/src/Makefile.am b/src/Makefile.am index de029b15..fba9bc98 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -1,6 +1,6 @@ bin_SCRIPTS = parallel sql niceload \ env_parallel env_parallel.bash env_parallel.zsh env_parallel.fish \ - env_parallel.ksh env_parallel.pdksh env_parallel.csh + env_parallel.ksh env_parallel.pdksh env_parallel.csh env_parallel.tcsh install-exec-hook: rm $(DESTDIR)$(bindir)/sem || true @@ -179,7 +179,7 @@ DISTCLEANFILES = parallel.1 env_parallel.1 sem.1 sql.1 niceload.1 \ EXTRA_DIST = parallel sem sql niceload env_parallel \ env_parallel.bash env_parallel.zsh env_parallel.fish env_parallel.ksh \ - env_parallel.pdksh env_parallel.csh \ + env_parallel.pdksh env_parallel.csh env_parallel.tcsh \ sem.pod parallel.pod env_parallel.pod niceload.pod parallel_tutorial.pod \ parallel_design.pod \ $(DISTCLEANFILES) diff --git a/src/Makefile.in b/src/Makefile.in index 5ef654c3..eb842a2c 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -219,7 +219,7 @@ top_builddir = @top_builddir@ top_srcdir = @top_srcdir@ bin_SCRIPTS = parallel sql niceload \ env_parallel env_parallel.bash env_parallel.zsh env_parallel.fish \ - env_parallel.ksh env_parallel.pdksh env_parallel.csh + env_parallel.ksh env_parallel.pdksh env_parallel.csh env_parallel.tcsh @DOCUMENTATION_TRUE@man_MANS = parallel.1 env_parallel.1 sem.1 sql.1 niceload.1 \ @DOCUMENTATION_TRUE@ parallel_tutorial.7 parallel_design.7 @@ -242,7 +242,7 @@ DISTCLEANFILES = parallel.1 env_parallel.1 sem.1 sql.1 niceload.1 \ EXTRA_DIST = parallel sem sql niceload env_parallel \ env_parallel.bash env_parallel.zsh env_parallel.fish env_parallel.ksh \ - env_parallel.pdksh env_parallel.csh \ + env_parallel.pdksh env_parallel.csh env_parallel.tcsh \ sem.pod parallel.pod env_parallel.pod niceload.pod parallel_tutorial.pod \ parallel_design.pod \ $(DISTCLEANFILES) diff --git a/src/env_parallel b/src/env_parallel index f62983a9..23adae80 100755 --- a/src/env_parallel +++ b/src/env_parallel @@ -46,7 +46,11 @@ pdksh: Put this in $HOME/.profile: source `which env_parallel.pdksh` csh: Put this in $HOME/.cshrc: source `which env_parallel.csh` E.g. by doing: echo 'source `which env_parallel.csh`' >> $HOME/.cshrc - Supports: aliases + Supports: aliases, variables, arrays with no special chars + +tcsh: Put this in $HOME/.tcshrc: source `which env_parallel.tcsh` + E.g. by doing: echo 'source `which env_parallel.tcsh`' >> $HOME/.tcshrc + Supports: aliases, variables, arrays with no special chars For details: see man env_parallel diff --git a/src/env_parallel.csh b/src/env_parallel.csh index fe912092..1e555b0b 100755 --- a/src/env_parallel.csh +++ b/src/env_parallel.csh @@ -25,4 +25,70 @@ # or write to the Free Software Foundation, Inc., 51 Franklin St, # Fifth Floor, Boston, MA 02110-1301 USA -alias env_parallel 'setenv PARALLEL_ENV "`alias | perl -pe s/\\047/\\047\\042\\047\\042\\047/g\;s/\^\(\\S+\)\(\\s+\)\\\(\(.\*\)\\\)/\\1\\2\\3/\;s/\^\(\\S+\)\(\\s+\)\(.\*\)/\\1\\2\\047\\3\\047/\;s/\^/\\001alias\ /\;s/\\\!/\\\\\\\!/g;`";parallel \!*; setenv PARALLEL_ENV' +if ("`alias env_parallel`" == '') then + # Activate alias + alias env_parallel 'setenv PARALLEL "\!*"; source `which env_parallel.csh`' +else + # Get the scalar and array variable names + set _vARnAmES=(`set | awk -e '{print $1}' |grep -v prompt2`) + + # Make a tmpfile for the variable definitions + set _tMpvARfILe=`tempfile` + + # Make a tmpfile for the variable definitions + alias + set _tMpaLLfILe=`tempfile` + foreach _vARnAmE ($_vARnAmES); + # if $?myvar && $#myvar <= 1 echo scalar_myvar=$var + eval if'($?'$_vARnAmE' && ${#'$_vARnAmE'} <= 1) echo scalar_'$_vARnAmE'="$'$_vARnAmE'"' >> $_tMpvARfILe; + # if $?myvar && $#myvar > 1 echo array_myvar=$var + eval if'($?'$_vARnAmE' && ${#'$_vARnAmE'} > 1) echo array_'$_vARnAmE'="$'$_vARnAmE'"' >> $_tMpvARfILe; + end + + # shell quote variables (--plain needed due to $PARALLEL abuse) + # Convert 'scalar_myvar=...' to 'set myvar=...' + # Convert 'array_myvar=...' to 'set array=(...)' + cat $_tMpvARfILe | parallel --plain --shellquote | perl -pe 's/^scalar_(\S+).=/set $1=/ or s/^array_(\S+).=(.*)/set $1=($2)/ && s/\\ / /g;' > $_tMpaLLfILe + + # Cleanup + rm $_tMpvARfILe; unset _tMpvARfILe _vARnAmE _vARnAmES + +# ALIAS TO EXPORT ALIASES: + +# Quote ' by putting it inside " +# s/'/'"'"'/g; +# ' => \047 " => \042 +# s/\047/\047\042\047\042\047/g; +# Quoted: s/\\047/\\047\\042\\047\\042\\047/g\; + +# Remove () from second column +# s/^(\S+)(\s+)\((.*)\)/\1\2\3/ +# \047 => ' +# s/^(\S+)(\s+)\((.*)\)/\1\2\3/; +# Quoted: s/\^\(\\S+\)\(\\s+\)\\\(\(.\*\)\\\)/\\1\\2\\3/\; + +# Add ' around second column +# s/^(\S+)(\s+)(.*)/\1\2'\3'/ +# \047 => ' +# s/^(\S+)(\s+)(.*)/\1\2\047\3\047/; +# Quoted: s/\^\(\\S+\)\(\\s+\)\(.\*\)/\\1\\2\\047\\3\\047/\; + +# Quote ! as \! +# s/\!/\\\!/g; +# Quoted: s/\\\!/\\\\\\\!/g; + +# Prepend with "\nalias " +# s/^/\001alias /; +# Quoted: s/\^/\\001alias\ /\; + alias | perl -pe s/\\047/\\047\\042\\047\\042\\047/g\;s/\^\(\\S+\)\(\\s+\)\\\(\(.\*\)\\\)/\\1\\2\\3/\;s/\^\(\\S+\)\(\\s+\)\(.\*\)/\\1\\2\\047\\3\\047/\;s/\^/\\001alias\ /\;s/\\\!/\\\\\\\!/g >> $_tMpaLLfILe + + setenv PARALLEL_ENV "`cat $_tMpaLLfILe; rm $_tMpaLLfILe`"; + unset _tMpaLLfILe; + # Use $PARALLEL set in calling alias + parallel + setenv PARALLEL_ENV + setenv PARALLEL +endif + +# Tested working for aliases +# alias env_parallel 'setenv PARALLEL_ENV "`alias | perl -pe s/\\047/\\047\\042\\047\\042\\047/g\;s/\^\(\\S+\)\(\\s+\)\\\(\(.\*\)\\\)/\\1\\2\\3/\;s/\^\(\\S+\)\(\\s+\)\(.\*\)/\\1\\2\\047\\3\\047/\;s/\^/\\001alias\ /\;s/\\\!/\\\\\\\!/g;`";parallel \!*; setenv PARALLEL_ENV' + diff --git a/src/env_parallel.fish b/src/env_parallel.fish index 38cf7308..dd5b403c 100755 --- a/src/env_parallel.fish +++ b/src/env_parallel.fish @@ -75,6 +75,8 @@ function env_parallel $val=~s/'"'"'/\\\$&/go; # Quote newline as '\n' $val =~ s/[\n]/\\\n/go; + # Empty => 2 single quotes = \047\047 + $val=~s/^$/\047\047/o; if($name ne $last and $last) { # The $name is different, so this is a new variable. # Print the last one. diff --git a/src/env_parallel.pod b/src/env_parallel.pod index a8b6a0b1..18f703ce 100644 --- a/src/env_parallel.pod +++ b/src/env_parallel.pod @@ -243,6 +243,9 @@ E.g. by doing: =head2 csh +B for B breaks B<$PARALLEL>, so do not use +B<$PARALLEL>. + Installation Put this in $HOME/.cshrc: @@ -267,15 +270,21 @@ Not supported by B. =item variables -Not supported + set myvar=test + env_parallel echo "\$myvar" ::: test + env_parallel -S csh@server echo "\$myvar" ::: test -=item arrays -Not supported +=item arrays with no special chars + + set myarray=(foo bar baz) + env_parallel echo "\${myarray\[\{\}\]}" ::: 1 2 3 + env_parallel -S csh@server echo "\${myarray\[\{\}\]}" ::: 1 2 3 =back + =head1 EXIT STATUS Same as GNU B. diff --git a/src/env_parallel.tcsh b/src/env_parallel.tcsh new file mode 100755 index 00000000..dfd457c4 --- /dev/null +++ b/src/env_parallel.tcsh @@ -0,0 +1,94 @@ +#!/bin/csh + +# This file must be sourced in csh: +# +# source `which env_parallel.csh` +# +# after which 'env_parallel' works +# +# +# Copyright (C) 2016 +# Ole Tange and Free Software Foundation, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, see +# or write to the Free Software Foundation, Inc., 51 Franklin St, +# Fifth Floor, Boston, MA 02110-1301 USA + +if ("`alias env_parallel`" == '') then + # Activate alias +## alias env_parallel 'setenv PARALLEL "\!*"; source `which env_parallel.csh`' + alias env_parallel 'setenv PARALLEL "\!*"; source /tmp/env_parallel.csh' +else + # Get the scalar and array variable names + set _vARnAmES=(`set | awk -e '{print $1}' |grep -vE '^(_|killring|prompt2)$'`) + + # Make a tmpfile for the variable definitions + set _tMpvARfILe=`tempfile` + + # Make a tmpfile for the variable definitions + alias + set _tMpaLLfILe=`tempfile` + foreach _vARnAmE ($_vARnAmES); + # if $?myvar && $#myvar <= 1 echo scalar_myvar=$var + eval if'($?'$_vARnAmE' && ${#'$_vARnAmE'} <= 1) echo scalar_'$_vARnAmE'=\"'\"\$$_vARnAmE\"'\"' >> $_tMpvARfILe; + # if $?myvar && $#myvar > 1 echo array_myvar=$var + eval if'($?'$_vARnAmE' && ${#'$_vARnAmE'} > 1) echo array_'$_vARnAmE'="$'$_vARnAmE'"' >> $_tMpvARfILe; + end + + # shell quote variables (--plain needed due to $PARALLEL abuse) + # Convert 'scalar_myvar=...' to 'set myvar=...' + # Convert 'array_myvar=...' to 'set array=(...)' + cat $_tMpvARfILe | parallel --plain --shellquote | perl -pe 's/^scalar_(\S+).=/set $1=/ or s/^array_(\S+).=(.*)/set $1=($2)/ && s/\\ / /g;' > $_tMpaLLfILe + # Cleanup + rm $_tMpvARfILe; unset _tMpvARfILe _vARnAmE _vARnAmES + +# ALIAS TO EXPORT ALIASES: + +# Quote ' by putting it inside " +# s/'/'"'"'/g; +# ' => \047 " => \042 +# s/\047/\047\042\047\042\047/g; +# Quoted: s/\\047/\\047\\042\\047\\042\\047/g\; + +# Remove () from second column +# s/^(\S+)(\s+)\((.*)\)/\1\2\3/ +# \047 => ' +# s/^(\S+)(\s+)\((.*)\)/\1\2\3/; +# Quoted: s/\^\(\\S+\)\(\\s+\)\\\(\(.\*\)\\\)/\\1\\2\\3/\; + +# Add ' around second column +# s/^(\S+)(\s+)(.*)/\1\2'\3'/ +# \047 => ' +# s/^(\S+)(\s+)(.*)/\1\2\047\3\047/; +# Quoted: s/\^\(\\S+\)\(\\s+\)\(.\*\)/\\1\\2\\047\\3\\047/\; + +# Quote ! as \! +# s/\!/\\\!/g; +# Quoted: s/\\\!/\\\\\\\!/g; + +# Prepend with "\nalias " +# s/^/\001alias /; +# Quoted: s/\^/\\001alias\ /\; + alias | perl -pe s/\\047/\\047\\042\\047\\042\\047/g\;s/\^\(\\S+\)\(\\s+\)\\\(\(.\*\)\\\)/\\1\\2\\3/\;s/\^\(\\S+\)\(\\s+\)\(.\*\)/\\1\\2\\047\\3\\047/\;s/\^/\\001alias\ /\;s/\\\!/\\\\\\\!/g >> $_tMpaLLfILe + + setenv PARALLEL_ENV "`cat $_tMpaLLfILe; rm $_tMpaLLfILe`"; + unset _tMpaLLfILe; + # Use $PARALLEL set in calling alias + parallel + setenv PARALLEL_ENV + setenv PARALLEL +endif + +# Tested working for aliases +# alias env_parallel 'setenv PARALLEL_ENV "`alias | perl -pe s/\\047/\\047\\042\\047\\042\\047/g\;s/\^\(\\S+\)\(\\s+\)\\\(\(.\*\)\\\)/\\1\\2\\3/\;s/\^\(\\S+\)\(\\s+\)\(.\*\)/\\1\\2\\047\\3\\047/\;s/\^/\\001alias\ /\;s/\\\!/\\\\\\\!/g;`";parallel \!*; setenv PARALLEL_ENV' + diff --git a/src/parallel b/src/parallel index 0dd076e5..cb49a54d 100755 --- a/src/parallel +++ b/src/parallel @@ -6865,7 +6865,7 @@ sub wrapped { $command = "perl -e ". ::shell_quote_scalar(base64_eval())." ". join(" ", ::shell_quote( - string_base64('exec "'. + string_base64("exec \"$Global::shell\",'-c',\"". ::perl_quote_scalar($command).'"'))); } $self->{'wrapped'} = $command; diff --git a/src/parallel_design.pod b/src/parallel_design.pod index 303d1a5e..a80be7bf 100644 --- a/src/parallel_design.pod +++ b/src/parallel_design.pod @@ -50,6 +50,82 @@ Fixing the problem in B often breaks it in B. In these cases the fix is often to use a small Perl script and call that. +=head2 env_parallel + +B is a dummy shell script that will run if +B is not an alias or a function and tell the user how to +activate the alias/function for the supported shells. + +The alias or function will copy the current environment and run the +command with GNU B in the copy of the environment. + +The problem is that you cannot access all of the current environment +inside Perl. E.g. aliases, functions and unexported shell variables. + +The idea is therefore to take the environment and put it in +B<$PARALLEL_ENV> which GNU B prepends to every command. + +The only way to have access to the environment is directly from the +shell, so the program must be written in a shell script that will be +sourced and there has to deal with the dialect of the relevant shell. + +Dumping the environment into a format that can be activated again is +easy in Bash: B and B basically give you the commands +you must run to re-create the environment. It is much harder in B +where there is no easy way to convert an environment into commands +that generate the environment. + + +=head3 env_parallel.bash / env_parallel.zsh / env_parallel.ksh / env_parallel.pdksh + +B sets the function B. It uses +B and B to dump the configuration (with a few +exceptions) into B<$PARALLEL_ENV> before running GNU B. + +After GNU B is finished, B<$PARALLEL_ENV> is deleted. + + +=head3 env_parallel.csh + +B has two purposes: If B is not an +alias: make it into an alias that sets B<$PARALLEL> with arguments +and calls B. + +If B is an alias, then B uses +B<$PARALLEL> as the arguments for GNU B. + +It exports the environment by writing a variable definition to a file +for each variable. The definitions of aliases are appended to this +file. Finally the file is put into B<$PARALLEL_ENV>. + +GNU B is then run and B<$PARALLEL_ENV> is deleted. + + +=head3 env_parallel.fish + +First all functions definitions are generated using a loop and +B. + +Dumping the scalar variable definitions is harder. + +B can represent non-printable characters in (at least) 2 +ways. To avoid problems all scalars are converted to \XX quoting. + +Then commands to generate the definitions are made and separated by +NUL. + +This is then piped into a Perl script that quotes all values. List +elements will be appended using two spaces. + +Finally \n is converted into \1 because B variables cannot +contain \n. GNU B will later convert all \1 from +B<$PARALLEL_ENV> into \n. + +This is then all saved in B<$PARALLEL_ENV>. + +GNU B is called, and B<$PARALLEL_ENV> is deleted. + + =head2 Job slots The easiest way to explain what GNU B does is to assume that @@ -64,10 +140,11 @@ been implemented as a stack with lazy evaluation: Draw one from an empty stack and the stack is extended by one. When a job is done, push the available job slot back on the stack. -This implementation also means that if you use remote executions, you -cannot assume that a given job slot will remain on the same remote -server. This goes double since number of job slots can be adjusted on -the fly (by giving B<--jobs> a file name). +This implementation also means that if you re-run the same jobs, you +cannot assume jobs will get the same slots. And if you use remote +executions, you cannot assume that a given job slot will remain on the +same remote server. This goes double since number of job slots can be +adjusted on the fly (by giving B<--jobs> a file name). =head2 Rsync protocol version @@ -84,12 +161,13 @@ Bs to talk to version 2.5.7. =head2 Compression -B<--compress> compresses the data in the temporary files. This is a -bit tricky because there should be no files to clean up if GNU -B is killed by a power outage. +GNU B buffers output in temporary files. B<--compress> +compresses the buffered data. This is a bit tricky because there +should be no files to clean up if GNU B is killed by a power +outage. GNU B first selects a compression program. If the user has not -selected one, the first of these that are in $PATH is used: B. They are sorted by speed on a 16 core machine. @@ -133,16 +211,17 @@ Local: B =item --cat - cat > {}; I {}; + cat > {}; <> {}; perl -e '$bash = shift; $csh = shift; for(@ARGV) { unlink;rmdir; } if($bash =~ s/h//) { exit $bash; } exit $csh;' "$?h" "$status" {}; -{} is set to $PARALLEL_TMP which is a tmpfile. The Perl script saves -the exit value, unlinks the tmpfile, and returns the exit value - no -matter if the shell is B (using $?) or B<*csh> (using $status). +{} is set to B<$PARALLEL_TMP> which is a tmpfile. The Perl script +saves the exit value, unlinks the tmpfile, and returns the exit value +- no matter if the shell is B/B/B (using $?) or +B<*csh>/B (using $status). =item --fifo @@ -161,35 +240,38 @@ matter if the shell is B (using $?) or B<*csh> (using $status). waitpid $pid,0; # Cleanup unlink $f; - exit $?/256;' I I $PARALLEL_TMP + exit $?/256;' <> -c <> $PARALLEL_TMP -This is an elaborate way of: mkfifo {}; run I in the -background using I; copying STDIN to {}; waiting for background -to complete; remove {} and exit with the exit code from I. +This is an elaborate way of: mkfifo {}; run I<<>> in the +background using I<<>>; copying STDIN to {}; waiting for background +to complete; remove {} and exit with the exit code from I<<>>. -It is made this way to be compatible with B<*csh>. +It is made this way to be compatible with B<*csh>/B. =item --pipepart - < file perl -e 'while(@ARGV) { + < <> perl -e 'while(@ARGV) { sysseek(STDIN,shift,0) || die; $left = shift; while($read = sysread(STDIN,$buf, ($left > 131072 ? 131072 : $left))){ $left -= $read; syswrite(STDOUT,$buf); } - }' startposition length + }' <> <> -This will read I bytes from I starting at -I and send it to STDOUT. +This will read I<<>> bytes from I<<>> starting at +I<<>> and send it to STDOUT. =item --sshlogin I -ssh I I +ssh I I + +Where I is the sshlogin and I is the +command quoted so it will be passed to the server. =item --transfer -( ssh I mkdir -p ./I;rsync --protocol 30 -rlDzR -essh ./{} I:./I ); I +( ssh I mkdir -p ./I;rsync --protocol 30 -rlDzR -essh ./{} I:./I ); I<<>> Read about B<--protocol 30> in the section B. @@ -203,7 +285,7 @@ Read about B<--protocol 30> in the section B. =item --return I -I; _EXIT_status=$?; mkdir -p I; rsync --protocol 30 --rsync-path=cd\ ./I\;\ rsync -rlDzR -essh I:./I ./I; exit $_EXIT_status; +I<<>>; _EXIT_status=$?; mkdir -p I<<>>; rsync --protocol 30 --rsync-path=cd\ ./I<<>>\;\ rsync -rlDzR -essh I<<>>:./I<<>> ./I<<>>; exit $_EXIT_status; The B<--rsync-path=cd ...> is needed because old versions of B do not support B<--no-implied-dirs>. @@ -214,7 +296,7 @@ wrapping 'sh -c' is enough? =item --cleanup -I _EXIT_status=$?; <> +I<<>> _EXIT_status=$?; <>; ssh I \(rm\ -f\ ./I/{}\;\ rmdir\ ./I\ \>\&/dev/null\;\); exit $_EXIT_status; diff --git a/testsuite/tests-to-run/parallel-local-ssh4.sh b/testsuite/tests-to-run/parallel-local-ssh4.sh index 3cafdb36..6630d352 100644 --- a/testsuite/tests-to-run/parallel-local-ssh4.sh +++ b/testsuite/tests-to-run/parallel-local-ssh4.sh @@ -90,6 +90,7 @@ echo '### Test tmux works on different shells' stdout ssh parallel@lo "$PARTMUX" 'false ::: 1 2 3 4; echo $?' | grep -v 'See output'; stdout ssh tcsh@lo "$PARTMUX" 'true ::: 1 2 3 4; echo $status' | grep -v 'See output'; stdout ssh tcsh@lo "$PARTMUX" 'false ::: 1 2 3 4; echo $status' | grep -v 'See output'; + echo "# command is currently too long for csh. Maybe it can be fixed?"; stdout ssh csh@lo "$PARTMUX" 'true ::: 1 2 3 4; echo $status' | grep -v 'See output'; stdout ssh csh@lo "$PARTMUX" 'false ::: 1 2 3 4; echo $status' | grep -v 'See output' @@ -131,7 +132,7 @@ echo "### Bash environment" #stdout ssh -t lo <<'EOS' myvar="myvar works" funky=$(perl -e 'print pack "c*", 1..255') -myarray=('' array_val2 3 '' 5) +myarray=('' array_val2 3 '' 5 ' space 6 ') declare -A assocarr assocarr[a]=assoc_val_a assocarr[b]=assoc_val_b @@ -139,7 +140,7 @@ alias alias_echo="echo 3 arg"; func_echo() { echo $*; echo "$myvar" - echo ${myarray[1]} + echo "${myarray[5]}" echo ${assocarr[a]} echo Funky-"$funky"-funky } @@ -158,7 +159,7 @@ echo "### Zsh environment" stdout ssh -q zsh@lo <<'EOS' | egrep -v 'Welcome to |packages can be updated|security updates' myvar="myvar works" funky=$(perl -e 'print pack "c*", 1..255') -myarray=('' array_val2 3 '' 5) +myarray=('' array_val2 3 '' 5 ' space 6 ') declare -A assocarr assocarr[a]=assoc_val_a assocarr[b]=assoc_val_b @@ -166,7 +167,7 @@ alias alias_echo="echo 3 arg"; func_echo() { echo $*; echo "$myvar" - echo $myarray[2] + echo "$myarray[6]" echo ${assocarr[a]} echo Funky-"$funky"-funky } @@ -186,7 +187,7 @@ echo "### Ksh environment" stdout ssh -q ksh@lo <<'EOS' | egrep -v 'Welcome to |packages can be updated|security updates' myvar="myvar works" funky=$(perl -e 'print pack "c*", 1..255') -myarray=('' array_val2 3 '' 5) +myarray=('' array_val2 3 '' 5 ' space 6 ') typeset -A assocarr assocarr[a]=assoc_val_a assocarr[b]=assoc_val_b @@ -195,7 +196,7 @@ alias alias_echo="echo 3 arg"; func_echo() { echo $*; echo "$myvar" - echo ${myarray[1]} + echo "${myarray[5]}" echo ${assocarr[a]} echo Funky-"$funky"-funky } @@ -217,7 +218,8 @@ setenv myenvvar "myenvvar works" set funky (perl -e 'print pack "c*", 1..255') setenv funkyenv (perl -e 'print pack "c*", 1..255') -set myarray '' array_val2 3 '' 5 +set myarray '' array_val2 3 '' 5 ' space 6 ' + # Assoc arrays do not exist #typeset -A assocarr #assocarr[a]=assoc_val_a @@ -228,14 +230,14 @@ function func_echo echo $argv; echo "$myvar" echo "$myenvvar" - echo $myarray[2] + echo "$myarray[6]" # Assoc arrays do not exist in fish # echo ${assocarr[a]} echo echo echo echo Funky-"$funky"-funky - echo Funky-"$funkyenv"-funky + echo Funkyenv-"$funkyenv"-funkyenv echo echo echo @@ -251,23 +253,15 @@ EOS echo echo "### csh environment" -# http://hyperpolyglot.org/unix-shells -# makealias: -# alias quote "/bin/sed -e 's/\\!/\\\\\!/g' -e 's/'\\\''/'\\\'\\\\\\\'\\\''/g' -e 's/^/'\''/' -e 's/"\$"/'\''/'" -# alias makealias "quote | /bin/sed 's/^/alias \!:1 /' \!:2*" -# -# makealias_with_newline -# perl -e '$/=undef;$_=<>;s/\n/\\\n/g;s/\047/\047\042\047\042\047/g;print' - stdout ssh -q csh@lo <<'EOS' | egrep -v 'Welcome to |packages can be updated|security updates' set myvar = "myvar works" -set funky = "`perl -e 'print pack q(c*), 1..255'`" -set myarray = ('' 'array_val2' '3' '' '5') +set funky = "`perl -e 'print pack q(c*), 2..255'`" +set myarray = ('' 'array_val2' '3' '' '5' ' space 6 ') #declare -A assocarr #assocarr[a]=assoc_val_a #assocarr[b]=assoc_val_b alias alias_echo echo 3 arg; -alias alias_echo_var 'echo $argv; echo $myvar; echo ${myarray[2]}; echo Funky-"$funky"-funky' +alias alias_echo_var 'echo $argv; echo "$myvar"; echo "${myarray[4]} special chars problem"; echo Funky-"$funky"-funky' #function func_echo # echo $argv; @@ -277,80 +271,11 @@ alias alias_echo_var 'echo $argv; echo $myvar; echo ${myarray[2]}; echo Funky-"$ # echo Funky-"$funky"-funky #end -# ALIAS TO EXPORT ALIASES: - -# Quote ' by putting it inside " -# s/'/'"'"'/g; -# ' => \047 " => \042 -# s/\047/\047\042\047\042\047/g; -# Quoted: s/\\047/\\047\\042\\047\\042\\047/g\; - -# Remove () from second column -# s/^(\S+)(\s+)\((.*)\)/\1\2\3/ -# \047 => ' -# s/^(\S+)(\s+)\((.*)\)/\1\2\3/; -# Quoted: s/\^\(\\S+\)\(\\s+\)\\\(\(.\*\)\\\)/\\1\\2\\3/\; - -# Add ' around second column -# s/^(\S+)(\s+)(.*)/\1\2'\3'/ -# \047 => ' -# s/^(\S+)(\s+)(.*)/\1\2\047\3\047/; -# Quoted: s/\^\(\\S+\)\(\\s+\)\(.\*\)/\\1\\2\\047\\3\\047/\; - -# Quote ! as \! -# s/\!/\\\!/g; -# Quoted: s/\\\!/\\\\\\\!/g; - -# Prepend with "\nalias " -# s/^/\001alias /; -# Quoted: s/\^/\\001alias\ /\; - -#!# alias env_parallel 'setenv PARALLEL_ENV "`alias | perl -pe s/\\047/\\047\\042\\047\\042\\047/g\;s/\^\(\\S+\)\(\\s+\)\\\(\(.\*\)\\\)/\\1\\2\\3/\;s/\^\(\\S+\)\(\\s+\)\(.\*\)/\\1\\2\\047\\3\\047/\;s/\^/\\001alias\ /\;s/\\\!/\\\\\\\!/g;`";parallel \!*; setenv PARALLEL_ENV' - - -## set tmpfile=`tempfile` -## foreach v (`set | awk -e '{print $1}' |grep -v prompt2`); -## eval if'($?'$v' && ${#'$v'} <= 1) echo scalar'$v'="$'$v'"' >> $tmpfile; -## eval if'($?'$v' && ${#'$v'} > 1) echo array'$v'="$'$v'"' >> $tmpfile; -## end -## cat $tmpfile | parallel --shellquote | perl -pe 's/^scalar(\S+).=/set $1=/ or s/^array(\S+).=(.*)/set $1=($2)/ && s/\\ / /g;'; rm $tmpfile -## -## set tmpfile=`tempfile` -## foreach _vARnAmE (`set | awk -e '{print $1}' |grep -v prompt2`); -## eval if'($?'$_vARnAmE' && ${#'$_vARnAmE'} <= 1) echo scalar'$_vARnAmE'="$'$_vARnAmE'"' >> $tmpfile; eval if'($?'$_vARnAmE' && ${#'$_vARnAmE'} > 1) echo array'$_vARnAmE'="$'$_vARnAmE'"' >> $tmpfile; -## end -## cat $tmpfile | parallel --shellquote | perl -pe 's/^scalar(\S+).=/set $1=/ or s/^array(\S+).=(.*)/set $1=($2)/ && s/\\ / /g;'; rm $tmpfile; unset tmpfile -## -## #!/bin/csh -## -## set _tmpfile=`tempfile`; -## foreach _vARnAmE (`set | awk -e '{print $1}' |grep -Ev 'prompt2|_tmpfile'`); -## eval if'($?'$_vARnAmE' && ${#'$_vARnAmE'} <= 1) echo scalar'$_vARnAmE'="$'$_vARnAmE'"' >> $_tmpfile; -## eval if'($?'$_vARnAmE' && ${#'$_vARnAmE'} > 1) echo array'$_vARnAmE'="$'$_vARnAmE'"' >> $_tmpfile; -## end -## setenv PARALLEL_ENV `cat $_tmpfile | parallel --shellquote | perl -pe 's/^scalar(\S+).=/set $1=/ or s/^array(\S+).=(.*)/set $1=($2)/ && s/\\ / /g; s/$/\001/';` -## rm $_tmpfile; -## unset _tmpfile -## -## setenv PARALLEL_ENV "$PARALLEL_ENV`alias | perl -pe s/\\047/\\047\\042\\047\\042\\047/g\;s/\^\(\\S+\)\(\\s+\)\\\(\(.\*\)\\\)/\\1\\2\\3/\;s/\^\(\\S+\)\(\\s+\)\(.\*\)/\\1\\2\\047\\3\\047/\;s/\^/\\001alias\ /\;s/\\\!/\\\\\\\!/g;`" -## parallel \!* -## setenv PARALLEL_ENV -## -## -## perl -e '$/=undef;$_=<>;s/\n/\\\\\n/g;s/\047/\047\042\047\042\047/g;print "eval \047$_\047"' -## -## foreach g (h i j) -## echo $g -## end -## -## - - env_parallel alias_echo ::: alias_works -env_parallel alias_echo_var ::: alias_var_does_not_work +env_parallel alias_echo_var ::: alias_var_works env_parallel func_echo ::: function_does_not_work env_parallel -S csh@lo alias_echo ::: alias_works_over_ssh -env_parallel -S csh@lo alias_echo_var ::: alias_var_does_not_work +env_parallel -S csh@lo alias_echo_var ::: alias_var_works_over_ssh env_parallel -S csh@lo func_echo ::: function_does_not_work_over_ssh echo echo "$funky" | parallel --shellquote diff --git a/testsuite/tests-to-run/parallel-tutorial.sh b/testsuite/tests-to-run/parallel-tutorial.sh index d428ce38..037963d3 100644 --- a/testsuite/tests-to-run/parallel-tutorial.sh +++ b/testsuite/tests-to-run/parallel-tutorial.sh @@ -66,7 +66,7 @@ perl -ne '$/="\n\n"; /^Output/../^[^O]\S/ and next; /^ / and print;' ../../src/ # Due to order is often mixed up s/echo \d; exit \d\n/echo X; exit X\n/; # Race condition causes outdir to sometime exist - s/(std(out|err):) Permission denied/$1 No such file or directory/; + s/(std(out|err)|seq): Permission denied/$1: No such file or directory/; ' # 3+3 .par files (from --files), 1 .tms-file from tmux attach find {$TMPDIR,/var/tmp,/tmp}/{fif,tms,par[^a]}* -mmin -10 2>/dev/null | wc -l diff --git a/testsuite/wanted-results/parallel-install b/testsuite/wanted-results/parallel-install index 09734428..c1474be0 100644 --- a/testsuite/wanted-results/parallel-install +++ b/testsuite/wanted-results/parallel-install @@ -175,7 +175,7 @@ pod2pdf --output-file ./parallel_design.pdf ./parallel_design.pod --title "GNU P Warning: pod2pdf not found. Using old parallel_design.pdf make[0]: Entering directory `/tmp/parallel-00000000/src' /bin/mkdir -p '/usr/local/bin' - /usr/bin/install -c parallel sql niceload env_parallel env_parallel.bash env_parallel.zsh env_parallel.fish env_parallel.ksh env_parallel.pdksh env_parallel.csh '/usr/local/bin' + /usr/bin/install -c parallel sql niceload env_parallel env_parallel.bash env_parallel.zsh env_parallel.fish env_parallel.ksh env_parallel.pdksh env_parallel.csh env_parallel.tcsh '/usr/local/bin' make install-exec-hook make[0]: Entering directory `/tmp/parallel-00000000/src' rm /usr/local/bin/sem || true diff --git a/testsuite/wanted-results/parallel-local-ssh4 b/testsuite/wanted-results/parallel-local-ssh4 index eafb3575..8d187260 100644 --- a/testsuite/wanted-results/parallel-local-ssh4 +++ b/testsuite/wanted-results/parallel-local-ssh4 @@ -81,14 +81,23 @@ echo '### Test tmux works on different shells' 0 (stdout parallel -Scsh@lo,tcsh@lo,parallel@lo,zsh@lo --tmux false ::: 1 2 3 4; echo $?) | grep -v 'See output'; 4 - export PARTMUX='parallel -Scsh@lo,tcsh@lo,parallel@lo,zsh@lo --tmux '; stdout ssh zsh@lo "$PARTMUX" 'true ::: 1 2 3 4; echo $status' | grep -v 'See output'; stdout ssh zsh@lo "$PARTMUX" 'false ::: 1 2 3 4; echo $status' | grep -v 'See output'; stdout ssh parallel@lo "$PARTMUX" 'true ::: 1 2 3 4; echo $?' | grep -v 'See output'; stdout ssh parallel@lo "$PARTMUX" 'false ::: 1 2 3 4; echo $?' | grep -v 'See output'; stdout ssh tcsh@lo "$PARTMUX" 'true ::: 1 2 3 4; echo $status' | grep -v 'See output'; stdout ssh tcsh@lo "$PARTMUX" 'false ::: 1 2 3 4; echo $status' | grep -v 'See output'; stdout ssh csh@lo "$PARTMUX" 'true ::: 1 2 3 4; echo $status' | grep -v 'See output'; stdout ssh csh@lo "$PARTMUX" 'false ::: 1 2 3 4; echo $status' | grep -v 'See output' + export PARTMUX='parallel -Scsh@lo,tcsh@lo,parallel@lo,zsh@lo --tmux '; stdout ssh zsh@lo "$PARTMUX" 'true ::: 1 2 3 4; echo $status' | grep -v 'See output'; stdout ssh zsh@lo "$PARTMUX" 'false ::: 1 2 3 4; echo $status' | grep -v 'See output'; stdout ssh parallel@lo "$PARTMUX" 'true ::: 1 2 3 4; echo $?' | grep -v 'See output'; stdout ssh parallel@lo "$PARTMUX" 'false ::: 1 2 3 4; echo $?' | grep -v 'See output'; stdout ssh tcsh@lo "$PARTMUX" 'true ::: 1 2 3 4; echo $status' | grep -v 'See output'; stdout ssh tcsh@lo "$PARTMUX" 'false ::: 1 2 3 4; echo $status' | grep -v 'See output'; echo "# command is currently too long for csh. Maybe it can be fixed?"; stdout ssh csh@lo "$PARTMUX" 'true ::: 1 2 3 4; echo $status' | grep -v 'See output'; stdout ssh csh@lo "$PARTMUX" 'false ::: 1 2 3 4; echo $status' | grep -v 'See output' 0 4 0 4 0 4 -0 +# command is currently too long for csh. Maybe it can be fixed? +Word too long. +Word too long. +Word too long. +Word too long. +4 +Word too long. +Word too long. +Word too long. +Word too long. 4 echo '### works' ### works @@ -133,7 +142,7 @@ Test env_parallel: 3 arg alias_works function_works myvar works -array_val2 + space 6 assoc_val_a Funky- @@ -143,7 +152,7 @@ Funky- 3 arg alias_works_over_ssh function_works_over_ssh myvar works -array_val2 + space 6 assoc_val_a Funky- @@ -162,7 +171,7 @@ Funky- zsh:130: command not found: alias_echo function_works myvar works -array_val2 + space 6 assoc_val_a Funky-  @@ -170,7 +179,7 @@ Funky- zsh:130: command not found: alias_echo function_works_over_ssh myvar works -array_val2 + space 6 assoc_val_a Funky-  @@ -187,14 +196,14 @@ Funky- 3 arg alias_works function_works myvar works -array_val2 + space 6 assoc_val_a Funky-  !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~-funky 3 arg alias_works_over_ssh function_works_over_ssh myvar works -array_val2 + space 6 assoc_val_a Funky-  !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~-funky @@ -215,12 +224,12 @@ env_parallel: Warning: ASCII value 1 in variables is not supported function_works myvar works myenvvar works -3 + space 6 Funky-   !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~-funky -Funky-   !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~-funky +Funkyenv-   !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~-funkyenv @@ -232,12 +241,12 @@ env_parallel: Warning: ASCII value 1 in variables is not supported function_works_over_ssh myvar works myenvvar works -3 + space 6 Funky-   !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~-funky -Funky-   !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~-funky +Funkyenv-   !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~-funkyenv @@ -253,9 +262,13 @@ Warning: no access to tty (Bad file descriptor). Thus no job control in this shell. 3 arg alias_works -myvar: Undefined variable. +myvar works +space special chars problem +Funky-  !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~-funky alias_var_works func_echo: Command not found. 3 arg alias_works_over_ssh -myvar: Undefined variable. +myvar works +space special chars problem +Funky-  !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~-funky alias_var_works_over_ssh func_echo: Command not found. -\\\\\\\\ \ \ \ \ \\\\\\\\\\\\\\ \!\"\#\$%\&\'\(\)\*+,-./0123456789:\;\<\=\>\?@ABCDEFGHIJKLMNOPQRSTUVWXYZ\[\\\]\^_\`abcdefghijklmnopqrstuvwxyz\{\|\}\~\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\ +\\\\\\\\ \ \ \ \ \\\\\\\\\\\\\\ \!\"\#\$%\&\'\(\)\*+,-./0123456789:\;\<\=\>\?@ABCDEFGHIJKLMNOPQRSTUVWXYZ\[\\\]\^_\`abcdefghijklmnopqrstuvwxyz\{\|\}\~\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\ diff --git a/testsuite/wanted-results/parallel-tutorial b/testsuite/wanted-results/parallel-tutorial index 7adffaeb..8b9b1d21 100644 --- a/testsuite/wanted-results/parallel-tutorial +++ b/testsuite/wanted-results/parallel-tutorial @@ -407,13 +407,13 @@ C outdir/1/C/seq outdir/1/C/stderr outdir/1/C/stdout -/bin/bash: outdir/1/A/seq: Permission denied +/bin/bash: outdir/1/A/seq: No such file or directory /bin/bash: line 1: outdir/1/A/stderr: No such file or directory /bin/bash: line 2: outdir/1/A/stdout: No such file or directory -/bin/bash: line 3: outdir/1/B/seq: Permission denied +/bin/bash: line 3: outdir/1/B/seq: No such file or directory /bin/bash: line 4: outdir/1/B/stderr: No such file or directory /bin/bash: line 5: outdir/1/B/stdout: No such file or directory -/bin/bash: line 6: outdir/1/C/seq: Permission denied +/bin/bash: line 6: outdir/1/C/seq: No such file or directory /bin/bash: line 7: outdir/1/C/stderr: No such file or directory /bin/bash: line 8: outdir/1/C/stdout: No such file or directory parallel --header : --results outdir echo ::: f1 A B ::: f2 C D @@ -734,7 +734,11 @@ pdksh: Put this in /home/tange/.profile: source /usr/local/bin/env_parallel.pdk csh: Put this in /home/tange/.cshrc: source /usr/local/bin/env_parallel.csh E.g. by doing: echo 'source /usr/local/bin/env_parallel.csh' >> /home/tange/.cshrc - Supports: aliases + Supports: aliases, variables, arrays with no special chars + +tcsh: Put this in /home/tange/.tcshrc: source /usr/local/bin/env_parallel.tcsh + E.g. by doing: echo 'source /usr/local/bin/env_parallel.tcsh' >> /home/tange/.tcshrc + Supports: aliases, variables, arrays with no special chars For details: see man env_parallel