From 8f63e3f5e752336d709af0f9b2d123c386aa0981 Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Fri, 1 Dec 2017 20:53:56 +0100 Subject: [PATCH] env_parallel: Aliases with newlines either supported (ash/dash/zsh) or warning (bash). testsuite: functionized freebsd tests. --- doc/release_new_version | 18 +- src/env_parallel.ash | 6 +- src/env_parallel.bash | 14 +- src/env_parallel.ksh | 3 +- src/env_parallel.pod | 53 ++++- src/env_parallel.sh | 8 +- src/env_parallel.zsh | 33 +-- src/parallel | 4 + src/parallel.pod | 8 +- src/parallel_alternatives.pod | 10 + src/parallel_book.pod | 52 +++++ src/sql | 2 +- testsuite/tests-to-run/parallel-freebsd.sh | 133 +++++++---- testsuite/tests-to-run/parallel-local-ssh7.sh | 213 +++++++++++------- testsuite/wanted-results/parallel-freebsd | 125 +++++----- testsuite/wanted-results/parallel-local-ssh7 | 175 ++++++++++---- 16 files changed, 571 insertions(+), 286 deletions(-) diff --git a/doc/release_new_version b/doc/release_new_version index 5829fef9..4f792488 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -205,23 +205,15 @@ GNU Parallel 20171122 ('Mugabe') <<[stable]>> has been released. It is available <> -Poem of the month: - - An ode to GNU parallel - An ode to GNU parallel - An ode to GNU parallel - An ode to GNU parallel - An ode to GNU parallel - An ode to GNU parallel - -- Adam Stuckert PoisonEcology@twitter +Haiku of the month: + <<>> New in this release: -* Using GNU Parallel to speed up Google Cloud Infrastructure management https://medium.com/@pczarkowski/using-gnu-parallel-to-speed-up-google-cloud-infrastructure-management-53e5c555ec05 +parset for ksh, zsh, +https://docs.computecanada.ca/wiki/GNU_Parallel +https://qiita.com/inouet/items/bfc208668c86caf8ff74 -* What is gnu parallel http://wiki.chpc.ac.za/howto:bioinformatics:gnu-parallel - -* Parallel, Scripts, Clusters, & Easy Use https://chiefio.wordpress.com/2017/11/10/parallel-scripts-clusters-easy-use/ <> diff --git a/src/env_parallel.ash b/src/env_parallel.ash index e6fa231b..81dc5a74 100755 --- a/src/env_parallel.ash +++ b/src/env_parallel.ash @@ -29,10 +29,12 @@ env_parallel() { # based on env_parallel.sh _names_of_ALIASES() { - alias | perl -pe 's/^alias //;s/=.*//' + alias | perl -ne 's/^alias //;s/^(\S+)=.*/$1/ and print' } _bodies_of_ALIASES() { - alias "$@" | perl -pe 's/^(alias )?/alias /' + for _i in "$@"; do + echo 'alias '"`alias $_i`" + done } _names_of_maybe_FUNCTIONS() { set | perl -ne '/^(\S+)\(\)\{$/ and print "$1\n"' diff --git a/src/env_parallel.bash b/src/env_parallel.bash index c0ff69d0..922a5845 100755 --- a/src/env_parallel.bash +++ b/src/env_parallel.bash @@ -32,6 +32,16 @@ env_parallel() { compgen -a } _bodies_of_ALIASES() { + local _i + for _i in $@; do + if [ $(alias $_i | wc -l) == 1 ] ; then + true Alias is a single line. Good. + else + _warning "Alias '$_i' contains newline." + _warning "Make sure the command has at least one newline after '$_i'." + _warning "See BUGS in 'man env_parallel'." + fi + done alias "$@" } _names_of_FUNCTIONS() { @@ -96,7 +106,9 @@ env_parallel() { print $vars ? "($vars)" : "(.*)"; ' -- "$@" } - + _warning() { + echo "env_parallel: Warning: $@" >&2 + } # Bash is broken in version 3.2.25 and 4.2.39 # The crazy '[ "`...`" == "" ]' is needed for the same reason diff --git a/src/env_parallel.ksh b/src/env_parallel.ksh index 3b32c86c..e1ff5210 100755 --- a/src/env_parallel.ksh +++ b/src/env_parallel.ksh @@ -225,7 +225,8 @@ _parset_main() { } exit $exitval; ' || return 255 - # Internal grep gives wrong exit code in Ksh + # Built-in grep gives wrong exit code in Ksh + # Use \grep to force using non-built-in if echo "$_parset_name" | \grep -E ',| ' >/dev/null ; then # $1 contains , or space # Split on , or space to get the names diff --git a/src/env_parallel.pod b/src/env_parallel.pod index e08ce3ea..ec44eb30 100644 --- a/src/env_parallel.pod +++ b/src/env_parallel.pod @@ -24,7 +24,7 @@ B is beta quality and not production ready, but please use it for everyday use and report bugs. B is 0.1 sec slower at startup than pure GNU -B, and takes up to 15 ms to start a job. +B, and takes up to 30% longer to start a job (typically 15 ms). Due to the problem with environment space (see below) the recommended usage is: @@ -111,6 +111,13 @@ E.g. by doing: env_parallel --env myecho myecho ::: work env_parallel --env myecho -S server myecho ::: work + alias multiline='echo multiline + echo aliases' + env_parallel multiline ::: work + env_parallel -S server multiline ::: work + env_parallel --env multiline multiline ::: work + env_parallel --env multiline -S server multiline ::: work + =item functions ash cannot list defined functions - thus is not supported. @@ -154,6 +161,17 @@ E.g. by doing: env_parallel --env myecho myecho ::: work env_parallel --env myecho -S server myecho ::: work + alias multiline='echo multiline + echo aliases' + env_parallel 'multiline {}; + echo but only when followed by a newline' ::: work + env_parallel -S server 'multiline {}; + echo but only when followed by a newline' ::: work + env_parallel --env multiline 'multiline {}; + echo but only when followed by a newline' ::: work + env_parallel --env multiline -S server 'multiline {}; + echo but only when followed by a newline' ::: work + =item functions myfunc() { echo functions $*; } @@ -180,6 +198,10 @@ E.g. by doing: =back +=head3 BUGS + +Due to a bug in Bash aliases containing newlines must be followed by a +newline in the command. =head2 csh @@ -257,6 +279,13 @@ E.g. by doing: env_parallel --env myecho myecho ::: work env_parallel --env myecho -S server myecho ::: work + alias multiline='echo multiline + echo aliases' + env_parallel multiline ::: work + env_parallel -S server multiline ::: work + env_parallel --env multiline multiline ::: work + env_parallel --env multiline -S server multiline ::: work + =item functions dash cannot list defined functions - thus is not supported. @@ -355,6 +384,13 @@ E.g. by doing: env_parallel --env myecho myecho ::: work env_parallel --env myecho -S server myecho ::: work + alias multiline='echo multiline + echo aliases' + env_parallel multiline ::: work + env_parallel -S server multiline ::: work + env_parallel --env multiline multiline ::: work + env_parallel --env multiline -S server multiline ::: work + =item functions myfunc() { echo functions $*; } @@ -554,6 +590,13 @@ E.g. by doing: env_parallel --env myecho myecho ::: work env_parallel --env myecho -S server myecho ::: work + alias multiline='echo multiline + echo aliases' + env_parallel multiline ::: work + env_parallel -S server multiline ::: work + env_parallel --env multiline multiline ::: work + env_parallel --env multiline -S server multiline ::: work + =item functions myfunc() { echo functions $*; } @@ -588,7 +631,7 @@ Same as GNU B. =head1 AUTHOR -When using GNU B for a publication please cite: +When using GNU B for a publication please cite: O. Tange (2011): GNU Parallel - The Command-Line Power Tool, ;login: The USENIX Magazine, February 2011:42-47. @@ -603,14 +646,10 @@ Copyright (C) 2008,2009,2010 Ole Tange, http://ole.tange.dk Copyright (C) 2010,2011,2012,2013,2014,2015,2016,2017 Ole Tange, http://ole.tange.dk and Free Software Foundation, Inc. -Parts of the manual concerning B compatibility is inspired by -the manual of B from GNU findutils 4.4.2. - =head1 LICENSE -Copyright (C) 2016 -Ole Tange and Free Software Foundation, Inc. +Copyright (C) 2016,2017 Free Software Foundation, Inc. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by diff --git a/src/env_parallel.sh b/src/env_parallel.sh index bfdd0cba..83f40c0c 100755 --- a/src/env_parallel.sh +++ b/src/env_parallel.sh @@ -35,7 +35,7 @@ env_parallel() { alias "$@" | perl -pe 's/^(alias )?/alias /' } _names_of_maybe_FUNCTIONS() { - set | perl -ne '/^(\S+)\(\)\{$/ and print "$1\n"' + set | perl -ne '/^([A-Z_0-9]+)\s*\(\)\s*\{?$/i and print "$1\n"' } _names_of_FUNCTIONS() { # myfunc is a function @@ -132,7 +132,7 @@ env_parallel() { fi # Grep alias names - _alias_NAMES="`_names_of_ALIASES | _remove_bad_NAMES`" + _alias_NAMES="`_names_of_ALIASES | _remove_bad_NAMES | xargs echo`" _list_alias_BODIES="_bodies_of_ALIASES $_alias_NAMES" if [ "$_alias_NAMES" = "" ] ; then # no aliases selected @@ -141,7 +141,7 @@ env_parallel() { unset _alias_NAMES # Grep function names - _function_NAMES="`_names_of_FUNCTIONS | _remove_bad_NAMES`" + _function_NAMES="`_names_of_FUNCTIONS | _remove_bad_NAMES | xargs echo`" _list_function_BODIES="_bodies_of_FUNCTIONS $_function_NAMES" if [ "$_function_NAMES" = "" ] ; then # no functions selected @@ -150,7 +150,7 @@ env_parallel() { unset _function_NAMES # Grep variable names - _variable_NAMES="`_names_of_VARIABLES | _remove_bad_NAMES`" + _variable_NAMES="`_names_of_VARIABLES | _remove_bad_NAMES | xargs echo`" _list_variable_VALUES="_bodies_of_VARIABLES $_variable_NAMES" if [ "$_variable_NAMES" = "" ] ; then # no variables selected diff --git a/src/env_parallel.zsh b/src/env_parallel.zsh index 00931f8c..f60912f5 100755 --- a/src/env_parallel.zsh +++ b/src/env_parallel.zsh @@ -32,19 +32,22 @@ env_parallel() { print -l ${(k)aliases} } _bodies_of_ALIASES() { - echo "alias "$(echo "$@"|xargs)" | perl -pe 's/^/alias /'" + local _i + for _i ($@); do + echo 'alias '"$(alias $_i)" + done } _names_of_FUNCTIONS() { print -l ${(k)functions} } _bodies_of_FUNCTIONS() { - echo "typeset -f "$(echo "$@"|xargs) + typeset -f "$@" } _names_of_VARIABLES() { print -l ${(k)parameters} } _bodies_of_VARIABLES() { - echo typeset -p "$(echo $@|xargs)" '| grep -aFvf <(typeset -pr)' + typeset -p "$@" } _remove_bad_NAMES() { # Do not transfer vars and funcs from env_parallel @@ -53,7 +56,8 @@ env_parallel() { grep -E "^$_grep_REGEXP"\$ | grep -vE "^$_ignore_UNDERSCORE"\$ | grep -v '=' | grep -Ev '^([-?#!$*@_0]|zsh_eval_context|ZSH_EVAL_CONTEXT|LINENO|IFS|commands|functions|options|aliases|EUID|EGID|UID|GID)$' | - grep -Ev '^(dis_patchars|patchars|terminfo|funcstack|galiases|keymaps|parameters|jobdirs|dirstack|functrace|funcsourcetrace|zsh_scheduled_events|dis_aliases|dis_reswords|dis_saliases|modules|reswords|saliases|widgets|userdirs|historywords|nameddirs|termcap|dis_builtins|dis_functions|jobtexts|funcfiletrace|dis_galiases|builtins|history|jobstates)$' + grep -Ev '^(dis_patchars|patchars|terminfo|funcstack|galiases|keymaps|parameters|jobdirs|dirstack|functrace|funcsourcetrace|zsh_scheduled_events|dis_aliases|dis_reswords|dis_saliases|modules|reswords|saliases|widgets|userdirs|historywords|nameddirs|termcap|dis_builtins|dis_functions|jobtexts|funcfiletrace|dis_galiases|builtins|history|jobstates)$' | + grep -aFvf <(typeset -pr) } _get_ignored_VARS() { @@ -93,13 +97,13 @@ env_parallel() { if which parallel | grep 'no parallel in' >/dev/null; then echo 'env_parallel: Error: parallel must be in $PATH.' >&2 - return 1 + return 255 fi if which parallel >/dev/null; then true which on linux else echo 'env_parallel: Error: parallel must be in $PATH.' >&2 - return 1 + return 255 fi # Grep regexp for vars given by --env @@ -118,10 +122,10 @@ env_parallel() { cat > $HOME/.parallel/ignored_vars return 0 fi - + # Grep alias names - _alias_NAMES="`_names_of_ALIASES | _remove_bad_NAMES`" - _list_alias_BODIES="alias "$(echo $_alias_NAMES|xargs)" | perl -pe 's/^/alias /'" + _alias_NAMES="`_names_of_ALIASES | _remove_bad_NAMES | xargs echo`" + _list_alias_BODIES="_bodies_of_ALIASES $_alias_NAMES" if [ "$_alias_NAMES" = "" ] ; then # no aliases selected _list_alias_BODIES="true" @@ -129,8 +133,8 @@ env_parallel() { unset _alias_NAMES # Grep function names - _function_NAMES="`_names_of_FUNCTIONS | _remove_bad_NAMES`" - _list_function_BODIES="typeset -f "$(echo $_function_NAMES|xargs) + _function_NAMES="`_names_of_FUNCTIONS | _remove_bad_NAMES | xargs echo`" + _list_function_BODIES="_bodies_of_FUNCTIONS $_function_NAMES" if [ "$_function_NAMES" = "" ] ; then # no functions selected _list_function_BODIES="true" @@ -138,10 +142,8 @@ env_parallel() { unset _function_NAMES # Grep variable names - _variable_NAMES="`_names_of_VARIABLES | _remove_bad_NAMES`" - _list_variable_VALUES="typeset -p "$(echo $_variable_NAMES|xargs)" | - grep -aFvf <(typeset -pr) - " + _variable_NAMES="`_names_of_VARIABLES | _remove_bad_NAMES | xargs echo`" + _list_variable_VALUES="_bodies_of_VARIABLES $_variable_NAMES" if [ "$_variable_NAMES" = "" ] ; then # no variables selected _list_variable_VALUES="true" @@ -233,4 +235,3 @@ _parset_main() { perl -pe 'chop;$_="\"\`cat $_; rm $_\`\" "' ) )" fi } - diff --git a/src/parallel b/src/parallel index 920cf2f8..ac869b5d 100755 --- a/src/parallel +++ b/src/parallel @@ -7484,6 +7484,10 @@ sub wrapped { my $self = shift; if(not defined $self->{'wrapped'}) { my $command = $self->replaced(); + # Bug in Bash and Ksh when running multiline aliases + # This will force them to run correctly, but will fail in + # tcsh so we do not do it. + # $command .= "\n\n"; if($opt::shellquote) { # Prepend /bin/echo (echo no-/bin is wrong in csh) # and quote twice diff --git a/src/parallel.pod b/src/parallel.pod index 9976edfa..7b452142 100644 --- a/src/parallel.pod +++ b/src/parallel.pod @@ -1042,7 +1042,7 @@ B<-l 0> is an alias for B<-l 1>. Implies B<-X> unless B<-m>, B<--xargs>, or B<--pipe> is set. -=item B<--limit> "I I" (beta testing) +=item B<--limit> "I I" Dynamic job limit. Before starting a new job run I with I. The exit value of I determines what GNU B @@ -1915,7 +1915,7 @@ You can even use multiple matches: See also: B<{= perl expression =}> B<--parens> -=item B<--rsync-opts> I (beta testing) +=item B<--rsync-opts> I Options to pass on to B. Setting B<--rsync-opts> takes precedence over setting the environment variable $PARALLEL_RSYNC_OPTS. @@ -4699,8 +4699,8 @@ the manual of B from GNU findutils 4.4.2. =head1 LICENSE -Copyright (C) 2007,2008,2009,2010,2011,2012,2013,2014,2015,2016 Free -Software Foundation, Inc. +Copyright (C) 2007,2008,2009,2010,2011,2012,2013,2014,2015,2016,2017 +Free Software Foundation, Inc. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by diff --git a/src/parallel_alternatives.pod b/src/parallel_alternatives.pod index 4cb279a6..ff2ec316 100644 --- a/src/parallel_alternatives.pod +++ b/src/parallel_alternatives.pod @@ -173,6 +173,16 @@ this. To see this in action try: ls -l out* md5sum out* +Or try this: + + slow_seq() { + seq "$@" | + perl -ne '$|=1; for(split//){ print; select($a,$a,$a,0.100);}' + } + export -f slow_seq + seq 5 | xargs -n1 -P0 -I {} bash -c 'slow_seq {}' + seq 5 | parallel -P0 slow_seq {} + B has no support for keeping the order of the output, therefore if running jobs in parallel using B the output of the second job cannot be postponed till the first job is done. diff --git a/src/parallel_book.pod b/src/parallel_book.pod index 5618b825..51f34f1f 100644 --- a/src/parallel_book.pod +++ b/src/parallel_book.pod @@ -154,6 +154,10 @@ function is exported using B: export -f my_func parallel my_func ::: 1 2 3 +=head2 Copying environment + +env_parallel + =head3 The replacement strings GNU B has some replacement strings to make it easier @@ -161,6 +165,11 @@ GNU B has some replacement strings to make it easier =head2 Controlling the output +parset +sql +cvs + + =head2 Controlling the execution =head3 Remote execution @@ -170,6 +179,49 @@ GNU B has some replacement strings to make it easier =head1 Advanced usage + env_parallel, parset, env_parset +Interfacing with R. + +Interfacing with JSON/jq + +4dl() { + board="$(printf -- '%s' "${1}" | cut -d '/' -f4)" + thread="$(printf -- '%s' "${1}" | cut -d '/' -f6)" + wget -qO- "https://a.4cdn.org/${board}/thread/${thread}.json" | + jq -r ' + .posts + | map(select(.tim != null)) + | map((.tim | tostring) + .ext) + | map("https://i.4cdn.org/'"${board}"'/"+.)[] + ' | + parallel --gnu -j 0 wget -nv +} + +Interfacing with XML/? + +Interfacing with HTML/? + +=head2 Controlling the execution + +--termseq + + +=head3 Remote execution + +seq 10 | parallel --sshlogin 'ssh -i "key.pem" a@b.com' echo + +seq 10 | PARALLLEL_SSH='ssh -i "key.pem"' parallel --sshlogin a@b.com echo + +seq 10 | parallel --ssh 'ssh -i "key.pem"' --sshlogin a@b.com echo + +ssh-agent + +The sshlogin file format + +Check if servers are up + + + =cut diff --git a/src/sql b/src/sql index 3bdae352..353c1b85 100755 --- a/src/sql +++ b/src/sql @@ -576,7 +576,7 @@ $Global::Initfile && unlink $Global::Initfile; exit ($err); sub parse_options { - $Global::version = 20171122; + $Global::version = 20171123; $Global::progname = 'sql'; # This must be done first as this may exec myself diff --git a/testsuite/tests-to-run/parallel-freebsd.sh b/testsuite/tests-to-run/parallel-freebsd.sh index 073c8192..a7a30cda 100644 --- a/testsuite/tests-to-run/parallel-freebsd.sh +++ b/testsuite/tests-to-run/parallel-freebsd.sh @@ -7,61 +7,106 @@ VBoxManage startvm FreeBSD71 >/dev/null 2>&1 ping -c 1 freebsd7.tange.dk >/dev/null 2>&1 ssh freebsd7.tange.dk touch .parallel/will-cite -scp -q .*/src/{parallel,sem,sql,niceload} freebsd7.tange.dk:bin/ +scp -q .*/src/{parallel,sem,sql,niceload,env_parallel*} freebsd7.tange.dk:bin/ -cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj9 -k --joblog /tmp/jl-`basename $0` -L1 --retries 3 -S freebsd7.tange.dk -echo 'bug #40136: FreeBSD: No more processes' - sem --jobs 3 --id my_id -u 'echo First started; sleep 10; echo The first finished;echo' && - sem --jobs 3 --id my_id -u 'echo Second started; sleep 11; echo The second finished;echo' && - sem --jobs 3 --id my_id -u 'echo Third started; sleep 12; echo The third finished;echo' && - sem --jobs 3 --id my_id -u 'echo Fourth started; sleep 13; echo The fourth finished;echo' && - sem --wait --id my_id +par_no_more_procs() { + echo 'bug #40136: FreeBSD: No more processes' + sem --jobs 3 --id my_id -u 'echo First started; sleep 10; echo The first finished;echo' && + sem --jobs 3 --id my_id -u 'echo Second started; sleep 11; echo The second finished;echo' && + sem --jobs 3 --id my_id -u 'echo Third started; sleep 12; echo The third finished;echo' && + sem --jobs 3 --id my_id -u 'echo Fourth started; sleep 13; echo The fourth finished;echo' && + sem --wait --id my_id +} -echo 'Test --compress --pipe' - jot 1000 | parallel --compress --pipe cat | wc +par_compress_pipe() { + echo 'Test --compress --pipe' + jot 1000 | parallel --compress --pipe cat | wc -echo 'bug #41613: --compress --line-buffer no newline'; - perl -e 'print "It worked"'| parallel --pipe --compress --line-buffer cat; echo + echo 'bug #41613: --compress --line-buffer no newline' + perl -e 'print "It worked"'| parallel --pipe --compress --line-buffer cat + echo +} -echo 'bug #40135: FreeBSD: sem --fg does not finish under /bin/sh' - sem --fg 'sleep 1; echo The job finished' +par_sem_fg() { + echo 'bug #40135: FreeBSD: sem --fg does not finish under /bin/sh' + sem --fg 'sleep 1; echo The job finished' +} -echo 'bug #40133: FreeBSD: --round-robin gives no output' - jot 1000000 | parallel --round-robin --pipe -kj3 cat | wc - jot 1000000 | parallel --round-robin --pipe -kj4 cat | wc +par_round_robin() { + echo 'bug #40133: FreeBSD: --round-robin gives no output' + jot 1000000 | parallel --round-robin --pipe -kj3 cat | wc + jot 1000000 | parallel --round-robin --pipe -kj4 cat | wc +} -echo 'bug #40134: FreeBSD: --shebang not working' - (echo '#!/usr/bin/env -S parallel --shebang -rk echo'; echo It; echo worked) > shebang; - chmod 755 ./shebang; ./shebang +par_shebang() { + echo 'bug #40134: FreeBSD: --shebang not working' + (echo '#!/usr/bin/env -S parallel --shebang -rk echo' + echo It + echo worked) > shebang + chmod 755 ./shebang; ./shebang -echo 'bug #40134: FreeBSD: --shebang(-wrap) not working' - (echo '#!/usr/bin/env -S parallel --shebang-wrap /usr/bin/perl :::'; echo 'print @ARGV,"\n";') > shebang-wrap; - chmod 755 ./shebang-wrap; ./shebang-wrap wrap works | sort -r + echo 'bug #40134: FreeBSD: --shebang(-wrap) not working' + (echo '#!/usr/bin/env -S parallel --shebang-wrap /usr/bin/perl :::'; + echo 'print @ARGV,"\n";') > shebang-wrap + chmod 755 ./shebang-wrap + ./shebang-wrap wrap works | sort -r -echo 'bug #40134: FreeBSD: --shebang(-wrap) with options not working' - (echo '#!/usr/bin/env -S parallel --shebang-wrap -v -k -j 0 /usr/bin/perl -w :::'; echo 'print @ARGV,"\n";') > shebang-wrap-opt; - chmod 755 ./shebang-wrap-opt; ./shebang-wrap-opt wrap works with options + echo 'bug #40134: FreeBSD: --shebang(-wrap) with options not working' + (echo '#!/usr/bin/env -S parallel --shebang-wrap -v -k -j 0 /usr/bin/perl -w :::' + echo 'print @ARGV,"\n";') > shebang-wrap-opt; + chmod 755 ./shebang-wrap-opt + ./shebang-wrap-opt wrap works with options +} -bash -c 'echo bug \#43358: shellshock breaks exporting functions using --env _; - echo Non-shellshock-hardened to non-shellshock-hardened; - funky() { echo Function $1; }; - export -f funky; - PARALLEL_SHELL=bash parallel --env funky -S localhost funky ::: non-shellshock-hardened' +par_shellshock_bug() { + bash -c 'echo bug \#43358: shellshock breaks exporting functions using --env _; + echo Non-shellshock-hardened to non-shellshock-hardened; + funky() { echo Function $1; }; + export -f funky; + PARALLEL_SHELL=bash parallel --env funky -S localhost funky ::: non-shellshock-hardened' -bash -c 'echo bug \#43358: shellshock breaks exporting functions using --env _; - echo Non-shellshock-hardened to shellshock-hardened; - funky() { echo Function $1; }; - export -f funky; - parallel --env funky -S parallel@192.168.1.72 funky ::: shellshock-hardened' + bash -c 'echo bug \#43358: shellshock breaks exporting functions using --env _; + echo Non-shellshock-hardened to shellshock-hardened; + funky() { echo Function $1; }; + export -f funky; + parallel --env funky -S parallel@192.168.1.72 funky ::: shellshock-hardened' +} -echo '### Test --load (must give 1=true)' - parallel -j0 -N0 --timeout 5 --nice 10 'bzip2 < /dev/zero >/dev/null' ::: 1 2 3 4 5 6 & - parallel --argsep ,, --joblog - -N0 parallel --load 100% echo ::: 1 ,, 1 | - parallel --colsep '\t' --header : echo '{=4 $_=$_>5=}' +par_load() { + echo '### Test --load (must give 1=true)' + parallel -j0 -N0 --timeout 5 --nice 10 'bzip2 < /dev/zero >/dev/null' ::: 1 2 3 4 5 6 & + parallel --argsep ,, --joblog - -N0 parallel --load 100% echo ::: 1 ,, 1 | + parallel --colsep '\t' --header : echo '{=4 $_=$_>5=}' +} -EOF +par_env_parallel() { + echo "### env_parallel on Freebsd" + . bin/env_parallel.sh + myvar="`echo 'myvar_line 1'; echo 'myvar_line 2'`" + alias myalias='echo myalias1 "$myvar";'"`echo` echo myalias2" + env_parallel myalias ::: foo +} + +# Moving the functions to FreeBSD is a bit tricky: +# We use env_parallel.bash to copy the functions to FreeBSD + +. `which env_parallel.bash` + +# GNU/Linux runs bash, but the FreeBSD runs (a)sh, +# (a)sh does not support 'export -f' so any function exported +# must be unset + +unset run_once +unset run_test +unset TMPDIR + +# As the copied environment is written in Bash dialect +# we get 'shopt'-errors and 'declare'-errors. +# We can safely ignore those. + +env_parallel --env _ -vj9 -k --joblog /tmp/jl-`basename $0` --retries 3 \ + -S freebsd7.tange.dk --tag '{} 2>&1' \ + ::: $(compgen -A function | grep par_ | sort) \ + 2> >(grep -Ev 'shopt: not found|declare: not found') VBoxManage controlvm FreeBSD71 savestate - - - diff --git a/testsuite/tests-to-run/parallel-local-ssh7.sh b/testsuite/tests-to-run/parallel-local-ssh7.sh index 522cc250..6b92e045 100755 --- a/testsuite/tests-to-run/parallel-local-ssh7.sh +++ b/testsuite/tests-to-run/parallel-local-ssh7.sh @@ -16,29 +16,41 @@ par_bash_man() { . `which env_parallel.bash`; - alias myecho="echo aliases"; - env_parallel myecho ::: work; - env_parallel -S server myecho ::: work; - env_parallel --env myecho myecho ::: work; + alias myecho='echo aliases' + env_parallel myecho ::: work + env_parallel -S server myecho ::: work + env_parallel --env myecho myecho ::: work env_parallel --env myecho -S server myecho ::: work - - myfunc() { echo functions $*; }; - env_parallel myfunc ::: work; - env_parallel -S server myfunc ::: work; - env_parallel --env myfunc myfunc ::: work; + + alias multiline='echo multiline + echo aliases' + env_parallel 'multiline {}; + echo but only when followed by a newline' ::: work + env_parallel -S server 'multiline {}; + echo but only when followed by a newline' ::: work + env_parallel --env multiline 'multiline {}; + echo but only when followed by a newline' ::: work + env_parallel --env multiline -S server 'multiline {}; + echo but only when followed by a newline' ::: work + alias multiline="dummy" + + myfunc() { echo functions $*; } + env_parallel myfunc ::: work + env_parallel -S server myfunc ::: work + env_parallel --env myfunc myfunc ::: work env_parallel --env myfunc -S server myfunc ::: work - - myvar=variables; - env_parallel echo "\$myvar" ::: work; - env_parallel -S server echo "\$myvar" ::: work; - env_parallel --env myvar echo "\$myvar" ::: work; - env_parallel --env myvar -S server echo "\$myvar" ::: work - - myarray=(arrays work, too); - env_parallel -k echo "\${myarray[{}]}" ::: 0 1 2; - env_parallel -k -S server echo "\${myarray[{}]}" ::: 0 1 2; - env_parallel -k --env myarray echo "\${myarray[{}]}" ::: 0 1 2; - env_parallel -k --env myarray -S server echo "\${myarray[{}]}" ::: 0 1 2 + + myvar=variables + env_parallel echo '$myvar' ::: work + env_parallel -S server echo '$myvar' ::: work + env_parallel --env myvar echo '$myvar' ::: work + env_parallel --env myvar -S server echo '$myvar' ::: work + + myarray=(arrays work, too) + env_parallel -k echo '${myarray[{}]}' ::: 0 1 2 + env_parallel -k -S server echo '${myarray[{}]}' ::: 0 1 2 + env_parallel -k --env myarray echo '${myarray[{}]}' ::: 0 1 2 + env_parallel -k --env myarray -S server echo '${myarray[{}]}' ::: 0 1 2 env_parallel ::: true false true false echo exit value $? should be 2 @@ -57,29 +69,36 @@ par_zsh_man() { . `which env_parallel.zsh`; - alias myecho="echo aliases"; - env_parallel myecho ::: work; - env_parallel -S server myecho ::: work; - env_parallel --env myecho myecho ::: work; + alias myecho='echo aliases' + env_parallel myecho ::: work + env_parallel -S server myecho ::: work + env_parallel --env myecho myecho ::: work env_parallel --env myecho -S server myecho ::: work - - myfunc() { echo functions $*; }; - env_parallel myfunc ::: work; - env_parallel -S server myfunc ::: work; - env_parallel --env myfunc myfunc ::: work; + + alias multiline='echo multiline + echo aliases' + env_parallel multiline ::: work + env_parallel -S server multiline ::: work + env_parallel --env multiline multiline ::: work + env_parallel --env multiline -S server multiline ::: work + + myfunc() { echo functions $*; } + env_parallel myfunc ::: work + env_parallel -S server myfunc ::: work + env_parallel --env myfunc myfunc ::: work env_parallel --env myfunc -S server myfunc ::: work - - myvar=variables; - env_parallel echo "\$myvar" ::: work; - env_parallel -S server echo "\$myvar" ::: work; - env_parallel --env myvar echo "\$myvar" ::: work; - env_parallel --env myvar -S server echo "\$myvar" ::: work - - myarray=(arrays work, too); - env_parallel -k echo "\${myarray[{}]}" ::: 1 2 3; - env_parallel -k -S server echo "\${myarray[{}]}" ::: 1 2 3; - env_parallel -k --env myarray echo "\${myarray[{}]}" ::: 1 2 3; - env_parallel -k --env myarray -S server echo "\${myarray[{}]}" ::: 1 2 3 + + myvar=variables + env_parallel echo '$myvar' ::: work + env_parallel -S server echo '$myvar' ::: work + env_parallel --env myvar echo '$myvar' ::: work + env_parallel --env myvar -S server echo '$myvar' ::: work + + myarray=(arrays work, too) + env_parallel -k echo '${myarray[{}]}' ::: 1 2 3 + env_parallel -k -S server echo '${myarray[{}]}' ::: 1 2 3 + env_parallel -k --env myarray echo '${myarray[{}]}' ::: 1 2 3 + env_parallel -k --env myarray -S server echo '${myarray[{}]}' ::: 1 2 3 env_parallel ::: true false true false echo exit value $? should be 2 @@ -97,32 +116,37 @@ par_ksh_man() { echo "### From man env_parallel" . `which env_parallel.ksh`; - alias myecho="echo aliases"; - env_parallel myecho ::: work; - env_parallel -S server myecho ::: work; - env_parallel --env myecho myecho ::: work; + + alias myecho='echo aliases' + env_parallel myecho ::: work + env_parallel -S server myecho ::: work + env_parallel --env myecho myecho ::: work env_parallel --env myecho -S server myecho ::: work - - . `which env_parallel.ksh`; - myfunc() { echo functions $*; }; - env_parallel myfunc ::: work; - env_parallel -S server myfunc ::: work; - env_parallel --env myfunc myfunc ::: work; + + alias multiline='echo multiline + echo aliases' + env_parallel multiline ::: work + env_parallel -S server multiline ::: work + env_parallel --env multiline multiline ::: work + env_parallel --env multiline -S server multiline ::: work + + myfunc() { echo functions $*; } + env_parallel myfunc ::: work + env_parallel -S server myfunc ::: work + env_parallel --env myfunc myfunc ::: work env_parallel --env myfunc -S server myfunc ::: work - - . `which env_parallel.ksh`; - myvar=variables; - env_parallel echo "\$myvar" ::: work; - env_parallel -S server echo "\$myvar" ::: work; - env_parallel --env myvar echo "\$myvar" ::: work; - env_parallel --env myvar -S server echo "\$myvar" ::: work - - . `which env_parallel.ksh`; - myarray=(arrays work, too); - env_parallel -k echo "\${myarray[{}]}" ::: 0 1 2; - env_parallel -k -S server echo "\${myarray[{}]}" ::: 0 1 2; - env_parallel -k --env myarray echo "\${myarray[{}]}" ::: 0 1 2; - env_parallel -k --env myarray -S server echo "\${myarray[{}]}" ::: 0 1 2 + + myvar=variables + env_parallel echo '$myvar' ::: work + env_parallel -S server echo '$myvar' ::: work + env_parallel --env myvar echo '$myvar' ::: work + env_parallel --env myvar -S server echo '$myvar' ::: work + + myarray=(arrays work, too) + env_parallel -k echo '${myarray[{}]}' ::: 0 1 2 + env_parallel -k -S server echo '${myarray[{}]}' ::: 0 1 2 + env_parallel -k --env myarray echo '${myarray[{}]}' ::: 0 1 2 + env_parallel -k --env myarray -S server echo '${myarray[{}]}' ::: 0 1 2 env_parallel ::: true false true false echo exit value $? should be 2 @@ -863,7 +887,7 @@ _EOF ssh tcsh@lo "$myscript" 2>&1 | sort } -par_bash_env_parallel_fifo() { +par_bash_env_parallel() { myscript=$(cat <<'_EOF' echo 'bug #50435: Remote fifo broke in 20150522' # Due to $PARALLEL_TMP being transferred @@ -871,26 +895,48 @@ par_bash_env_parallel_fifo() { OK=OK echo data from stdin | env_parallel --pipe -S lo --fifo 'cat {} && echo $OK' echo data from stdin | env_parallel --pipe -S lo --cat 'cat {} && echo $OK' + + echo 'bug #52534: Tail of multiline alias is ignored' + alias myalias='echo alias line 1 + echo alias line 2 + echo alias line 3 + ' + alias myalias2='echo alias2 line 1 + echo alias2 line 2 + ' + env_parallel myalias ::: myalias2 + env_parallel -S lo myalias ::: myalias2 _EOF ) # Order is often different. Dunno why. So sort ssh bash@lo "$myscript" 2>&1 | sort } -par_zsh_env_parallel_fifo() { +par_zsh_env_parallel() { myscript=$(cat <<'_EOF' echo 'bug #50435: Remote fifo broke in 20150522' # Due to $PARALLEL_TMP being transferred OK=OK echo data from stdin | env_parallel --pipe -S lo --fifo 'cat {} && echo $OK' echo data from stdin | env_parallel --pipe -S lo --cat 'cat {} && echo $OK' + + echo 'bug #52534: Tail of multiline alias is ignored' + alias myalias='echo alias line 1 + echo alias line 2 + echo alias line 3 + ' + alias myalias2='echo alias2 line 1 + echo alias2 line 2 + ' + env_parallel myalias ::: myalias2 + env_parallel -S lo myalias ::: myalias2 _EOF ) # Order is often different. Dunno why. So sort ssh zsh@lo "$myscript" 2>&1 | sort } -par_ksh_env_parallel_fifo() { +par_ksh_env_parallel() { myscript=$(cat <<'_EOF' . `which env_parallel.ksh`; echo 'bug #50435: Remote fifo broke in 20150522' @@ -898,13 +944,24 @@ par_ksh_env_parallel_fifo() { OK=OK echo data from stdin | env_parallel --pipe -S lo --fifo 'cat {} && echo $OK' echo data from stdin | env_parallel --pipe -S lo --cat 'cat {} && echo $OK' + + echo 'bug #52534: Tail of multiline alias is ignored' + alias myalias='echo alias line 1 + echo alias line 2 + echo alias line 3 + ' + alias myalias2='echo alias2 line 1 + echo alias2 line 2 + ' + env_parallel myalias ::: myalias2 + env_parallel -S lo myalias ::: myalias2 _EOF ) # Order is often different. Dunno why. So sort ssh ksh@lo "$myscript" 2>&1 | sort } -par_fish_env_parallel_fifo() { +par_fish_env_parallel() { myscript=$(cat <<'_EOF' echo 'bug #50435: Remote fifo broke in 20150522' # Due to $PARALLEL_TMP being transferred @@ -916,7 +973,7 @@ _EOF ssh fish@lo "$myscript" } -par_csh_env_parallel_fifo() { +par_csh_env_parallel() { myscript=$(cat <<'_EOF' echo 'bug #50435: Remote fifo broke in 20150522' # Due to $PARALLEL_TMP being transferred @@ -928,7 +985,7 @@ _EOF ssh csh@lo "$myscript" } -par_tcsh_env_parallel_fifo() { +par_tcsh_env_parallel() { myscript=$(cat <<'_EOF' echo 'bug #50435: Remote fifo broke in 20150522' # Due to $PARALLEL_TMP being transferred @@ -950,16 +1007,16 @@ par_bash_environment_too_big() { env_parallel echo ::: OK_bigvar env_parallel -S lo echo ::: OK_bigvar_remote - bigvar="$(perl -e 'print "\""x61000')" + bigvar="$(perl -e 'print "\""x60000')" env_parallel echo ::: OK_bigvar_quote env_parallel -S lo echo ::: OK_bigvar_quote_remote bigvar=u - eval 'bigfunc() { a="'"$(perl -e 'print "x"x122000')"'"; };' + eval 'bigfunc() { a="'"$(perl -e 'print "x"x121000')"'"; };' env_parallel echo ::: OK_bigfunc env_parallel -S lo echo ::: OK_bigfunc_remote - eval 'bigfunc() { a="'"$(perl -e 'print "\""x122000')"'"; };' + eval 'bigfunc() { a="'"$(perl -e 'print "\""x121000')"'"; };' env_parallel echo ::: OK_bigfunc_quote env_parallel -S lo echo ::: OK_bigfunc_quote_remote bigfunc() { true; } @@ -970,16 +1027,16 @@ par_bash_environment_too_big() { env_parallel echo ::: fail_bigvar env_parallel -S lo echo ::: fail_bigvar_remote - bigvar="$(perl -e 'print "\""x62000')" + bigvar="$(perl -e 'print "\""x61000')" env_parallel echo ::: fail_bigvar_quote env_parallel -S lo echo ::: fail_bigvar_quote_remote bigvar=u - eval 'bigfunc() { a="'"$(perl -e 'print "x"x1230000')"'"; };' + eval 'bigfunc() { a="'"$(perl -e 'print "x"x1220000')"'"; };' env_parallel echo ::: fail_bigfunc env_parallel -S lo echo ::: fail_bigfunc_remote - eval 'bigfunc() { a="'"$(perl -e 'print "\""x123000')"'"; };' + eval 'bigfunc() { a="'"$(perl -e 'print "\""x122000')"'"; };' env_parallel echo ::: fail_bigfunc_quote env_parallel -S lo echo ::: fail_bigfunc_quote_remote diff --git a/testsuite/wanted-results/parallel-freebsd b/testsuite/wanted-results/parallel-freebsd index d336284f..961d80fe 100644 --- a/testsuite/wanted-results/parallel-freebsd +++ b/testsuite/wanted-results/parallel-freebsd @@ -1,68 +1,61 @@ ### These tests requires VirtualBox running with the following images tange@freebsd7 -echo 'bug #40136: FreeBSD: No more processes' -bug #40136: FreeBSD: No more processes - sem --jobs 3 --id my_id -u 'echo First started; sleep 10; echo The first finished;echo' && sem --jobs 3 --id my_id -u 'echo Second started; sleep 11; echo The second finished;echo' && sem --jobs 3 --id my_id -u 'echo Third started; sleep 12; echo The third finished;echo' && sem --jobs 3 --id my_id -u 'echo Fourth started; sleep 13; echo The fourth finished;echo' && sem --wait --id my_id -First started -Second started -Third started -The first finished - -Fourth started -The second finished - -The third finished - -The fourth finished - -echo 'Test --compress --pipe' -Test --compress --pipe - jot 1000 | parallel --compress --pipe cat | wc - 1000 1000 3893 -echo 'bug #41613: --compress --line-buffer no newline'; perl -e 'print "It worked"'| parallel --pipe --compress --line-buffer cat; echo -bug #41613: --compress --line-buffer no newline -It worked -echo 'bug #40135: FreeBSD: sem --fg does not finish under /bin/sh' -bug #40135: FreeBSD: sem --fg does not finish under /bin/sh - sem --fg 'sleep 1; echo The job finished' -The job finished -echo 'bug #40133: FreeBSD: --round-robin gives no output' -bug #40133: FreeBSD: --round-robin gives no output - jot 1000000 | parallel --round-robin --pipe -kj3 cat | wc - 1000000 1000000 6888896 - jot 1000000 | parallel --round-robin --pipe -kj4 cat | wc - 1000000 1000000 6888896 -echo 'bug #40134: FreeBSD: --shebang not working' -bug #40134: FreeBSD: --shebang not working - (echo '#!/usr/bin/env -S parallel --shebang -rk echo'; echo It; echo worked) > shebang; chmod 755 ./shebang; ./shebang -It -worked -echo 'bug #40134: FreeBSD: --shebang(-wrap) not working' -bug #40134: FreeBSD: --shebang(-wrap) not working - (echo '#!/usr/bin/env -S parallel --shebang-wrap /usr/bin/perl :::'; echo 'print @ARGV,"\n";') > shebang-wrap; chmod 755 ./shebang-wrap; ./shebang-wrap wrap works | sort -r -wrap -works -echo 'bug #40134: FreeBSD: --shebang(-wrap) with options not working' -bug #40134: FreeBSD: --shebang(-wrap) with options not working - (echo '#!/usr/bin/env -S parallel --shebang-wrap -v -k -j 0 /usr/bin/perl -w :::'; echo 'print @ARGV,"\n";') > shebang-wrap-opt; chmod 755 ./shebang-wrap-opt; ./shebang-wrap-opt wrap works with options -/usr/bin/perl -w ./shebang-wrap-opt wrap -wrap -/usr/bin/perl -w ./shebang-wrap-opt works -works -/usr/bin/perl -w ./shebang-wrap-opt with -with -/usr/bin/perl -w ./shebang-wrap-opt options -options -bash -c 'echo bug \#43358: shellshock breaks exporting functions using --env _; echo Non-shellshock-hardened to non-shellshock-hardened; funky() { echo Function $1; }; export -f funky; PARALLEL_SHELL=bash parallel --env funky -S localhost funky ::: non-shellshock-hardened' -bug #43358: shellshock breaks exporting functions using --env _ -Non-shellshock-hardened to non-shellshock-hardened -Function non-shellshock-hardened -bash -c 'echo bug \#43358: shellshock breaks exporting functions using --env _; echo Non-shellshock-hardened to shellshock-hardened; funky() { echo Function $1; }; export -f funky; parallel --env funky -S parallel@192.168.1.72 funky ::: shellshock-hardened' -bug #43358: shellshock breaks exporting functions using --env _ -Non-shellshock-hardened to shellshock-hardened -Function shellshock-hardened -echo '### Test --load (must give 1=true)' -### Test --load (must give 1=true) - parallel -j0 -N0 --timeout 5 --nice 10 'bzip2 < /dev/zero >/dev/null' ::: 1 2 3 4 5 6 & parallel --argsep ,, --joblog - -N0 parallel --load 100% echo ::: 1 ,, 1 | parallel --colsep '\t' --header : echo '{=4 $_=$_>5=}' - -1 +par_compress_pipe 2>&1 +par_compress_pipe Test --compress --pipe +par_compress_pipe 1000 1000 3893 +par_compress_pipe bug #41613: --compress --line-buffer no newline +par_compress_pipe It worked +par_env_parallel 2>&1 +par_env_parallel ### env_parallel on Freebsd +par_env_parallel myalias1 myvar_line 1 +par_env_parallel myvar_line 2 +par_env_parallel myalias2 foo +par_load 2>&1 +par_load ### Test --load (must give 1=true) +par_load +par_load 1 +par_no_more_procs 2>&1 +par_no_more_procs bug #40136: FreeBSD: No more processes +par_no_more_procs First started +par_no_more_procs Second started +par_no_more_procs Third started +par_no_more_procs The first finished +par_no_more_procs +par_no_more_procs Fourth started +par_no_more_procs The second finished +par_no_more_procs +par_no_more_procs The third finished +par_no_more_procs +par_no_more_procs The fourth finished +par_no_more_procs +par_round_robin 2>&1 +par_round_robin bug #40133: FreeBSD: --round-robin gives no output +par_round_robin 1000000 1000000 6888896 +par_round_robin 1000000 1000000 6888896 +par_sem_fg 2>&1 +par_sem_fg bug #40135: FreeBSD: sem --fg does not finish under /bin/sh +par_sem_fg The job finished +par_shebang 2>&1 +par_shebang bug #40134: FreeBSD: --shebang not working +par_shebang It +par_shebang worked +par_shebang bug #40134: FreeBSD: --shebang(-wrap) not working +par_shebang wrap +par_shebang works +par_shebang bug #40134: FreeBSD: --shebang(-wrap) with options not working +par_shebang /usr/bin/perl -w ./shebang-wrap-opt wrap +par_shebang wrap +par_shebang /usr/bin/perl -w ./shebang-wrap-opt works +par_shebang works +par_shebang /usr/bin/perl -w ./shebang-wrap-opt with +par_shebang with +par_shebang /usr/bin/perl -w ./shebang-wrap-opt options +par_shebang options +par_shellshock_bug 2>&1 +par_shellshock_bug bug #43358: shellshock breaks exporting functions using --env _ +par_shellshock_bug Non-shellshock-hardened to non-shellshock-hardened +par_shellshock_bug Function non-shellshock-hardened +par_shellshock_bug bug #43358: shellshock breaks exporting functions using --env _ +par_shellshock_bug Non-shellshock-hardened to shellshock-hardened +par_shellshock_bug parallel: Warning: Shell functions may not be supported in /bin/sh. +par_shellshock_bug Function shellshock-hardened diff --git a/testsuite/wanted-results/parallel-local-ssh7 b/testsuite/wanted-results/parallel-local-ssh7 index c42ed717..a9b5d84a 100644 --- a/testsuite/wanted-results/parallel-local-ssh7 +++ b/testsuite/wanted-results/parallel-local-ssh7 @@ -72,6 +72,14 @@ par_zsh_man aliases work par_zsh_man aliases work par_zsh_man aliases work par_zsh_man aliases work +par_zsh_man multiline +par_zsh_man aliases work +par_zsh_man multiline +par_zsh_man aliases work +par_zsh_man multiline +par_zsh_man aliases work +par_zsh_man multiline +par_zsh_man aliases work par_zsh_man functions work par_zsh_man functions work par_zsh_man functions work @@ -124,19 +132,30 @@ par_zsh_environment_too_big OK_bigfunc_remote par_zsh_environment_too_big OK_bigfunc_quote par_zsh_environment_too_big OK_bigfunc_quote_remote par_zsh_environment_too_big Rest should fail -par_zsh_environment_too_big env_parallel:132: argument list too long: /usr/local/bin/parallel -par_zsh_environment_too_big env_parallel:132: argument list too long: /usr/local/bin/parallel -par_zsh_environment_too_big env_parallel:132: argument list too long: /usr/local/bin/parallel -par_zsh_environment_too_big env_parallel:132: argument list too long: /usr/local/bin/parallel -par_zsh_environment_too_big env_parallel:132: argument list too long: /usr/local/bin/parallel -par_zsh_environment_too_big env_parallel:132: argument list too long: /usr/local/bin/parallel -par_zsh_environment_too_big env_parallel:132: argument list too long: /usr/local/bin/parallel -par_zsh_environment_too_big env_parallel:132: argument list too long: /usr/local/bin/parallel -par_zsh_env_parallel_fifo OK -par_zsh_env_parallel_fifo OK -par_zsh_env_parallel_fifo bug #50435: Remote fifo broke in 20150522 -par_zsh_env_parallel_fifo data from stdin -par_zsh_env_parallel_fifo data from stdin +par_zsh_environment_too_big env_parallel:135: argument list too long: /usr/local/bin/parallel +par_zsh_environment_too_big env_parallel:135: argument list too long: /usr/local/bin/parallel +par_zsh_environment_too_big env_parallel:135: argument list too long: /usr/local/bin/parallel +par_zsh_environment_too_big env_parallel:135: argument list too long: /usr/local/bin/parallel +par_zsh_environment_too_big env_parallel:135: argument list too long: /usr/local/bin/parallel +par_zsh_environment_too_big env_parallel:135: argument list too long: /usr/local/bin/parallel +par_zsh_environment_too_big env_parallel:135: argument list too long: /usr/local/bin/parallel +par_zsh_environment_too_big env_parallel:135: argument list too long: /usr/local/bin/parallel +par_zsh_env_parallel OK +par_zsh_env_parallel OK +par_zsh_env_parallel alias line 1 +par_zsh_env_parallel alias line 1 +par_zsh_env_parallel alias line 2 +par_zsh_env_parallel alias line 2 +par_zsh_env_parallel alias line 3 +par_zsh_env_parallel alias line 3 +par_zsh_env_parallel alias2 line 1 +par_zsh_env_parallel alias2 line 1 +par_zsh_env_parallel alias2 line 2 +par_zsh_env_parallel alias2 line 2 +par_zsh_env_parallel bug #50435: Remote fifo broke in 20150522 +par_zsh_env_parallel bug #52534: Tail of multiline alias is ignored +par_zsh_env_parallel data from stdin +par_zsh_env_parallel data from stdin par_tcsh_underscore ### tcsh par_tcsh_underscore ### Testing of --env _ par_tcsh_underscore #: Command not found. @@ -236,11 +255,11 @@ par_tcsh_funky myvar works par_tcsh_funky space special chars problem par_tcsh_funky space special chars problem par_tcsh_environment_too_big Not implemented -par_tcsh_env_parallel_fifo OK -par_tcsh_env_parallel_fifo OK -par_tcsh_env_parallel_fifo bug #50435: Remote fifo broke in 20150522 -par_tcsh_env_parallel_fifo data from stdin -par_tcsh_env_parallel_fifo data from stdin +par_tcsh_env_parallel OK +par_tcsh_env_parallel OK +par_tcsh_env_parallel bug #50435: Remote fifo broke in 20150522 +par_tcsh_env_parallel data from stdin +par_tcsh_env_parallel data from stdin par_sh_parset parset par_sh_parset ### parset into vars with comma par_sh_parset foo bar baz @@ -377,6 +396,14 @@ par_ksh_man aliases work par_ksh_man aliases work par_ksh_man aliases work par_ksh_man aliases work +par_ksh_man multiline +par_ksh_man aliases work +par_ksh_man multiline +par_ksh_man aliases work +par_ksh_man multiline +par_ksh_man aliases work +par_ksh_man multiline +par_ksh_man aliases work par_ksh_man functions work par_ksh_man functions work par_ksh_man functions work @@ -475,11 +502,22 @@ par_ksh_environment_too_big env_parallel: Error: Try running this in a clean env par_ksh_environment_too_big env_parallel: Error: env_parallel --record-env par_ksh_environment_too_big env_parallel: Error: And the use '--env _' par_ksh_environment_too_big env_parallel: Error: For details see: man env_parallel -par_ksh_env_parallel_fifo OK -par_ksh_env_parallel_fifo OK -par_ksh_env_parallel_fifo bug #50435: Remote fifo broke in 20150522 -par_ksh_env_parallel_fifo data from stdin -par_ksh_env_parallel_fifo data from stdin +par_ksh_env_parallel OK +par_ksh_env_parallel OK +par_ksh_env_parallel alias line 1 +par_ksh_env_parallel alias line 1 +par_ksh_env_parallel alias line 2 +par_ksh_env_parallel alias line 2 +par_ksh_env_parallel alias line 3 +par_ksh_env_parallel alias line 3 +par_ksh_env_parallel alias2 line 1 +par_ksh_env_parallel alias2 line 1 +par_ksh_env_parallel alias2 line 2 +par_ksh_env_parallel alias2 line 2 +par_ksh_env_parallel bug #50435: Remote fifo broke in 20150522 +par_ksh_env_parallel bug #52534: Tail of multiline alias is ignored +par_ksh_env_parallel data from stdin +par_ksh_env_parallel data from stdin par_fish_underscore ### fish par_fish_underscore ### Testing of --env _ par_fish_underscore variables in aliases and arrays in functions work @@ -575,14 +613,14 @@ par_fish_funky par_fish_funky par_fish_funky \\\\\\\\ \ \ \ \ \\\\\\\\\\\\\\ \ \!\"\#\$%\&\'\(\)\*+,-./0123456789:\;\<\=\>\?@ABCDEFGHIJKLMNOPQRSTUVWXYZ\[\\\]\^_\`abcdefghijklmnopqrstuvwxyz\{\|\}\~\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\ par_fish_environment_too_big Not implemented -par_fish_env_parallel_fifo bug #50435: Remote fifo broke in 20150522 -par_fish_env_parallel_fifo data from stdin -par_fish_env_parallel_fifo OK -par_fish_env_parallel_fifo data from stdin -par_fish_env_parallel_fifo OK -par_fish_env_parallel_fifo $? is not the exit status. In fish, please use $status. -par_fish_env_parallel_fifo fish: cat $PARALLEL_TMP; and echo $OK;perl -e '$bash=shift;$csh=shift;for(@ARGV){unlink;rmdir;}if($bash=~s/h//){exit$bash;}exit$csh;' "$?h" "$status" $PARALLEL_TMP -par_fish_env_parallel_fifo ^ +par_fish_env_parallel bug #50435: Remote fifo broke in 20150522 +par_fish_env_parallel data from stdin +par_fish_env_parallel OK +par_fish_env_parallel data from stdin +par_fish_env_parallel OK +par_fish_env_parallel $? is not the exit status. In fish, please use $status. +par_fish_env_parallel fish: cat $PARALLEL_TMP; and echo $OK;perl -e '$bash=shift;$csh=shift;for(@ARGV){unlink;rmdir;}if($bash=~s/h//){exit$bash;}exit$csh;' "$?h" "$status" $PARALLEL_TMP +par_fish_env_parallel ^ par_dash_parset parset par_dash_parset ### parset into vars with comma par_dash_parset foo bar baz @@ -733,11 +771,11 @@ par_csh_funky Funky-  !"#$%&'()*+,-./0123456789:;<= par_csh_funky func_echo: Command not found. par_csh_funky \\\\\\\\ \ \ \ \ \\\\\\\\\\\\\\ \!\"\#\$%\&\'\(\)\*+,-./0123456789:\;\<\=\>\?@ABCDEFGHIJKLMNOPQRSTUVWXYZ\[\\\]\^_\`abcdefghijklmnopqrstuvwxyz\{\|\}\~\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\ par_csh_environment_too_big Not implemented -par_csh_env_parallel_fifo bug #50435: Remote fifo broke in 20150522 -par_csh_env_parallel_fifo data from stdin -par_csh_env_parallel_fifo OK -par_csh_env_parallel_fifo data from stdin -par_csh_env_parallel_fifo OK +par_csh_env_parallel bug #50435: Remote fifo broke in 20150522 +par_csh_env_parallel data from stdin +par_csh_env_parallel OK +par_csh_env_parallel data from stdin +par_csh_env_parallel OK par_bash_underscore ### bash par_bash_underscore ### Testing of --env _ par_bash_underscore variables in aliases in and arrays in functions work @@ -803,6 +841,30 @@ par_bash_man aliases work par_bash_man aliases work par_bash_man aliases work par_bash_man aliases work +par_bash_man env_parallel: Warning: Alias 'multiline' contains newline. +par_bash_man env_parallel: Warning: Make sure the command has at least one newline after 'multiline'. +par_bash_man env_parallel: Warning: See BUGS in 'man env_parallel'. +par_bash_man multiline +par_bash_man aliases work +par_bash_man but only when followed by a newline +par_bash_man env_parallel: Warning: Alias 'multiline' contains newline. +par_bash_man env_parallel: Warning: Make sure the command has at least one newline after 'multiline'. +par_bash_man env_parallel: Warning: See BUGS in 'man env_parallel'. +par_bash_man multiline +par_bash_man aliases work +par_bash_man but only when followed by a newline +par_bash_man env_parallel: Warning: Alias 'multiline' contains newline. +par_bash_man env_parallel: Warning: Make sure the command has at least one newline after 'multiline'. +par_bash_man env_parallel: Warning: See BUGS in 'man env_parallel'. +par_bash_man multiline +par_bash_man aliases work +par_bash_man but only when followed by a newline +par_bash_man env_parallel: Warning: Alias 'multiline' contains newline. +par_bash_man env_parallel: Warning: Make sure the command has at least one newline after 'multiline'. +par_bash_man env_parallel: Warning: See BUGS in 'man env_parallel'. +par_bash_man multiline +par_bash_man aliases work +par_bash_man but only when followed by a newline par_bash_man functions work par_bash_man functions work par_bash_man functions work @@ -862,59 +924,74 @@ par_bash_environment_too_big OK_bigfunc_remote par_bash_environment_too_big OK_bigfunc_quote par_bash_environment_too_big OK_bigfunc_quote_remote par_bash_environment_too_big Rest should fail -par_bash_environment_too_big /usr/local/bin/env_parallel.bash: line 170: /usr/bin/which: Argument list too long +par_bash_environment_too_big /usr/local/bin/env_parallel.bash: line 182: /usr/bin/which: Argument list too long par_bash_environment_too_big env_parallel: Error: Your environment is too big. par_bash_environment_too_big env_parallel: Error: Try running this in a clean environment once: par_bash_environment_too_big env_parallel: Error: env_parallel --record-env par_bash_environment_too_big env_parallel: Error: And the use '--env _' par_bash_environment_too_big env_parallel: Error: For details see: man env_parallel -par_bash_environment_too_big /usr/local/bin/env_parallel.bash: line 170: /usr/bin/which: Argument list too long +par_bash_environment_too_big /usr/local/bin/env_parallel.bash: line 182: /usr/bin/which: Argument list too long par_bash_environment_too_big env_parallel: Error: Your environment is too big. par_bash_environment_too_big env_parallel: Error: Try running this in a clean environment once: par_bash_environment_too_big env_parallel: Error: env_parallel --record-env par_bash_environment_too_big env_parallel: Error: And the use '--env _' par_bash_environment_too_big env_parallel: Error: For details see: man env_parallel -par_bash_environment_too_big /usr/local/bin/env_parallel.bash: line 170: /usr/bin/which: Argument list too long +par_bash_environment_too_big /usr/local/bin/env_parallel.bash: line 182: /usr/bin/which: Argument list too long par_bash_environment_too_big env_parallel: Error: Your environment is too big. par_bash_environment_too_big env_parallel: Error: Try running this in a clean environment once: par_bash_environment_too_big env_parallel: Error: env_parallel --record-env par_bash_environment_too_big env_parallel: Error: And the use '--env _' par_bash_environment_too_big env_parallel: Error: For details see: man env_parallel -par_bash_environment_too_big /usr/local/bin/env_parallel.bash: line 170: /usr/bin/which: Argument list too long +par_bash_environment_too_big /usr/local/bin/env_parallel.bash: line 182: /usr/bin/which: Argument list too long par_bash_environment_too_big env_parallel: Error: Your environment is too big. par_bash_environment_too_big env_parallel: Error: Try running this in a clean environment once: par_bash_environment_too_big env_parallel: Error: env_parallel --record-env par_bash_environment_too_big env_parallel: Error: And the use '--env _' par_bash_environment_too_big env_parallel: Error: For details see: man env_parallel -par_bash_environment_too_big /usr/local/bin/env_parallel.bash: line 170: /usr/bin/which: Argument list too long +par_bash_environment_too_big /usr/local/bin/env_parallel.bash: line 182: /usr/bin/which: Argument list too long par_bash_environment_too_big env_parallel: Error: Your environment is too big. par_bash_environment_too_big env_parallel: Error: Try running this in a clean environment once: par_bash_environment_too_big env_parallel: Error: env_parallel --record-env par_bash_environment_too_big env_parallel: Error: And the use '--env _' par_bash_environment_too_big env_parallel: Error: For details see: man env_parallel -par_bash_environment_too_big /usr/local/bin/env_parallel.bash: line 170: /usr/bin/which: Argument list too long +par_bash_environment_too_big /usr/local/bin/env_parallel.bash: line 182: /usr/bin/which: Argument list too long par_bash_environment_too_big env_parallel: Error: Your environment is too big. par_bash_environment_too_big env_parallel: Error: Try running this in a clean environment once: par_bash_environment_too_big env_parallel: Error: env_parallel --record-env par_bash_environment_too_big env_parallel: Error: And the use '--env _' par_bash_environment_too_big env_parallel: Error: For details see: man env_parallel -par_bash_environment_too_big /usr/local/bin/env_parallel.bash: line 170: /usr/bin/which: Argument list too long +par_bash_environment_too_big /usr/local/bin/env_parallel.bash: line 182: /usr/bin/which: Argument list too long par_bash_environment_too_big env_parallel: Error: Your environment is too big. par_bash_environment_too_big env_parallel: Error: Try running this in a clean environment once: par_bash_environment_too_big env_parallel: Error: env_parallel --record-env par_bash_environment_too_big env_parallel: Error: And the use '--env _' par_bash_environment_too_big env_parallel: Error: For details see: man env_parallel -par_bash_environment_too_big /usr/local/bin/env_parallel.bash: line 170: /usr/bin/which: Argument list too long +par_bash_environment_too_big /usr/local/bin/env_parallel.bash: line 182: /usr/bin/which: Argument list too long par_bash_environment_too_big env_parallel: Error: Your environment is too big. par_bash_environment_too_big env_parallel: Error: Try running this in a clean environment once: par_bash_environment_too_big env_parallel: Error: env_parallel --record-env par_bash_environment_too_big env_parallel: Error: And the use '--env _' par_bash_environment_too_big env_parallel: Error: For details see: man env_parallel -par_bash_env_parallel_fifo OK -par_bash_env_parallel_fifo OK -par_bash_env_parallel_fifo bug #50435: Remote fifo broke in 20150522 -par_bash_env_parallel_fifo data from stdin -par_bash_env_parallel_fifo data from stdin +par_bash_env_parallel OK +par_bash_env_parallel OK +par_bash_env_parallel alias line 1 +par_bash_env_parallel alias line 1 +par_bash_env_parallel bug #50435: Remote fifo broke in 20150522 +par_bash_env_parallel bug #52534: Tail of multiline alias is ignored +par_bash_env_parallel data from stdin +par_bash_env_parallel data from stdin +par_bash_env_parallel env_parallel: Warning: Alias 'myalias' contains newline. +par_bash_env_parallel env_parallel: Warning: Alias 'myalias' contains newline. +par_bash_env_parallel env_parallel: Warning: Alias 'myalias2' contains newline. +par_bash_env_parallel env_parallel: Warning: Alias 'myalias2' contains newline. +par_bash_env_parallel env_parallel: Warning: Make sure the command has at least one newline after 'myalias'. +par_bash_env_parallel env_parallel: Warning: Make sure the command has at least one newline after 'myalias'. +par_bash_env_parallel env_parallel: Warning: Make sure the command has at least one newline after 'myalias2'. +par_bash_env_parallel env_parallel: Warning: Make sure the command has at least one newline after 'myalias2'. +par_bash_env_parallel env_parallel: Warning: See BUGS in 'man env_parallel'. +par_bash_env_parallel env_parallel: Warning: See BUGS in 'man env_parallel'. +par_bash_env_parallel env_parallel: Warning: See BUGS in 'man env_parallel'. +par_bash_env_parallel env_parallel: Warning: See BUGS in 'man env_parallel'. par_ash_parset parset par_ash_parset ### parset into vars with comma par_ash_parset foo bar baz