From c4100ef3d7d55cd3570844820587f04e66ab6e92 Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Wed, 22 Oct 2014 21:59:06 +0200 Subject: [PATCH] parallel: passes testsuite. --- doc/release_new_version | 4 +++ src/parallel | 4 +-- src/parallel.pod | 29 ++++++++++++++++++-- testsuite/tests-to-run/niceload02.sh | 8 ++++-- testsuite/wanted-results/niceload02 | 2 +- testsuite/wanted-results/parallel-local-ssh1 | 8 +++--- 6 files changed, 43 insertions(+), 12 deletions(-) diff --git a/doc/release_new_version b/doc/release_new_version index 563e36dd..003679e1 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -252,12 +252,16 @@ New in this release: * GNU Parallel was used (unfortunately without citation) in: De Novo Insertions and Deletions of Predominantly Paternal Origin Are Associated with Autism Spectrum Disorder http://www.cell.com/cms/attachment/2019079667/2039310868/mmc1.pdf +* GNU Parallel was determined 'popular' by Statistical Analysis of Popular Open Source Software Projects and Their Communities http://andiwre.itmaranatha.org/prosiding/ICITEE2014/PaperAndiWRE-ICITEE2014.pdf + * GNU Parallel is demonstrated in: Data Science at the Command Line: Facing the Future with Time-Tested Tools * GNU Parallel was covered in a talk at Sydney Perl Mongers group http://www.openfusion.net/talks/pwp/#24 * GNU Parallel and how to get started with it http://www.jduck.net/blog/2014/09/30/gnu-paralell/ +* Behat/Selenium/Parallel http://jaffamonkey.com/behat-selenium-parallel/ + * Comparing Golang, Scala, Elixir and Ruby for ETL http://www.dimroc.com/2014/09/29/etl-language-showdown/ * Parallelizing Batch Jobs for Fun and Profit http://mikeseidle.com/tech/programming/2013/03/parallelizing-batch-jobs diff --git a/src/parallel b/src/parallel index f5447f52..18c2318a 100755 --- a/src/parallel +++ b/src/parallel @@ -826,7 +826,7 @@ sub parse_options { if(@opt::v) { $Global::verbose = $#opt::v+1; } # Convert -v -v to v=2 $Global::debug = $opt::D; - $Global::shell = parent_shell($$) || $ENV{'SHELL'} || "/bin/sh"; + $Global::shell = $ENV{'PARALLEL_SHELL'} || parent_shell($$) || $ENV{'SHELL'} || "/bin/sh"; if(defined $opt::X) { $Global::ContextReplace = 1; } if(defined $opt::silent) { $Global::verbose = 0; } if(defined $opt::group) { $Global::grouped = 1; } @@ -1152,7 +1152,7 @@ sub parse_env_var { my @qcsh = (map { my $a=$_; "setenv $a " . env_quote($ENV{$a}) } grep { not /^parallel_bash_environment$/ } @non_functions); my @qbash = (map { my $a=$_; "export $a=" . env_quote($ENV{$a}) } - @non_functions, @bash_pre_shellshock); + grep { not /^parallel_bash_environment$/ } @non_functions, @bash_pre_shellshock); push @qbash, map { my $a=$_; "eval $a\"\$$a\"" } @bash_pre_shellshock; push @qbash, map { /BASH_FUNC_(.*)\(\)/; "$1 $ENV{$_}" } @bash_post_shellshock; diff --git a/src/parallel.pod b/src/parallel.pod index f4db40cb..afd5b325 100644 --- a/src/parallel.pod +++ b/src/parallel.pod @@ -81,7 +81,7 @@ first. To use aliases copy the full environment as described under B<--env> and use B instead of B. If it is a zsh function you will need to use this helper function -B to export and to set $SHELL to bash: +B to export and to set $PARALLEL_SHELL to bash: function exportf (){ export $(echo $1)="`whence -f $1 | sed -e "s/$1 //" `" @@ -93,7 +93,7 @@ B to export and to set $SHELL to bash: } exportf my_func - SHELL=/bin/bash parallel "my_func {}" ::: 1 2 + PARALLEL_SHELL=/bin/bash parallel "my_func {}" ::: 1 2 The command cannot contain the character \257 (macron: ¯). @@ -3198,6 +3198,31 @@ by: B. This only works on the local computer. +=item $PARALLEL_SHELL (alpha testing) + +Use this shell the shell for the commands run by GNU Parallel: + +=over 2 + +=item * + +$PARALLEL_SHELL. If undefined use: + +=item * + +The shell that started GNU Parallel. If that cannot be determined: + +=item * + +$SHELL. If undefined use: + +=item * + +/bin/sh + +=back + + =item $PARALLEL_SEQ $PARALLEL_SEQ will be set to the sequence number of the job diff --git a/testsuite/tests-to-run/niceload02.sh b/testsuite/tests-to-run/niceload02.sh index cee07087..0492234f 100755 --- a/testsuite/tests-to-run/niceload02.sh +++ b/testsuite/tests-to-run/niceload02.sh @@ -14,6 +14,8 @@ perl -e '$|=1;while($t++<3){sleep(1);print "."}' & stdout /usr/bin/time -f %e niceload -l 8 -p $! | perl -ne '$_ >= 5 and print "OK\n"' echo "### Test --sensor -l negative" -timeout 10 nice nice dd iflag=fullblock if=/dev/zero of=/dev/null bs=11G & -niceload -t 1 --sensor 'free | field 3 | head -3|tail -1' -l -10000000 "free -g|egrep -q /.*1[0-9]. && echo more than 6 GB used" - +# When the size is bigger, then run +SIZET=/tmp/parallel_sizetest +rm -f $SIZET +tmux new-session -d -n 10 "seq 10000 | pv -qL 1000 > $SIZET" +niceload -t .01 --sensor "stat -c %b $SIZET" -l -10 "stat -c %b $SIZET" diff --git a/testsuite/wanted-results/niceload02 b/testsuite/wanted-results/niceload02 index 524c7575..9264fbc8 100644 --- a/testsuite/wanted-results/niceload02 +++ b/testsuite/wanted-results/niceload02 @@ -4,4 +4,4 @@ ### Test -p ...OK ### Test --sensor -l negative -more than 6 GB used +16 diff --git a/testsuite/wanted-results/parallel-local-ssh1 b/testsuite/wanted-results/parallel-local-ssh1 index 795134ac..e932c50b 100644 --- a/testsuite/wanted-results/parallel-local-ssh1 +++ b/testsuite/wanted-results/parallel-local-ssh1 @@ -14,22 +14,22 @@ echo '### bug #41805: Idea: propagate --env for parallel --number-of-cores' ** test_zsh FOO=test_zsh parallel --env FOO,PATH -S zsh@lo env ::: "" |sort|egrep 'FOO|PATH' FOO=test_zsh -PATH=/home/tange/bin:/home/tange/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/sbin:/usr/sbin:/home/tange/terminals/data-import:/usr/lib/oracle/xe/app/oracle/product/10.2.0/server/bin:/home/tange/bin +PATH=/home/tange/bin:/home/tange/bin:/home/tange/bin:/home/tange/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/sbin:/usr/sbin:/home/tange/terminals/data-import:/usr/lib/oracle/xe/app/oracle/product/10.2.0/server/bin:/home/tange/bin:/sbin:/usr/sbin:/home/tange/terminals/data-import:/usr/lib/oracle/xe/app/oracle/product/10.2.0/server/bin:/home/tange/bin:/sbin:/usr/sbin:/usr/lib/oracle/xe/app/oracle/product/10.2.0/server/bin echo '** test_zsh_filter' ** test_zsh_filter FOO=test_zsh_filter parallel --filter-hosts --env FOO,PATH -S zsh@lo env ::: "" |sort|egrep 'FOO|PATH' FOO=test_zsh_filter -PATH=/home/tange/bin:/home/tange/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/sbin:/usr/sbin:/home/tange/terminals/data-import:/usr/lib/oracle/xe/app/oracle/product/10.2.0/server/bin:/home/tange/bin +PATH=/home/tange/bin:/home/tange/bin:/home/tange/bin:/home/tange/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/sbin:/usr/sbin:/home/tange/terminals/data-import:/usr/lib/oracle/xe/app/oracle/product/10.2.0/server/bin:/home/tange/bin:/sbin:/usr/sbin:/home/tange/terminals/data-import:/usr/lib/oracle/xe/app/oracle/product/10.2.0/server/bin:/home/tange/bin:/sbin:/usr/sbin:/usr/lib/oracle/xe/app/oracle/product/10.2.0/server/bin echo '** test_csh' ** test_csh FOO=test_csh parallel --env FOO,PATH -S csh@lo env ::: "" |sort|egrep 'FOO|PATH' FOO=test_csh -PATH=/home/tange/bin:/home/tange/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/sbin:/usr/sbin:/home/tange/terminals/data-import:/usr/lib/oracle/xe/app/oracle/product/10.2.0/server/bin:/home/tange/bin +PATH=/home/tange/bin:/home/tange/bin:/home/tange/bin:/home/tange/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/sbin:/usr/sbin:/home/tange/terminals/data-import:/usr/lib/oracle/xe/app/oracle/product/10.2.0/server/bin:/home/tange/bin:/sbin:/usr/sbin:/home/tange/terminals/data-import:/usr/lib/oracle/xe/app/oracle/product/10.2.0/server/bin:/home/tange/bin:/sbin:/usr/sbin:/usr/lib/oracle/xe/app/oracle/product/10.2.0/server/bin echo '** test_csh_filter' ** test_csh_filter FOO=test_csh_filter parallel --filter-hosts --env FOO,PATH -S csh@lo env ::: "" |sort|egrep 'FOO|PATH' FOO=test_csh_filter -PATH=/home/tange/bin:/home/tange/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/sbin:/usr/sbin:/home/tange/terminals/data-import:/usr/lib/oracle/xe/app/oracle/product/10.2.0/server/bin:/home/tange/bin +PATH=/home/tange/bin:/home/tange/bin:/home/tange/bin:/home/tange/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/sbin:/usr/sbin:/home/tange/terminals/data-import:/usr/lib/oracle/xe/app/oracle/product/10.2.0/server/bin:/home/tange/bin:/sbin:/usr/sbin:/home/tange/terminals/data-import:/usr/lib/oracle/xe/app/oracle/product/10.2.0/server/bin:/home/tange/bin:/sbin:/usr/sbin:/usr/lib/oracle/xe/app/oracle/product/10.2.0/server/bin echo '** bug #41805 done' ** bug #41805 done echo '### Deal with long command lines on remote servers'