Merge branch 'master' of ssh://git.sv.gnu.org/srv/git/parallel

This commit is contained in:
Ole Tange 2017-01-06 23:03:31 +01:00
commit dacefe9c3e
4 changed files with 225 additions and 0 deletions

View file

@ -0,0 +1,126 @@
#!/bin/bash
# GNU SQL tests
# The tests must be able to run in parallel
export SQLITE=sqlite3:///%2Frun%2Fshm%2Fparallel.db
export PG=pg://`whoami`:`whoami`@lo/`whoami`
export MYSQL=mysql://`whoami`:`whoami`@lo/`whoami`
export DEBUG=false
#p_showsqlresult() {
# SERVERURL=$1
# TABLE=$2
# sql $SERVERURL "select Host,Command,V1,V2,Stdout,Stderr from $TABLE order by seq;"
#}
#
#p_wrapper() {
# INNER=$1
# SERVERURL=$(eval echo $2)
# TABLE=TBL$RANDOM
# DBURL=$SERVERURL/$TABLE
# T1=$(tempfile)
# T2=$(tempfile)
# eval "$INNER"
# echo Exit=$?
# wait
# echo Exit=$?
# $DEBUG && sort -u $T1 $T2;
# rm $T1 $T2
# p_showsqlresult $SERVERURL $TABLE
# $DEBUG || sql $SERVERURL "drop table $TABLE;" >/dev/null 2>/dev/null
#}
#
#p_template() {
# (sleep 2;
# parallel --sqlworker $DBURL "$@" sleep .3\;echo >$T1) &
# parallel --sqlandworker $DBURL "$@" sleep .3\;echo ::: {1..5} ::: {a..e} >$T2;
#}
#
#par_sqlandworker() {
# p_template
#}
#
#par_sqlandworker_lo() {
# p_template -S lo
#}
#
#par_sqlandworker_results() {
# p_template --results /tmp/out--sql
#}
#
#par_sqlandworker_linebuffer() {
# p_template --linebuffer
#}
#
#par_sqlandworker_tag() {
# p_template --tag
#}
#
#par_sqlandworker_linebuffer_tag() {
# p_template --linebuffer --tag
#}
#
#par_sqlandworker_compress_linebuffer_tag() {
# p_template --compress --linebuffer --tag
#}
#
#par_sqlandworker_unbuffer() {
# p_template -u
#}
#
#par_sqlandworker_total_jobs() {
# p_template echo {#} of '{=1 $_=total_jobs(); =};'
#}
#
#par_append() {
# parallel --sqlmaster $DBURL sleep .3\;echo ::: {1..5} ::: {a..e} >$T2;
# parallel --sqlmaster +$DBURL sleep .3\;echo ::: {11..15} ::: {A..E} >>$T2;
# parallel --sqlworker $DBURL sleep .3\;echo >$T1
#}
#
#par_shuf() {
# MD5=$(echo $SERVERURL | md5sum | perl -pe 's/(...).*/$1/')
# T=/tmp/parallel-bug49791-$MD5
# [ -e $T ] && rm -rf $T
# export PARALLEL="--shuf --result $T"
# parallel --sqlandworker $DBURL sleep .3\;echo \
# ::: {1..5} ::: {a..e} >$T2;
# parallel --sqlworker $DBURL sleep .3\;echo >$T2 &
# parallel --sqlworker $DBURL sleep .3\;echo >$T2 &
# parallel --sqlworker $DBURL sleep .3\;echo >$T2 &
# parallel --sqlworker $DBURL sleep .3\;echo >$T2 &
# unset PARALLEL
# wait;
# # Did it compute correctly?
# cat $T/1/*/*/*/stdout
# # Did it shuffle
# SHUF=$(sql $SERVERURL "select Host,Command,V1,V2,Stdout,Stderr from $TABLE order by seq;")
# export PARALLEL="--result $T"
# parallel --sqlandworker $DBURL sleep .3\;echo \
# ::: {1..5} ::: {a..e} >$T2;
# parallel --sqlworker $DBURL sleep .3\;echo >$T2 &
# parallel --sqlworker $DBURL sleep .3\;echo >$T2 &
# parallel --sqlworker $DBURL sleep .3\;echo >$T2 &
# parallel --sqlworker $DBURL sleep .3\;echo >$T2 &
# unset PARALLEL
# wait;
# NOSHUF=$(sql $SERVERURL "select Host,Command,V1,V2,Stdout,Stderr from $TABLE order by seq;")
# DIFFSIZE=$(diff <(echo "$SHUF") <(echo "$NOSHUF") | wc -c)
# if [ $DIFFSIZE -gt 2500 ]; then
# echo OK: Diff bigger than 2500 char
# fi
# [ -e $T ] && rm -rf $T
# touch $T1
#}
#
par_empty() {
true;
}
export -f $(compgen -A function | egrep 'p_|par_')
# Tested that -j0 in parallel is fastest (up to 15 jobs)
compgen -A function | grep par_ | sort |
stdout parallel -vj5 -k --tag --joblog /tmp/jl-`basename $0` p_wrapper \
:::: - ::: \$MYSQL \$PG \$SQLITE

View file

@ -0,0 +1,65 @@
#!/bin/bash
# Each test should at most run 1 ssh against parallel@lo or lo
par_path_remote_bash() {
echo 'bug #47695: How to set $PATH on remote? Bash'
rm -rf /tmp/parallel
cp /usr/local/bin/parallel /tmp
cat <<'_EOS' | stdout ssh nopathbash@lo -T | grep -Ev 'packages can be updated|System restart required|Welcome to'
echo BASH Path before: $PATH with no parallel
parallel echo ::: 1
# Race condition stderr/stdout
sleep 1
echo '^^^^^^^^ Not found is OK'
# Exporting a big variable should not fail
export A="`seq 1000`"
PATH=$PATH:/tmp
. /usr/local/bin/env_parallel.bash
# --filter to see if $PATH with parallel is transferred
env_parallel --filter --env A,PATH -Slo echo '$PATH' ::: OK
_EOS
echo
}
par_path_remote_csh() {
echo 'bug #47695: How to set $PATH on remote? csh'
rm -rf /tmp/parallel
cp /usr/local/bin/parallel /tmp
cat <<'_EOS' | stdout ssh nopathcsh@lo -T | grep -Ev 'packages can be updated|System restart required|Welcome to'
echo CSH Path before: $PATH with no parallel
which parallel >& /dev/stdout
echo '^^^^^^^^ Not found is OK'
alias parallel=/tmp/parallel
# Exporting a big variable should not fail
setenv A "`seq 1000`"
setenv PATH ${PATH}:/tmp
cp /usr/local/bin/env_parallel* /tmp
if ("`alias env_parallel`" == '') then
source `which env_parallel.csh`
endif
# --filter to see if $PATH with parallel is transferred
env_parallel --filter --env A,PATH -Slo echo '$PATH' ::: OK
echo Right now it seems csh does not respect $PATH if set from Perl
echo Done
_EOS
}
par_keep_order() {
echo '### Test --keep-order'
seq 0 2 |
parallel --keep-order -j100% -S 1/:,2/parallel@lo -q perl -e 'sleep 1;print "job{}\n";exit({})'
}
par_keeporder() {
echo '### Test --keeporder'
seq 0 2 |
parallel --keeporder -j100% -S 1/:,2/parallel@lo -q perl -e 'sleep 1;print "job{}\n";exit({})'
}
export -f $(compgen -A function | grep par_)
#compgen -A function | grep par_ | sort | parallel --delay $D -j$P --tag -k '{} 2>&1'
compgen -A function | grep par_ | sort | parallel --delay 0.1 -j10 --tag -k '{} 2>&1'

View file

@ -0,0 +1,6 @@
p_wrapper par_empty \$MYSQL
par_empty $MYSQL /bin/bash: p_wrapper: command not found
p_wrapper par_empty \$PG
par_empty $PG /bin/bash: p_wrapper: command not found
p_wrapper par_empty \$SQLITE
par_empty $SQLITE /bin/bash: p_wrapper: command not found

View file

@ -0,0 +1,28 @@
par_keep_order ### Test --keep-order
par_keep_order job0
par_keep_order job1
par_keep_order job2
par_keeporder ### Test --keeporder
par_keeporder job0
par_keeporder job1
par_keeporder job2
par_path_remote_bash bug #47695: How to set $PATH on remote? Bash
par_path_remote_bash
par_path_remote_bash * Documentation: http://www.linuxmint.com
par_path_remote_bash BASH Path before: /bin:/usr/bin with no parallel
par_path_remote_bash -bash: line 2: parallel: command not found
par_path_remote_bash ^^^^^^^^ Not found is OK
par_path_remote_bash /bin:/usr/bin:/tmp OK
par_path_remote_bash
par_path_remote_csh bug #47695: How to set $PATH on remote? csh
par_path_remote_csh
par_path_remote_csh * Documentation: http://www.linuxmint.com
par_path_remote_csh Warning: no access to tty (Bad file descriptor).
par_path_remote_csh Thus no job control in this shell.
par_path_remote_csh CSH Path before: /bin:/usr/bin with no parallel
par_path_remote_csh parallel: Command not found.
par_path_remote_csh ^^^^^^^^ Not found is OK
par_path_remote_csh parallel: Warning: Removed lo.
par_path_remote_csh parallel: Error: There are no hosts left to run on.
par_path_remote_csh Right now it seems csh does not respect /bin:/usr/bin:/tmp if set from Perl
par_path_remote_csh Done