parallel/testsuite/tests-to-run/parallel-local-sql01.sh

154 lines
4.3 KiB
Bash
Raw Normal View History

2017-01-06 21:53:21 +00:00
#!/bin/bash
# SPDX-FileCopyrightText: 2021-2022 Ole Tange, http://ole.tange.dk and Free Software and Foundation, Inc.
#
# SPDX-License-Identifier: GPL-3.0-or-later
# GNU Parallel SQL tests
2017-01-06 21:53:21 +00:00
# The tests must be able to run in parallel
export SQLITE=sqlite3:///%2Frun%2Fshm%2Fparallel.db
export PG=pg://`whoami`:`whoami`@lo/`whoami`
export MYSQL=mysql://`whoami`:`whoami`@lo/`whoami`
export CSV=csv:///%2Frun%2Fshm
2017-01-06 21:53:21 +00:00
export DEBUG=false
rm -f /run/shm/parallel.db
mkdir -p /run/shm/csv
p_showsqlresult() {
2022-05-13 23:27:17 +00:00
# print results stored in $SERVERURL/$TABLE
SERVERURL=$1
TABLE=$2
sql $SERVERURL "select Host,Command,V1,V2,Stdout,Stderr from $TABLE order by seq;"
}
p_wrapper() {
2022-05-13 23:27:17 +00:00
INNER=$1
SERVERURL=$(eval echo $2)
# Use a random table for each test
TABLE=TBL$RANDOM
DBURL=$SERVERURL/$TABLE
2022-05-22 08:36:45 +00:00
T1=$(mktemp)
T2=$(mktemp)
2022-05-13 23:27:17 +00:00
# Run $INNER (all the par_* functions)
eval "$INNER"
echo Exit=$?
# $INNER can start background processes - wait for those
wait
echo Exit=$?
# For debugging show the tempfiles
$DEBUG && sort -u "$T1" "$T2";
rm "$T1" "$T2"
2022-05-13 23:27:17 +00:00
p_showsqlresult $SERVERURL $TABLE
# Drop the table if not debugging
$DEBUG || sql $SERVERURL "drop table $TABLE;" >/dev/null 2>/dev/null
}
p_template() {
2022-05-13 23:27:17 +00:00
# Run the
(
# Make sure there is work to be done
sleep 6;
parallel --sqlworker $DBURL "$@" sleep .3\;echo >"$T1"
2022-05-13 23:27:17 +00:00
) &
parallel --sqlandworker $DBURL "$@" sleep .3\;echo ::: {1..5} ::: {a..e} >"$T2";
}
par_sqlandworker() {
2022-05-13 23:27:17 +00:00
p_template
}
par_sqlandworker_lo() {
2022-05-13 23:27:17 +00:00
p_template -S lo
}
par_sqlandworker_results() {
2022-05-13 23:27:17 +00:00
p_template --results /tmp/out--sql
}
par_sqlandworker_linebuffer() {
2022-05-13 23:27:17 +00:00
p_template --linebuffer
}
par_sqlandworker_tag() {
2022-05-13 23:27:17 +00:00
p_template --tag
}
par_sqlandworker_linebuffer_tag() {
2022-05-13 23:27:17 +00:00
p_template --linebuffer --tag
}
par_sqlandworker_compress_linebuffer_tag() {
2022-05-13 23:27:17 +00:00
p_template --compress --linebuffer --tag
}
par_sqlandworker_unbuffer() {
2022-05-13 23:27:17 +00:00
p_template -u
}
par_sqlandworker_total_jobs() {
2022-05-13 23:27:17 +00:00
p_template echo {#} of '{=1 $_=total_jobs(); =};'
}
par_append() {
parallel --sqlmaster $DBURL sleep .3\;echo ::: {1..5} ::: {a..e} >"$T2";
parallel --sqlmaster +$DBURL sleep .3\;echo ::: {11..15} ::: {A..E} >>"$T2";
parallel --sqlworker $DBURL sleep .3\;echo >"$T1"
}
par_shuf() {
2022-05-13 23:27:17 +00:00
MD5=$(echo $SERVERURL | md5sum | perl -pe 's/(...).*/$1/')
T=/tmp/parallel-bug49791-$MD5
[ -e $T ] && rm -rf $T
export PARALLEL="--shuf --result $T"
parallel --sqlandworker $DBURL sleep .3\;echo \
::: {1..5} ::: {a..e} >"$T2";
parallel --sqlworker $DBURL sleep .3\;echo >"$T2" &
parallel --sqlworker $DBURL sleep .3\;echo >"$T2" &
parallel --sqlworker $DBURL sleep .3\;echo >"$T2" &
parallel --sqlworker $DBURL sleep .3\;echo >"$T2" &
2022-05-13 23:27:17 +00:00
unset PARALLEL
wait;
# Did it compute correctly?
cat $T/1/*/*/*/stdout
# Did it shuffle (Compare job table to non-shuffled)
SHUF=$(sql $SERVERURL "select Host,Command,V1,V2,Stdout,Stderr from $TABLE order by seq;")
export PARALLEL="--result $T"
parallel --sqlandworker $DBURL sleep .3\;echo \
::: {1..5} ::: {a..e} >"$T2";
parallel --sqlworker $DBURL sleep .3\;echo >"$T2" &
parallel --sqlworker $DBURL sleep .3\;echo >"$T2" &
parallel --sqlworker $DBURL sleep .3\;echo >"$T2" &
parallel --sqlworker $DBURL sleep .3\;echo >"$T2" &
2022-05-13 23:27:17 +00:00
unset PARALLEL
wait;
NOSHUF=$(sql $SERVERURL "select Host,Command,V1,V2,Stdout,Stderr from $TABLE order by seq;")
DIFFSIZE=$(diff <(echo "$SHUF") <(echo "$NOSHUF") | wc -c)
if [ $DIFFSIZE -gt 2500 ]; then
echo OK: Diff bigger than 2500 char
fi
[ -e $T ] && rm -rf $T
touch "$T1"
}
2017-01-06 21:53:21 +00:00
par_empty() {
2022-05-13 23:27:17 +00:00
echo Do nothing: TBL99999 does not exist because it is not created
2017-01-06 21:53:21 +00:00
true;
}
2018-08-22 22:21:29 +00:00
hostname=`hostname`
2017-01-06 21:53:21 +00:00
export -f $(compgen -A function | egrep 'p_|par_')
# Tested that -j0 in parallel is fastest (up to 15 jobs)
# -j5: SQLite complains about locked database.
2017-01-06 21:53:21 +00:00
compgen -A function | grep par_ | sort |
2022-05-13 23:27:17 +00:00
stdout parallel -vj4 -k --tag --joblog /tmp/jl-`basename $0` p_wrapper \
:::: - ::: \$MYSQL \$PG \$SQLITE \$CSV |
perl -pe 's/tbl\d+/TBL99999/gi;' |
perl -pe 's/(from TBL99999 order) .*/$1/g' |
perl -pe 's/ *\b'"$hostname"'\b */hostname/g' |
grep -v -- --------------- |
perl -pe 's/ *\bhost\b */host/g' |
perl -pe 's/ +/ /g'