parallel/testsuite/tests-to-run/parallel-local-sql.sh

#!/bin/bash

# SPDX-FileCopyrightText: 2021-2022 Ole Tange, http://ole.tange.dk and Free Software and Foundation, Inc.
#
# SPDX-License-Identifier: GPL-3.0-or-later

export SQLITE=sqlite3:///%2Frun%2Fshm%2Fparallel.db
export PG=pg://`whoami`:`whoami`@lo/`whoami`
export MYSQL=mysql://`whoami`:`whoami`@lo/`whoami`

export DEBUG=false

p_showsqlresult() {
    SERVERURL=$1
    TABLE=$2
    # No hostname as it can differ
    sql $SERVERURL "select Command,V1,V2,Stdout,Stderr from $TABLE order by seq;"
}

p_wrapper() {
    INNER=$1
    SERVERURL=$(eval echo $2)
    TABLE=TBL$RANDOM
    DBURL=$SERVERURL/$TABLE
    T1=$(mktemp)
    T2=$(mktemp)
    eval "$INNER"
    echo Exit=$?
    wait
    echo Exit=$?
    $DEBUG && sort -u "$T1" "$T2";
    rm "$T1" "$T2"
    p_showsqlresult $SERVERURL $TABLE
    $DEBUG || sql $SERVERURL "drop table $TABLE;" >/dev/null 2>/dev/null
}

p_template() {
    (
	# Make sure all jobs are inserted before starting a worker
	sleep 10;
	parallel --sqlworker $DBURL    "$@" sleep .3\;echo >"$T1"
    ) &
    parallel  --sqlandworker $DBURL "$@" sleep .3\;echo ::: {1..5} ::: {a..e} >"$T2";
}

par_sqlandworker() {
    p_template
}

par_sqlandworker_lo() {
    p_template -S lo
}

par_sqlandworker_results() {
    p_template --results /tmp/out--sql
}

par_sqlandworker_tag() {
    p_template --tag
}

par_sqlandworker_unbuffer() {
    p_template -u
}

par_sqlandworker_total_jobs() {
    p_template echo {#} of '{=1 $_=total_jobs(); =};'
}

par_append_different_cmd() {
    parallel --sqlmaster  "$DBURL" sleep .3\;echo ::: {1..5} ::: {a..e} >"$T2";
    parallel --sqlmaster +"$DBURL" sleep .3\;echo {2}-{1} ::: {11..15} ::: {A..E} >>"$T2";
    parallel --sqlworker  "$DBURL" >"$T1"
}

par_shuf() {
    MD5=$(echo "$SERVERURL" | md5sum | perl -pe 's/(...).*/$1/')
    T=/tmp/parallel-bug49791-"  <$MD5"
    [ -e "$T" ] && rm -rf "$T"
    export PARALLEL="--shuf --result '$T'"
    parallel --sqlandworker $DBURL sleep .3\;echo \
	     ::: {1..5} ::: {a..e} >"$T2";
    parallel --sqlworker    $DBURL >"$T2" &
    parallel --sqlworker    $DBURL >"$T2" &
    parallel --sqlworker    $DBURL >"$T2" &
    parallel --sqlworker    $DBURL >"$T2" &
    unset PARALLEL
    wait;
    # Did it compute correctly?
    cat "$T"/1/*/*/*/stdout
    # Did it shuffle
    SHUF=$(sql "$SERVERURL" "select Host,Command,V1,V2,Stdout,Stderr from $TABLE order by seq;")
    export PARALLEL="--result '$T'"
    parallel --sqlandworker "$DBURL" sleep .3\;echo \
	     ::: {1..5} ::: {a..e} >"$T2";
    parallel --sqlworker    "$DBURL" >"$T2" &
    parallel --sqlworker    "$DBURL" >"$T2" &
    parallel --sqlworker    "$DBURL" >"$T2" &
    parallel --sqlworker    "$DBURL" >"$T2" &
    unset PARALLEL
    wait;
    NOSHUF=$(sql $SERVERURL "select Host,Command,V1,V2,Stdout,Stderr from $TABLE order by seq;")
    DIFFSIZE=$(diff <(echo "$SHUF") <(echo "$NOSHUF") | wc -c)
    if [ $DIFFSIZE -gt 2500 ]; then
	echo OK: Diff bigger than 2500 char
    fi
    [ -e "$T" ] && rm -rf "$T"
    touch "$T1"
}

par_sql_joblog() {
    echo '### should only give a single --joblog heading'
    echo '### --sqlmaster/--sqlworker'
    parallel -k --joblog - --sqlmaster $DBURL --wait sleep .3\;echo ::: {1..5} ::: {a..e} |
	perl -pe 's/\d+\.\d+/999.999/g' | sort -n &
    sleep 0.5
    T=$(mktemp)
    parallel -k --joblog - --sqlworker $DBURL > "$T"
    wait
    # Needed because of race condition
    cat "$T"; rm "$T"
    echo '### --sqlandworker'
    parallel -k --joblog - --sqlandworker $DBURL sleep .3\;echo ::: {1..5} ::: {a..e} |
	perl -pe 's/\d+\.\d+/999.999/g' | sort -n
    # TODO --sqlandworker --wait
}

par_no_table() {
    echo 'bug #50018: --dburl without table dies'
    parallel --sqlworker $SERVERURL
    echo $?
    parallel --sqlandworker $SERVERURL echo ::: no_output
    echo $?
    parallel --sqlmaster $SERVERURL echo ::: no_output
    echo $?
    # For p_wrapper to remove table
    parallel --sqlandworker $DBURL true ::: dummy ::: dummy
}

export -f $(compgen -A function | egrep 'p_|par_')
# Tested that -j0 in parallel is fastest (up to 15 jobs)
# more than 3 jobs: sqlite locks
export LC_ALL=C
compgen -A function | grep par_ | sort |
  stdout parallel --timeout 2000 -vj50% -k --tag --joblog /tmp/jl-`basename $0` p_wrapper \
    :::: - ::: \$MYSQL \$PG \$SQLITE