parallel/testsuite/tests-to-run/parallel-polarhome.sh

230 lines
7.9 KiB
Bash
Raw Permalink Normal View History

2013-04-08 19:17:59 +00:00
#!/bin/bash
2024-01-24 01:02:07 +00:00
# SPDX-FileCopyrightText: 2021-2024 Ole Tange, http://ole.tange.dk and Free Software and Foundation, Inc.
#
# SPDX-License-Identifier: GPL-3.0-or-later
2015-03-03 23:12:00 +00:00
# Check servers up on http://www.polarhome.com/service/status/
unset TIMEOUT
. `which env_parallel.bash`
env_parallel --session
P_ALL="openstep qnx pidora alpha tru64 hpux-ia64 syllable raspbian solaris openindiana aix hpux debian-ppc suse solaris-x86 mandriva ubuntu scosysv unixware centos miros macosx redhat netbsd openbsd freebsd debian dragonfly vax ultrix minix irix hurd beaglebone cubieboard2"
2019-04-21 18:16:33 +00:00
# Skip ultrix
# Skip irix until Perl is upgraded (I cannot due to too small disk quota)
P_ALL="openstep qnx pidora alpha tru64 hpux-ia64 syllable raspbian solaris openindiana aix hpux debian-ppc suse solaris-x86 mandriva ubuntu scosysv unixware centos miros macosx redhat netbsd openbsd freebsd debian dragonfly vax minix hurd beaglebone cubieboard2"
2018-08-22 22:21:29 +00:00
P="$P_ALL"
#P="unixware freebsd netbsd"
# tru64 takes 22s to run 4 parallels
MAXTIME=50
RETRIES=3
# 11 too much for debian
MAXPROC=${maxproc:-9}
# 3 is too much for freebsd
MAXINNERPROC=${maxinnerproc:-2}
export PARALLEL_SSH="ssh -oLogLevel=quiet"
# select a running master (debian-ppc, suse, ubuntu, redhat, or debian)
# 2019-06-25 debian has too little free memory (and swap)
2020-04-22 18:15:59 +00:00
# 2020-04-22 debian-ppc has read-only disk
MASTER=$(parallel -j0 --delay 0.1 --halt now,success=1 $PARALLEL_SSH {} echo {} \
2020-04-22 18:15:59 +00:00
::: {ubuntu,suse,redhat}.polarhome.com)
2013-04-08 19:17:59 +00:00
parallel -j0 --delay 0.1 --retries $RETRIES \
rsync -L /usr/local/bin/{parallel,env_parallel,env_parallel.*[^~],parcat,stdout} \
::: $MASTER:bin/
2018-04-22 21:32:22 +00:00
doit() {
# Avoid the stupid /etc/issue.net banner at Polarhome: -oLogLevel=quiet
PARALLEL_SSH="ssh -oLogLevel=quiet"
export PARALLEL_SSH
export MAXTIME
export RETRIES
export MAXPROC
export RET_TIME_K="--memfree 100m -k --retries $RETRIES --timeout $MAXTIME"
LC_ALL=C
. `which env_parallel.bash`
env_parallel --session
MAXPROC=$(echo $(seq 300 | parallel -j0 echo {%} | sort -n | tail -n1) /$MAXINNERPROC | bc)
echo MAXTIME=$MAXTIME RETRIES=$RETRIES MAXPROC=$MAXPROC MAXINNERPROC=$MAXINNERPROC
2018-04-22 21:32:22 +00:00
2018-08-22 22:21:29 +00:00
echo '### Filter out working servers'
2020-01-22 17:03:29 +00:00
# pidora and syllable often gives false positive
parallel --timeout $MAXTIME -j10 ssh syllable true ::: {1..10} 2>/dev/null >/dev/null &
2020-01-22 17:03:29 +00:00
parallel --timeout $MAXTIME -j10 ssh pidora true ::: {1..10} 2>/dev/null >/dev/null &
POLAR_ALL="`bin/parallel --memfree 100m -j0 -k --timeout 10 echo {} ::: $P`"
POLAR="`bin/parallel --memfree 100m -j0 -k --timeout 10 $PARALLEL_SSH {} echo {} ::: $P`"
diff <(echo "$POLAR_ALL") <(echo "$POLAR")
S_POLAR=`bin/parallel -j0 $RET_TIME_K echo -S 1/{} ::: $POLAR`
sshwithpass() {
# Minix requires sshpass. The other servers will use ssh-keys
host="$1"
shift
if [ "$host" == "minix" ] ; then
q="$(parallel --shellquote ::: "$@")"
# This only works from suse and redhat
ssh -T -oLogLevel=quiet suse \
sshpass -f ~/.ssh/minix.password ssh -T -oLogLevel=quiet $host $q
else
ssh -oLogLevel=quiet $host "$@"
fi
}
export -f sshwithpass
2018-04-22 21:32:22 +00:00
copy() {
# scp, but atomic (avoid half files if disconnected)
host=$1
src="$2"
dst="$3"
cat "$src" |
sshwithpass $host "mkdir -p bin;cat > bin/'$dst'.tmp && chmod 755 bin/'$dst'.tmp && mv bin/'$dst'.tmp bin/'$dst'" 2>&1
2018-04-22 21:32:22 +00:00
}
export -f copy
2018-04-22 21:32:22 +00:00
par_nonall() {
parallel -j$MAXPROC $RET_TIME_K --delay 0.1 --tag \
--nonall $S_POLAR -S "1/sshwithpass minix" --argsep ,:- \
2021-08-22 20:05:37 +00:00
'source ./setupenv 2>/dev/null; . `pwd`/setupenv;' "$@"
# setupenv contains something like this (adapted to the local path and shell)
#
# PATH=$HOME/bin:$PATH:/usr/local/bin
# export PATH
# . bin/env_parallel.sh
2018-04-22 21:32:22 +00:00
}
export -f par_nonall
echo '### Copy commands to servers'
# Dont copy stdout - it depends on /bin/bash
2022-04-23 09:38:57 +00:00
env_parallel -j$MAXPROC $RET_TIME_K --delay 0.03 --tag copy {2} {1} {1/} \
::: bin/{parallel,env_parallel,env_parallel.*[^~],parcat} \
::: $POLAR minix
2018-04-22 21:32:22 +00:00
echo Done copying
env_parallel -d '\n\n' -vkj$MAXINNERPROC --delay 2 <<'EOF' |
2018-08-22 22:21:29 +00:00
echo
echo '### Works on ...'
echo
par_nonall parallel echo Works on {} ::: '`hostname`' 2>&1
echo
echo '### --number-of-cores/--number-of-cpus should work with no error'
echo
par_nonall 'parallel --number-of-sockets; parallel --number-of-cores' 2>&1
par_nonall 'parallel --number-of-threads; parallel --number-of-cpus' 2>&1
2018-04-22 21:32:22 +00:00
echo
echo '### Fails if tmpdir is R/O'
2018-04-22 21:32:22 +00:00
echo
par_nonall "stdout parallel --tmpdir / echo ::: test read-only tmp |
perl -pe '\$exit += s:/[a-z0-9_]+.arg:/XXXXXXXX.arg:gi; \$exit += s/[0-9][0-9][0-9][0-9]/0000/gi; END { exit not \$exit }' &&
echo OK readonly tmp" 2>&1
2018-04-22 21:32:22 +00:00
echo
echo '### Does exporting a bash function make parallel fail?'
echo 'If login shell is not bash compatible it fails'
2018-04-22 21:32:22 +00:00
echo
# http://zmwangx.github.io/blog/2015-11-25-bash-function-exporting-fiasco.html
par_nonall 'echo test funcA
funcA() {
cat <(echo bash only A)
}
export -f funcA;
bin/parallel funcA ::: 1' 2>&1 | LANG=C sort
echo
echo '### Does PARALLEL_SHELL help exporting a bash function'
echo 'If login shell is not bash compatible it should work'
echo
mkdir -p tmp/bin
cp /bin/bash tmp/bin
cd tmp
export PARALLEL_SHELL=bin/bash
par_nonall 'echo test funcB
funcB() {
cat <(echo bash only B)
}
export -f funcB
export PARALLEL_SHELL=bin/bash
bin/parallel funcB ::: 1' 2>&1
2018-04-22 21:32:22 +00:00
echo
echo '### env_parallel --install'
2018-08-22 22:21:29 +00:00
echo '(bash ksh mksh zsh only)'
2018-04-22 21:32:22 +00:00
echo
par_nonall 'bin/env_parallel --install && echo install-OK' 2>&1
echo
echo '### env_parallel echo env_parallel ::: run-OK'
echo '(bash ksh mksh zsh only)'
echo
2018-04-22 21:32:22 +00:00
par_nonall 'env_parallel echo env_parallel ::: run-OK' 2>&1
echo
echo '### env_parallel echo reading from process substitution :::: <(echo OK)'
echo '(bash ksh mksh zsh only)'
echo
# csh on NetBSD does not support process substitution
par_nonall 'env_parallel echo reading from process substitution :::: <(echo OK)' 2>&1 |
2018-04-22 21:32:22 +00:00
grep -v ': /tmp/.*: No such file or directory'
echo
echo '### Test empty command name in process list'
echo '(bash ksh mksh zsh only)'
echo
test_empty_cmd() {
echo '### Test if empty command name in process list causes problems'
perl -e '$0=" ";sleep 1000' &
pid=$!
parallel echo ::: OK_with_empty_cmd
kill $pid
}
export -f test_empty_cmd
export PARALLEL_SHELL=bin/bash
PARALLEL='--env test_empty_cmd' par_nonall test_empty_cmd 2>&1
2016-04-05 22:57:07 +00:00
echo
2018-04-22 21:32:22 +00:00
echo '### parset arr seq ::: 2 3 4'
2018-08-22 22:21:29 +00:00
echo '(bash ksh mksh zsh only)'
2016-04-05 22:57:07 +00:00
echo
2018-04-22 21:32:22 +00:00
par_nonall 'parset arr seq ::: 2 3 4; echo ${arr[*]}' 2>&1
echo '### env_parset arr seq ::: 2 3 4'
par_nonall 'start=2;env_parset arr seq \$start ::: 2 3 4; echo ${arr[*]}' 2>&1
echo
echo '### parset var1,var2,var3 seq ::: 2 3 4'
2018-08-22 22:21:29 +00:00
echo '(bash ksh mksh zsh ash dash only)'
2018-04-22 21:32:22 +00:00
echo
par_nonall 'parset var1,var2,var3 seq ::: 2 3 4; echo $var1,$var2,$var3' 2>&1
echo '### env_parset var1,var2,var3 seq ::: 2 3 4'
par_nonall 'start=2; env_parset var1,var2,var3 seq \$start ::: 2 3 4; echo $var1,$var2,$var3' 2>&1
EOF
perl -ne 'm{UX:sh ./bin/sh.: ERROR: source: Not found} and next;
m{/usr/X11R7/bin/.: Permission denied.} and next;
print'
}
env_parallel -u -S$MASTER doit ::: 1|
perl -pe 's:/home/(t/)?tange:~:g' |
2022-04-23 09:38:57 +00:00
perl -pe 's:par-job-\d+_\w+:parScript:g'
# eval 'myfunc() { echo '$(perl -e 'print "x"x20000')'; }'
# env_parallel myfunc ::: a | wc # OK
# eval 'myfunc2() { echo '$(perl -e 'print "x"x120000')'; }'
# env_parallel myfunc ::: a | wc # Fail too big env
2021-07-22 20:31:06 +00:00
# Supported SSH keylength:
# 16300:
# debian-ppc netbsd openbsd qnx aix centos freebsd hpux hpux-ia64
# macosx mandriva miros raspbian redhat scosysv suse tru64 unixware
# 8000:
# solaris-x86
# 4000:
# openindiana solaris