parallel/testsuite/tests-to-run/parallel-local-100s.sh

161 lines
4.5 KiB
Bash
Raw Normal View History

#!/bin/bash
# SPDX-FileCopyrightText: 2021-2023 Ole Tange, http://ole.tange.dk and Free Software and Foundation, Inc.
#
# SPDX-License-Identifier: GPL-3.0-or-later
# Simple jobs that never fails
# Each should be taking 100-300s and be possible to run in parallel
# I.e.: No race conditions, no logins
# tmpdir with > 5 GB available
TMP5G=${TMP5G:-/dev/shm}
export TMP5G
rm -f /tmp/*.{tmx,pac,arg,all,log,swp,loa,ssh,df,pip,tmb,chr,tms,par}
par_squared() {
export PARALLEL="--load 300%"
squared() {
i=$1
i2=$[i*i]
seq $i2 | parallel -j0 --load 300% -kX echo {} | wc
seq 1 ${i2}0000 |
parallel -kj20 --recend "\n" --spreadstdin gzip -1 |
zcat | sort -n | md5sum
}
export -f squared
seq 10 -1 2 | stdout parallel -j5 -k squared |
grep -Ev 'processes took|Consider adjusting -j'
}
linebuffer_matters() {
echo "### (--linebuffer) --compress $TAG should give different output"
nolbfile=$(mktemp)
lbfile=$(mktemp)
controlfile=$(mktemp)
randomfile=$(mktemp)
# Random data because it does not compress well
# forcing the compress tool to spit out compressed blocks
perl -pe 'y/[A-Za-z]//cd; $t++ % 1000 or print "\n"' < /dev/urandom |
head -c 10000000 > "$randomfile"
export randomfile
testfunc() {
linebuffer="$1"
incompressible_ascii() {
# generate some incompressible ascii
# with lines starting with the same string
id=$1
shuf "$randomfile" | perl -pe 's/^/'$id' /'
# Sleep to give time to linebuffer-print the first part
sleep 10
shuf "$randomfile" | perl -pe 's/^/'$id' /'
echo
}
export -f incompressible_ascii
nowarn() {
# Ignore certain warnings
# parallel: Warning: Starting 11 processes took > 2 sec.
# parallel: Warning: Consider adjusting -j. Press CTRL-C to stop.
grep -v '^parallel: Warning: (Starting|Consider)' >&2
}
parallel -j0 $linebuffer --compress $TAG \
incompressible_ascii ::: {0..10} 2> >(nowarn) |
perl -ne '/^(\d+)\s/ and print "$1\n"' |
uniq |
sort
}
# These can run in parallel if there are enough ressources
testfunc > "$nolbfile"
testfunc > "$controlfile"
testfunc --linebuffer > "$lbfile"
wait
nolb="$(cat "$nolbfile")"
control="$(cat "$controlfile")"
lb="$(cat "$lbfile")"
rm "$nolbfile" "$lbfile" "$controlfile" "$randomfile"
if [ "$nolb" == "$control" ] ; then
if [ "$lb" == "$nolb" ] ; then
echo "BAD: --linebuffer makes no difference"
else
echo "OK: --linebuffer makes a difference"
fi
else
echo "BAD: control and nolb are not the same"
fi
}
export -f linebuffer_matters
par_linebuffer_matters_compress_tag() {
export TAG=--tag
linebuffer_matters
}
par_linebuffer_matters_compress() {
linebuffer_matters
}
par_linebuffer_files() {
echo 'bug #48658: --linebuffer --files'
rm -rf /tmp/par48658-*
doit() {
compress="$1"
echo "normal"
parallel --linebuffer --compress-program $compress seq ::: 100000 |
wc -l
echo "--files"
parallel --files --linebuffer --compress-program $1 seq ::: 100000 |
wc -l
echo "--results"
parallel --results /tmp/par48658-$compress --linebuffer --compress-program $compress seq ::: 100000 |
wc -l
rm -rf "/tmp/par48658-$compress"
}
export -f doit
# lrz complains 'Warning, unable to set nice value on thread'
parallel -j1 --tag -k doit ::: zstd pzstd clzip lz4 lzop pigz pxz gzip plzip pbzip2 lzma xz lzip bzip2 lbzip2 lrz
2016-08-14 14:54:27 +00:00
}
par_timeout() {
echo "### test --timeout"
stdout time -f %e parallel --timeout 1s sleep ::: 10 |
perl -ne '1 < $_ and $_ < 10 and print "OK\n"'
stdout time -f %e parallel --timeout 1m sleep ::: 100 |
perl -ne '10 < $_ and $_ < 100 and print "OK\n"'
}
par_bug57364() {
echo '### bug #57364: Race condition creating len cache file.'
j=32
set -e
for i in $(seq 1 50); do
# Clear cache (simple 'rm -rf' causes race condition)
mv "${HOME}/.parallel/tmp" "${HOME}/.parallel/tmp-$$" &&
rm -rf "${HOME}/.parallel/tmp-$$"
# Try to launch multiple parallel simultaneously.
seq $j |
xargs -P $j -n 1 parallel true $i :::
done 2>&1
}
#par_crashing() {
# echo '### bug #56322: sem crashed when running with input from seq'
# echo "### This should not fail"
# doit() { seq 100000000 |xargs -P 80 -n 1 sem true; }
# export -f doit
# parallel -j1 --timeout 100 --nice 11 doit ::: 1
#}
export -f $(compgen -A function | grep par_)
compgen -A function | grep par_ | LC_ALL=C sort |
parallel --timeout 1000% -j10 --tag -k --joblog /tmp/jl-`basename $0` '{} 2>&1'