2013-12-19 01:19:19 +00:00
|
|
|
#!/bin/bash
|
|
|
|
|
2021-12-26 22:29:28 +00:00
|
|
|
# SPDX-FileCopyrightText: 2021-2022 Ole Tange, http://ole.tange.dk and Free Software and Foundation, Inc.
|
2021-03-22 20:16:35 +00:00
|
|
|
#
|
|
|
|
# SPDX-License-Identifier: GPL-3.0-or-later
|
|
|
|
|
2013-12-19 01:19:19 +00:00
|
|
|
# Simple jobs that never fails
|
2020-01-12 08:14:21 +00:00
|
|
|
# Each should be taking 100-300s and be possible to run in parallel
|
2013-12-19 01:19:19 +00:00
|
|
|
# I.e.: No race conditions, no logins
|
2015-07-22 06:28:05 +00:00
|
|
|
|
2016-06-27 19:00:19 +00:00
|
|
|
# tmpdir with > 5 GB available
|
2015-07-22 06:28:05 +00:00
|
|
|
TMP5G=${TMP5G:-/dev/shm}
|
|
|
|
export TMP5G
|
|
|
|
|
2016-08-03 21:45:13 +00:00
|
|
|
rm -f /tmp/*.{tmx,pac,arg,all,log,swp,loa,ssh,df,pip,tmb,chr,tms,par}
|
|
|
|
|
2020-12-30 12:42:02 +00:00
|
|
|
par_squared() {
|
|
|
|
export PARALLEL="--load 300%"
|
|
|
|
squared() {
|
|
|
|
i=$1
|
|
|
|
i2=$[i*i]
|
|
|
|
seq $i2 | parallel -j0 --load 300% -kX echo {} | wc
|
|
|
|
seq 1 ${i2}0000 |
|
|
|
|
parallel -kj20 --recend "\n" --spreadstdin gzip -1 |
|
|
|
|
zcat | sort -n | md5sum
|
|
|
|
}
|
|
|
|
export -f squared
|
|
|
|
|
|
|
|
seq 10 -1 2 | stdout parallel -j5 -k squared |
|
|
|
|
grep -Ev 'processes took|Consider adjusting -j'
|
|
|
|
}
|
|
|
|
|
2020-01-12 08:14:21 +00:00
|
|
|
linebuffer_matters() {
|
|
|
|
echo "### (--linebuffer) --compress $TAG should give different output"
|
|
|
|
nolbfile=$(mktemp)
|
|
|
|
lbfile=$(mktemp)
|
|
|
|
controlfile=$(mktemp)
|
|
|
|
randomfile=$(mktemp)
|
|
|
|
# Random data because it does not compress well
|
|
|
|
# forcing the compress tool to spit out compressed blocks
|
|
|
|
perl -pe 'y/[A-Za-z]//cd; $t++ % 1000 or print "\n"' < /dev/urandom |
|
2023-03-23 21:05:14 +00:00
|
|
|
head -c 10000000 > "$randomfile"
|
2020-01-12 08:14:21 +00:00
|
|
|
export randomfile
|
|
|
|
|
|
|
|
testfunc() {
|
|
|
|
linebuffer="$1"
|
|
|
|
|
|
|
|
incompressible_ascii() {
|
|
|
|
# generate some incompressible ascii
|
|
|
|
# with lines starting with the same string
|
|
|
|
id=$1
|
2023-03-23 21:05:14 +00:00
|
|
|
shuf "$randomfile" | perl -pe 's/^/'$id' /'
|
2020-01-12 08:14:21 +00:00
|
|
|
# Sleep to give time to linebuffer-print the first part
|
|
|
|
sleep 10
|
2023-03-23 21:05:14 +00:00
|
|
|
shuf "$randomfile" | perl -pe 's/^/'$id' /'
|
2020-01-12 08:14:21 +00:00
|
|
|
echo
|
|
|
|
}
|
|
|
|
export -f incompressible_ascii
|
2018-01-06 18:58:01 +00:00
|
|
|
|
2020-01-12 08:14:21 +00:00
|
|
|
nowarn() {
|
|
|
|
# Ignore certain warnings
|
|
|
|
# parallel: Warning: Starting 11 processes took > 2 sec.
|
|
|
|
# parallel: Warning: Consider adjusting -j. Press CTRL-C to stop.
|
|
|
|
grep -v '^parallel: Warning: (Starting|Consider)' >&2
|
|
|
|
}
|
|
|
|
|
|
|
|
parallel -j0 $linebuffer --compress $TAG \
|
|
|
|
incompressible_ascii ::: {0..10} 2> >(nowarn) |
|
|
|
|
perl -ne '/^(\d+)\s/ and print "$1\n"' |
|
|
|
|
uniq |
|
|
|
|
sort
|
2018-01-06 18:58:01 +00:00
|
|
|
}
|
|
|
|
|
2020-01-12 08:14:21 +00:00
|
|
|
# These can run in parallel if there are enough ressources
|
2023-03-23 21:05:14 +00:00
|
|
|
testfunc > "$nolbfile"
|
|
|
|
testfunc > "$controlfile"
|
|
|
|
testfunc --linebuffer > "$lbfile"
|
2020-01-12 08:14:21 +00:00
|
|
|
wait
|
2018-01-06 18:58:01 +00:00
|
|
|
|
2023-03-23 21:05:14 +00:00
|
|
|
nolb="$(cat "$nolbfile")"
|
|
|
|
control="$(cat "$controlfile")"
|
|
|
|
lb="$(cat "$lbfile")"
|
|
|
|
rm "$nolbfile" "$lbfile" "$controlfile" "$randomfile"
|
2016-07-10 21:00:20 +00:00
|
|
|
|
2020-01-12 08:14:21 +00:00
|
|
|
if [ "$nolb" == "$control" ] ; then
|
|
|
|
if [ "$lb" == "$nolb" ] ; then
|
|
|
|
echo "BAD: --linebuffer makes no difference"
|
|
|
|
else
|
|
|
|
echo "OK: --linebuffer makes a difference"
|
|
|
|
fi
|
|
|
|
else
|
|
|
|
echo "BAD: control and nolb are not the same"
|
|
|
|
fi
|
2016-07-10 21:00:20 +00:00
|
|
|
}
|
2020-01-12 08:14:21 +00:00
|
|
|
export -f linebuffer_matters
|
2016-07-10 21:00:20 +00:00
|
|
|
|
2020-01-12 08:14:21 +00:00
|
|
|
par_linebuffer_matters_compress_tag() {
|
|
|
|
export TAG=--tag
|
|
|
|
linebuffer_matters
|
2016-07-10 21:00:20 +00:00
|
|
|
}
|
|
|
|
|
2020-01-12 08:14:21 +00:00
|
|
|
par_linebuffer_matters_compress() {
|
|
|
|
linebuffer_matters
|
|
|
|
}
|
2018-05-08 21:16:48 +00:00
|
|
|
|
2020-01-12 08:14:21 +00:00
|
|
|
par_linebuffer_files() {
|
|
|
|
echo 'bug #48658: --linebuffer --files'
|
|
|
|
rm -rf /tmp/par48658-*
|
|
|
|
|
|
|
|
doit() {
|
|
|
|
compress="$1"
|
|
|
|
echo "normal"
|
|
|
|
parallel --linebuffer --compress-program $compress seq ::: 100000 |
|
|
|
|
wc -l
|
|
|
|
echo "--files"
|
|
|
|
parallel --files --linebuffer --compress-program $1 seq ::: 100000 |
|
|
|
|
wc -l
|
|
|
|
echo "--results"
|
|
|
|
parallel --results /tmp/par48658-$compress --linebuffer --compress-program $compress seq ::: 100000 |
|
|
|
|
wc -l
|
|
|
|
rm -rf "/tmp/par48658-$compress"
|
2017-07-21 19:43:34 +00:00
|
|
|
}
|
2020-01-12 08:14:21 +00:00
|
|
|
export -f doit
|
|
|
|
# lrz complains 'Warning, unable to set nice value on thread'
|
|
|
|
parallel -j1 --tag -k doit ::: zstd pzstd clzip lz4 lzop pigz pxz gzip plzip pbzip2 lzma xz lzip bzip2 lbzip2 lrz
|
2016-08-14 14:54:27 +00:00
|
|
|
}
|
|
|
|
|
2017-05-15 23:27:11 +00:00
|
|
|
par_timeout() {
|
|
|
|
echo "### test --timeout"
|
|
|
|
stdout time -f %e parallel --timeout 1s sleep ::: 10 |
|
|
|
|
perl -ne '1 < $_ and $_ < 10 and print "OK\n"'
|
|
|
|
stdout time -f %e parallel --timeout 1m sleep ::: 100 |
|
|
|
|
perl -ne '10 < $_ and $_ < 100 and print "OK\n"'
|
|
|
|
}
|
|
|
|
|
2020-06-25 11:07:02 +00:00
|
|
|
par_bug57364() {
|
|
|
|
echo '### bug #57364: Race condition creating len cache file.'
|
|
|
|
j=32
|
|
|
|
set -e
|
|
|
|
for i in $(seq 1 50); do
|
|
|
|
# Clear cache (simple 'rm -rf' causes race condition)
|
|
|
|
mv "${HOME}/.parallel/tmp" "${HOME}/.parallel/tmp-$$" &&
|
|
|
|
rm -rf "${HOME}/.parallel/tmp-$$"
|
|
|
|
# Try to launch multiple parallel simultaneously.
|
|
|
|
seq $j |
|
|
|
|
xargs -P $j -n 1 parallel true $i :::
|
|
|
|
done 2>&1
|
|
|
|
}
|
|
|
|
|
2019-05-19 23:05:14 +00:00
|
|
|
#par_crashing() {
|
|
|
|
# echo '### bug #56322: sem crashed when running with input from seq'
|
|
|
|
# echo "### This should not fail"
|
|
|
|
# doit() { seq 100000000 |xargs -P 80 -n 1 sem true; }
|
|
|
|
# export -f doit
|
|
|
|
# parallel -j1 --timeout 100 --nice 11 doit ::: 1
|
|
|
|
#}
|
|
|
|
|
2016-07-10 21:00:20 +00:00
|
|
|
export -f $(compgen -A function | grep par_)
|
2020-01-12 08:14:21 +00:00
|
|
|
compgen -A function | grep par_ | LC_ALL=C sort |
|
|
|
|
parallel --timeout 1000% -j10 --tag -k --joblog /tmp/jl-`basename $0` '{} 2>&1'
|