tange/parallel
1
0
Fork 0
mirror of https://git.savannah.gnu.org/git/parallel.git synced 2024-12-22 20:57:53 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

testsuite: Fixup of polarhome and added X.Xs.sh.

Browse source
This commit is contained in:
Ole Tange 2013-12-19 02:19:19 +01:00
parent fbfd7d3bf7
commit bfb01e89df
18 changed files with 176 additions and 125 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
NEWS
View file

@ -19,8 +19,7 @@
* --transfer and --basefile support paths relative to the --workdir by
inserting /./ into the path.
* GNU Parallel was used (unfortunately with improper citation) in:
'fastphylo: Fast tools for phylogenetics'
* GNU Parallel was cited in: 'fastphylo: Fast tools for phylogenetics'
http://www.biomedcentral.com/1471-2105/14/334/abstract
* Using GNU parallel

15
doc/release_new_version
View file

@ -244,12 +244,12 @@ New in this release:
Analysis Pipeline for Discovery of Human Genetic Variation
http://www.google.com/patents/US20130311106
* Encrypt and decrypt in parallel with GPG
http://blog.tkassembled.com/412/encrypt-and-decrypt-in-parallel-with-gpg/
* Using GNU Parallel to create a SVM classifier
http://ivanyu.me/blog/2013/12/01/parallel/
* Mon make à moi (French)
http://www.slideshare.net/lindenb/mon-make-moi-tout-sauf-galaxy
* Parallel xz compression
http://anthon.home.xs4all.nl/rants/2013/parallel_xz/
@ -262,9 +262,18 @@ New in this release:
* Gnu Parallel for fun and profit
https://gist.github.com/celoyd/f7eb55ad69c9b33fd8c3
* Parallelization of Exploitation
http://rantsideasstuff.com/parallelization-of-exploitation/
* Running assemblies in parallel
https://khmer-protocols.readthedocs.org/en/v0.8.3/metagenomics/4-assemble.html
* Procesando la contabilidad del PP
http://www.neorazorx.com/2013/07/procesando-la-contabilidad-del-pp.html
* Mon make à moi (French)
http://www.slideshare.net/lindenb/mon-make-moi-tout-sauf-galaxy
* Bug fixes and man page updates.

2
src/parallel.pod
View file

@ -3619,7 +3619,7 @@ B<seq>, B<cat>, B<echo>, and B<sleep> can reproduce most errors. If
your example requires large files, see if you can make them by
something like B<seq 1000000> > B<file> or B<yes | head -n 10000000> >
B<file>. If your example requires remote execution, see if you can
use B<localhost>.
use B<localhost> - maybe using another login.
=item *

19
testsuite/tests-to-run/parallel-local-0.3s.sh Normal file
View file

@ -0,0 +1,19 @@
#!/bin/bash
# Simple jobs that never fails
# Each should be taking 0.3-1s and be possible to run in parallel
# I.e.: No race conditions, no logins
cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -k -j4 -L1
echo '### Test exit val - true';
echo true | parallel;
echo $?
echo '**'
echo '### Test exit val - false';
echo false | parallel;
echo $?
echo '**'
EOF

23
testsuite/tests-to-run/parallel-local-100s.sh Normal file
View file

@ -0,0 +1,23 @@
#!/bin/bash
# Simple jobs that never fails
# Each should be taking >100s and be possible to run in parallel
# I.e.: No race conditions, no logins
cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -k -j4 -L1
echo '### Test if we can deal with output > 4 GB'
## echo | niceload --io 10 parallel -q perl -e '"\$a=\"x\"x1000000;for(0..4300){print \$a}"' | md5sum
echo | parallel -q perl -e '$a="x"x1000000;for(0..4300){print $a}' | md5sum
echo '**'
echo "### Test Force outside the file handle limit, 2009-02-17 Gave fork error"
(echo echo Start; seq 1 20000 | perl -pe 's/^/true /'; echo echo end) | stdout parallel -uj 0 | egrep -v 'processes took|adjusting'
echo '**'
echo '### Test of --retries on unreachable host'
seq 2 | stdout parallel -k --retries 2 -v -S 4.3.2.1,: echo
echo '**'
EOF

7
testsuite/tests-to-run/parallel-local-10s.sh Normal file
View file

@ -0,0 +1,7 @@
#!/bin/bash
# Simple jobs that never fails
# Each should be taking 10-30s and be possible to run in parallel
# I.e.: No race conditions, no logins
cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -k -j4 -L1
EOF

19
testsuite/tests-to-run/parallel-local-30s.sh Normal file
View file

@ -0,0 +1,19 @@
#!/bin/bash
# Simple jobs that never fails
# Each should be taking 30-100s and be possible to run in parallel
# I.e.: No race conditions, no logins
cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -k -j4 -L1
echo '### Test race condition on 8 CPU (my laptop)';
seq 1 5000000 > /tmp/parallel_test;
seq 1 10 | parallel -k "cat /tmp/parallel_test | parallel --pipe --recend '' -k gzip >/dev/null; echo {}"
echo '**'
echo "### Test --tmpdir running full. bug #40733 was caused by this"
stdout parallel -j1 --tmpdir /run/shm cat /dev/zero ::: dummy
echo '**'
EOF

43
testsuite/tests-to-run/parallel-local11.sh → testsuite/tests-to-run/parallel-local-3s.sh Executable file → Normal file
View file

@ -1,48 +1,9 @@
#!/bin/bash
# Simple jobs that never fails
# Each should be taking 100s and be possible to run in parallel
# Each should be taking 3-10s and be possible to run in parallel
# I.e.: No race conditions, no logins
cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -k -j4 -L1
echo '### Test if we can deal with output > 4 GB'
## echo | niceload --io 10 parallel -q perl -e '"\$a=\"x\"x1000000;for(0..4300){print \$a}"' | md5sum
echo | parallel -q perl -e '$a="x"x1000000;for(0..4300){print $a}' | md5sum
echo '**'
echo "### Test --tmpdir running full. bug #40733 was caused by this"
stdout parallel -j1 --tmpdir /run/shm cat /dev/zero ::: dummy
echo '**'
echo "### Test Force outside the file handle limit, 2009-02-17 Gave fork error"
(echo echo Start; seq 1 20000 | perl -pe 's/^/true /'; echo echo end) | stdout parallel -uj 0 | egrep -v 'processes took|adjusting'
echo '**'
echo '### Test of --retries on unreachable host'
seq 2 | stdout parallel -k --retries 2 -v -S 4.3.2.1,: echo
echo '**'
echo '### Test race condition on 8 CPU (my laptop)';
seq 1 5000000 > /tmp/parallel_test;
seq 1 10 | parallel -k "cat /tmp/parallel_test | parallel --pipe --recend '' -k gzip >/dev/null; echo {}"
echo '**'
echo '### Test exit val - true';
echo true | parallel;
echo $?
echo '**'
echo '### Test exit val - false';
echo false | parallel;
echo $?
echo '**'
echo '### Test --halt-on-error 0';
(echo "sleep 1;true"; echo "sleep 2;false";echo "sleep 3;true") | parallel -j10 --halt-on-error 0;
echo $?;
@ -80,6 +41,4 @@ echo '**'
echo '### Test slow arguments generation - https://savannah.gnu.org/bugs/?32834';
seq 1 3 | parallel -j1 "sleep 2; echo {}" | parallel -kj2 echo
EOF

2
testsuite/tests-to-run/parallel-local105.sh
View file

@ -1,6 +1,6 @@
#!/bin/bash
TMP=/tmp/parallel_local105
TMP=/run/shm/parallel_local105
rm -rf $TMP 2>/dev/null
mkdir -p $TMP
tar -C $TMP -xf input-files/random_dirs_with_newline.tar.bz2

8
testsuite/tests-to-run/parallel-local15.sh
View file

@ -1,7 +1,9 @@
#!/bin/bash
rsync -Ha --delete input-files/testdir/ /tmp/parallel_$$
cd /tmp/parallel_$$
TMP=/run/shm/parallel_$$
rsync -Ha --delete input-files/testdir/ $TMP/
cd $TMP/
echo echo test of cat pipe sh | parallel -j 50 2>&1
find . -name '*.jpg' | parallel -j +0 convert -geometry 120 {} {//}/thumb_{/}
@ -25,4 +27,4 @@ find . -type f | parallel -qv --group perl -ne '/^\S+\s+\S+$/ and print $ARGV,"\
EOF
cd -
rm -rf /tmp/parallel_$$
rm -rf $TMP/

2
testsuite/tests-to-run/parallel-local2.sh
View file

@ -27,7 +27,7 @@ echo "bug #38441: CPU usage goes to 100% if load is higher than --load at first
echo '### Test slow arguments generation - https://savannah.gnu.org/bugs/?32834'
seq 1 3 | parallel -j1 "sleep 2; echo {}" | parallel -kj2 echo
echo '### Test too slow spawning - TODO THIS CURRENTLY DOES NOT OVERLOAD'
echo '### Test too slow spawning'
# Let the commands below run during high load
seq `parallel --number-of-cores` | parallel -j200% -N0 timeout -k 25 26 burnP6 &
sleep 1;

17
testsuite/tests-to-run/parallel-polarhome.sh
View file

@ -13,17 +13,14 @@ POLAR=`parallel -k echo {}.polarhome.com ::: $P`
echo '### Tests on polarhome machines'
echo 'Setup on polarhome machines'
stdout parallel -kj0 ssh -oLogLevel=quiet {} mkdir -p bin ::: $POLAR &
# scp to each polarhome machine do not work. Use cat
copy_to_host() {
copy_and_test() {
H=$1
# scp to each polarhome machine do not work. Use cat
# Avoid the stupid /etc/issue.net banner with -oLogLevel=quiet
ssh -oLogLevel=quiet $H rm -f bin/parallel
cat `which parallel` | ssh -oLogLevel=quiet $H 'cat > bin/parallel; chmod 755 bin/parallel'
echo '### Run the test on '$H
cat `which parallel` | ssh -oLogLevel=quiet $H 'cat > bin/p.tmp && chmod 755 bin/p.tmp && mv bin/p.tmp bin/parallel; bin/perl bin/parallel echo Works on ::: '$H
}
export -f copy_to_host
stdout parallel -j0 --timeout 20 --tag -kj0 -v copy_to_host {} ::: $POLAR
# Now test
echo '### Run the test on polarhome machines'
stdout parallel -j0 --argsep // -k --tag ssh -oLogLevel=quiet {} bin/perl bin/parallel -k echo Works on ::: {} // $POLAR
export -f copy_and_test
stdout parallel -j0 -k --timeout 80 --delay 0.1 --tag -v copy_and_test {} ::: $POLAR

6
testsuite/wanted-results/parallel-local-0.3s Normal file
View file

@ -0,0 +1,6 @@
### Test exit val - true
0
**
### Test exit val - false
1
**

17
testsuite/wanted-results/parallel-local-100s Normal file
View file

@ -0,0 +1,17 @@
### Test if we can deal with output > 4 GB
46a318993dfc8e2afd71ff2bc6f605f1 -
**
### Test Force outside the file handle limit, 2009-02-17 Gave fork error
parallel: Warning: Only enough file handles to run 506 jobs in parallel.
Raising ulimit -n or /etc/security/limits.conf may help.
Start
end
**
### Test of --retries on unreachable host
ssh: connect to host 4.3.2.1 port 22: Connection timed out
parallel: Warning: Could not figure out number of cpus on 4.3.2.1 (). Using 1.
echo 1
1
echo 2
2
**

16
testsuite/wanted-results/parallel-local-30s Normal file
View file

@ -0,0 +1,16 @@
### Test race condition on 8 CPU (my laptop)
1
2
3
4
5
6
7
8
9
10
**
### Test --tmpdir running full. bug #40733 was caused by this
parallel: Error: Output is incomplete. Cannot append to buffer file in $TMPDIR. Is the disk full?
parallel: Error: Change $TMPDIR with --tmpdir.
**

39
testsuite/wanted-results/parallel-local11 → testsuite/wanted-results/parallel-local-3s
View file

@ -1,42 +1,3 @@
### Test if we can deal with output > 4 GB
46a318993dfc8e2afd71ff2bc6f605f1 -
**
### Test --tmpdir running full. bug #40733 was caused by this
parallel: Error: Output is incomplete. Cannot append to buffer file in $TMPDIR. Is the disk full?
parallel: Error: Change $TMPDIR with --tmpdir.
**
### Test Force outside the file handle limit, 2009-02-17 Gave fork error
parallel: Warning: Only enough file handles to run 506 jobs in parallel.
Raising ulimit -n or /etc/security/limits.conf may help.
Start
end
**
### Test of --retries on unreachable host
ssh: connect to host 4.3.2.1 port 22: Connection timed out
parallel: Warning: Could not figure out number of cpus on 4.3.2.1 (). Using 1.
echo 1
1
echo 2
2
**
### Test race condition on 8 CPU (my laptop)
1
2
3
4
5
6
7
8
9
10
**
### Test exit val - true
0
**
### Test exit val - false
1
**
### Test --halt-on-error 0
1
2

5
testsuite/wanted-results/parallel-local2
View file

@ -5,6 +5,5 @@ Less than 1 secs user time: OK
1
2
3
### Test too slow spawning - TODO THIS CURRENTLY DOES NOT OVERLOAD
parallel: Warning: Only enough file handles to run 506 jobs in parallel.
Raising ulimit -n or /etc/security/limits.conf may help.
### Test too slow spawning
OK

58
testsuite/wanted-results/parallel-polarhome
View file

@ -1,43 +1,61 @@
### Tests on polarhome machines
Setup on polarhome machines
copy_to_host freebsd.polarhome.com
copy_to_host solaris.polarhome.com
copy_to_host openbsd.polarhome.com
copy_to_host netbsd.polarhome.com
copy_to_host debian.polarhome.com
copy_to_host aix.polarhome.com
copy_to_host redhat.polarhome.com
copy_to_host hpux.polarhome.com
copy_to_host qnx.polarhome.com
copy_to_host openindiana.polarhome.com
copy_to_host suse.polarhome.com
copy_to_host mandriva.polarhome.com
copy_to_host ubuntu.polarhome.com
copy_to_host scosysv.polarhome.com
copy_to_host unixware.polarhome.com
copy_to_host dragonfly.polarhome.com
copy_to_host centos.polarhome.com
copy_to_host miros.polarhome.com
copy_to_host hurd.polarhome.com
### Run the test on polarhome machines
copy_and_test freebsd.polarhome.com
freebsd.polarhome.com ### Run the test on freebsd.polarhome.com
freebsd.polarhome.com Works on freebsd.polarhome.com
copy_and_test solaris.polarhome.com
solaris.polarhome.com ### Run the test on solaris.polarhome.com
solaris.polarhome.com Works on solaris.polarhome.com
copy_and_test openbsd.polarhome.com
openbsd.polarhome.com ### Run the test on openbsd.polarhome.com
openbsd.polarhome.com Works on openbsd.polarhome.com
copy_and_test netbsd.polarhome.com
netbsd.polarhome.com ### Run the test on netbsd.polarhome.com
netbsd.polarhome.com Works on netbsd.polarhome.com
copy_and_test debian.polarhome.com
debian.polarhome.com ### Run the test on debian.polarhome.com
debian.polarhome.com Works on debian.polarhome.com
copy_and_test aix.polarhome.com
aix.polarhome.com ### Run the test on aix.polarhome.com
aix.polarhome.com Works on aix.polarhome.com
copy_and_test redhat.polarhome.com
redhat.polarhome.com ### Run the test on redhat.polarhome.com
redhat.polarhome.com Works on redhat.polarhome.com
copy_and_test hpux.polarhome.com
hpux.polarhome.com ### Run the test on hpux.polarhome.com
hpux.polarhome.com Works on hpux.polarhome.com
copy_and_test qnx.polarhome.com
qnx.polarhome.com ### Run the test on qnx.polarhome.com
qnx.polarhome.com Works on qnx.polarhome.com
qnx.polarhome.com parallel: Warning: Cannot figure out number of CPU cores. Using 1.
copy_and_test openindiana.polarhome.com
openindiana.polarhome.com ### Run the test on openindiana.polarhome.com
openindiana.polarhome.com Works on openindiana.polarhome.com
openindiana.polarhome.com parallel: Warning: Cannot figure out number of CPU cores. Using 1.
copy_and_test suse.polarhome.com
suse.polarhome.com ### Run the test on suse.polarhome.com
suse.polarhome.com Works on suse.polarhome.com
copy_and_test mandriva.polarhome.com
mandriva.polarhome.com ### Run the test on mandriva.polarhome.com
mandriva.polarhome.com Works on mandriva.polarhome.com
copy_and_test ubuntu.polarhome.com
ubuntu.polarhome.com ### Run the test on ubuntu.polarhome.com
ubuntu.polarhome.com Works on ubuntu.polarhome.com
copy_and_test scosysv.polarhome.com
scosysv.polarhome.com ### Run the test on scosysv.polarhome.com
scosysv.polarhome.com Works on scosysv.polarhome.com
copy_and_test unixware.polarhome.com
unixware.polarhome.com ### Run the test on unixware.polarhome.com
unixware.polarhome.com Works on unixware.polarhome.com
copy_and_test dragonfly.polarhome.com
dragonfly.polarhome.com ### Run the test on dragonfly.polarhome.com
dragonfly.polarhome.com Works on dragonfly.polarhome.com
copy_and_test centos.polarhome.com
centos.polarhome.com ### Run the test on centos.polarhome.com
centos.polarhome.com Works on centos.polarhome.com
copy_and_test miros.polarhome.com
miros.polarhome.com ### Run the test on miros.polarhome.com
miros.polarhome.com Works on miros.polarhome.com
copy_and_test hurd.polarhome.com
hurd.polarhome.com ### Run the test on hurd.polarhome.com
hurd.polarhome.com Works on hurd.polarhome.com