tange/parallel
1
0
Fork 0
mirror of https://git.savannah.gnu.org/git/parallel.git synced 2024-11-22 14:07:55 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

testsuite: virtualbox added. Other tests made more portable.

Browse source
This commit is contained in:
Ole Tange 2012-10-22 08:32:39 +02:00
parent c8e032b279
commit 75f54b795f
21 changed files with 123 additions and 94 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

26
src/optional/python/README
View file

@ -7,13 +7,14 @@ DataFrame object. See `help(gnuparallel.load)` for details.
Installation:
# python setup.py install
# aptitude install python-pandas
Sample usage:
1. Generate some results files by running parallel from the command line:
# mkdir outdir
# parallel --results outdir/ echo {arg1} {arg2} ::: arg1 1 2 ::: arg2 three four
# mkdir out
# parallel --header : --results out/pfx echo {arg1} {arg2} ::: arg1 1 2 ::: arg2 three four
2. Load the results using the gnuparallel Python package:
@ -23,18 +24,15 @@ Sample usage:
Type "help", "copyright", "credits" or "license" for more information.
>>> import gnuparallel
>>> help(gnuparallel.load)
>>> myresults = gnuparallel.load('outdir')
>>> myresults
1 2 _prefix _stream resfile
0 2 four NaN stdout outdir/stdout\t1\t2\t2\tfour
1 2 three NaN stdout outdir/stdout\t1\t2\t2\tthree
2 2 arg2 NaN stdout outdir/stdout\t1\t2\t2\targ2
3 1 four NaN stdout outdir/stdout\t1\t1\t2\tfour
4 1 three NaN stdout outdir/stdout\t1\t1\t2\tthree
5 1 arg2 NaN stdout outdir/stdout\t1\t1\t2\targ2
6 arg1 four NaN stdout outdir/stdout\t1\targ1\t2\tfour
7 arg1 three NaN stdout outdir/stdout\t1\targ1\t2\tthree
8 arg1 arg2 NaN stdout outdir/stdout\t1\targ1\t2\targ2
>>> my_df = gnuparallel.load('out')
>>> my_df
_prefix _stream arg1 arg2 resfile
0 pfx stdout 1 three out/pfxstdout arg1 1 arg2 three
1 pfx stdout 1 four out/pfxstdout arg1 1 arg2 four
2 pfx stdout 2 three out/pfxstdout arg1 2 arg2 three
3 pfx stdout 2 four out/pfxstdout arg1 2 arg2 four
>>> my_df.tail(1)
3 pfx stdout 2 four out/pfxstdout arg1 2 arg2 four
See documentation for the pandas project (http://pandas.pydata.org/) for
instructions on how to access and manipulate the loaded results.

2
src/parallel
View file

@ -606,7 +606,7 @@ sub get_options_from_array {
sub parse_options {
# Returns: N/A
# Defaults:
$Global::version = 20121015;
$Global::version = 20121021;
$Global::progname = 'parallel';
$Global::infinity = 2**31;
$Global::debug = 0;

4
testsuite/tests-to-run/parallel-local2.sh
View file

@ -70,8 +70,8 @@ stdout parallel -S localhost --env TWOSPACES --env THREESPACES echo 'a"$TWOSPACE
stdout parallel -S localhost --env TWOSPACES,THREESPACES echo 'a"$TWOSPACES"b' 'a"$THREESPACES"b' ::: 2a
echo '### Test --env all chars except \n - single and double - no output is good'
perl -e 'for(1..9,9,11..255) { printf "%c%c %c%d\0",$_,$_,$_,$_ }' | stdout parallel --nice 19 -j4 -k -I // --arg-sep _ -0 V=// V2=V2=// parallel -k -j1 -S :,1/lo,1/tcsh@lo,1/csh@lo --env V,V2 echo \''"{}$V$V2"'\' ::: {#} {#} {#} {#} | uniq -c | grep -v ' 4 '|grep -v xauth |grep -v X11
perl -e 'for(1..9,9,11..159,159,161..255) { printf "%c%c %c%d\0",$_,$_,$_,$_ }' | stdout parallel --nice 19 -j4 -k -I // --arg-sep _ -0 V=// V2=V2=// parallel -k -j1 -S :,1/lo,1/tcsh@lo,1/csh@lo --env V,V2 echo \''"{}$V$V2"'\' ::: {#} {#} {#} {#} | uniq -c | grep -v ' 4 '|grep -v xauth |grep -v X11
echo '### Test --env all chars except \n - single and double --onall - no output is good'
perl -e 'for(1..9,9,11..255) { printf "%c%c %c%d\0",$_,$_,$_,$_ }' | stdout parallel --nice 19 -j4 -k -I // --arg-sep _ -0 V=// V2=V2=// parallel -k -j1 -S :,1/lo,1/tcsh@lo,1/csh@lo --onall --env V,V2 echo \''"{}$V$V2"'\' ::: {#} | uniq -c | grep -v ' 4 '|grep -v xauth |grep -v X11
perl -e 'for(1..9,9,11..159,159,161..255) { printf "%c%c %c%d\0",$_,$_,$_,$_ }' | stdout parallel --nice 19 -j4 -k -I // --arg-sep _ -0 V=// V2=V2=// parallel -k -j1 -S :,1/lo,1/tcsh@lo,1/csh@lo --onall --env V,V2 echo \''"{}$V$V2"'\' ::: {#} | uniq -c | grep -v ' 4 '|grep -v xauth |grep -v X11

2
testsuite/tests-to-run/parallel-local20.sh
View file

@ -35,4 +35,4 @@ echo '### bug #36595: silent loss of input with --pipe and --sshlogin'
seq 10000 | xargs | parallel --pipe -S 10/localhost cat | wc
echo 'bug #36707: --controlmaster eats jobs'
seq 2 | parallel --controlmaster --sshlogin localhost echo OK{}
seq 2 | parallel -k --controlmaster --sshlogin localhost echo OK{}

10
testsuite/tests-to-run/parallel-virtualbox1.sh Normal file
View file

@ -0,0 +1,10 @@
#!/bin/bash
echo "### These tests requires VirtualBox running with the following images"
echo "tange@redhat9"
echo "tange@centos3"
echo "tange@centos5"
echo "### bug #37589: Red Hat 9 (Shrike) perl v5.8.0 built for i386-linux-thread-multi error"
cp `which parallel` /tmp/parallel
stdout parallel --argsep == --basefile /tmp/parallel --tag --nonall -S redhat9.tange.dk,centos3.tange.dk,centos5.tange.dk /tmp/parallel echo ::: OK_if_no_perl_warnings | sort

25
testsuite/tests-to-run/sql01.sh
View file

@ -1,15 +1,19 @@
#!/bin/bash
MYSQL_ADMIN_DBURL=mysql://tange:tange@
# Setup
sql mysql://root@ "drop user 'sqlunittest'@'localhost'"
sql mysql://root@ DROP DATABASE sqlunittest;
sql mysql://root@ CREATE DATABASE sqlunittest;
sql mysql://root@ "CREATE USER 'sqlunittest'@'localhost' IDENTIFIED BY 'CB5A1FFFA5A';"
sql mysql://root@ "GRANT ALL PRIVILEGES ON sqlunittest.* TO 'sqlunittest'@'localhost';"
sql $MYSQL_ADMIN_DBURL "drop user 'sqlunittest'@'localhost'"
sql $MYSQL_ADMIN_DBURL DROP DATABASE sqlunittest;
sql $MYSQL_ADMIN_DBURL CREATE DATABASE sqlunittest;
sql $MYSQL_ADMIN_DBURL "CREATE USER 'sqlunittest'@'localhost' IDENTIFIED BY 'CB5A1FFFA5A';"
sql $MYSQL_ADMIN_DBURL "GRANT ALL PRIVILEGES ON sqlunittest.* TO 'sqlunittest'@'localhost';"
MYSQL_TEST_DBURL=mysql://sqlunittest:CB5A1FFFA5A@
echo '### Test of #! -Y with file as input'
cat >/tmp/shebang <<EOF
#!/usr/local/bin/sql -Y mysql:///tange
#!/usr/local/bin/sql -Y $MYSQL_TEST_DBURL
SELECT 'Yes it does' AS 'Testing if -Y works';
EOF
@ -18,7 +22,7 @@ chmod 755 /tmp/shebang
echo '### Test of #! --shebang with file as input'
cat >/tmp/shebang <<EOF
#!/usr/local/bin/sql --shebang mysql:///tange
#!/usr/local/bin/sql --shebang $MYSQL_TEST_DBURL
SELECT 'Yes it does' AS 'Testing if --shebang works';
EOF
@ -26,7 +30,7 @@ chmod 755 /tmp/shebang
/tmp/shebang
echo '### Test reading sql on command line'
sql mysql:///tange "SELECT 'Yes it does' as 'Test reading SQL from command line';"
sql $MYSQL_TEST_DBURL "SELECT 'Yes it does' as 'Test reading SQL from command line';"
echo '### Test reading sql from file'
cat >/tmp/unittest.sql <<EOF
@ -39,12 +43,13 @@ INSERT INTO unittest VALUES (1,'abc');
INSERT INTO unittest VALUES (3,'def');
SELECT 'Yes it does' as 'Test reading SQL from file works';
EOF
sql mysql:///tange </tmp/unittest.sql
sql $MYSQL_TEST_DBURL/sqlunittest </tmp/unittest.sql
echo '### Test dburl with username password host port'
sql mysql://sqlunittest:CB5A1FFFA5A@localhost:3306/sqlunittest </tmp/unittest.sql
echo "### Test .sql/aliases"
mkdir -p ~/.sql
echo :sqlunittest mysql://sqlunittest:CB5A1FFFA5A@localhost:3306/sqlunittest >> ~/.sql/aliases
perl -i -ne '$seen{$_}++ || print' ~/.sql/aliases
sql :sqlunittest "SELECT 'Yes it does' as 'Test if .sql/aliases works';"
@ -71,7 +76,7 @@ sql --passthrough -H :sqlunittest 'select * from unittest'
echo
echo "### Test --html";
sql --html mysql:///tange 'select * from unittest'
sql --html $MYSQL_TEST_DBURL/sqlunittest 'select * from unittest'
echo
echo "### Test --show-processlist|proclist|listproc";

18
testsuite/tests-to-run/sql03.sh
View file

@ -1,17 +1,21 @@
#!/bin/bash
MYSQL_ADMIN_DBURL=mysql://tange:tange@
# Setup
sql mysql://root@ "drop user 'sqlunittest'@'localhost'"
sql mysql://root@ DROP DATABASE sqlunittest;
sql mysql://root@ CREATE DATABASE sqlunittest;
sql mysql://root@ "CREATE USER 'sqlunittest'@'localhost' IDENTIFIED BY 'CB5A1FFFA5A';"
sql mysql://root@ "GRANT ALL PRIVILEGES ON sqlunittest.* TO 'sqlunittest'@'localhost';"
sql $MYSQL_ADMIN_DBURL "drop user 'sqlunittest'@'localhost'"
sql $MYSQL_ADMIN_DBURL DROP DATABASE sqlunittest;
sql $MYSQL_ADMIN_DBURL CREATE DATABASE sqlunittest;
sql $MYSQL_ADMIN_DBURL "CREATE USER 'sqlunittest'@'localhost' IDENTIFIED BY 'CB5A1FFFA5A';"
sql $MYSQL_ADMIN_DBURL "GRANT ALL PRIVILEGES ON sqlunittest.* TO 'sqlunittest'@'localhost';"
MYSQL_TEST_DBURL=mysql://sqlunittest:CB5A1FFFA5A@
echo '### Test reading sql from url command line'
echo | sql "mysql:///tange?SELECT 'Yes it works' as 'Test reading SQL from command line';"
echo | sql "$MYSQL_TEST_DBURL/?SELECT 'Yes it works' as 'Test reading SQL from command line';"
echo '### Test reading sql from url command line %-quoting'
echo | sql "mysql:///tange?SELECT 'Yes it%20works' as 'Test%20%-quoting%20SQL from command line';"
echo | sql "$MYSQL_TEST_DBURL/?SELECT 'Yes it%20works' as 'Test%20%-quoting%20SQL from command line';"
echo "### Test .sql/aliases with url on commandline"
echo :sqlunittest mysql://sqlunittest:CB5A1FFFA5A@localhost:3306/sqlunittest >> ~/.sql/aliases

14
testsuite/tests-to-run/test35.sh
View file

@ -11,35 +11,35 @@ echo '### Test --wd newtempdir/newdir/tmp/ with space dirs'
ssh $SERVER2 rm -rf newtempdir
stdout parallel -k --wd newtempdir/newdir/tmp/ --basefile 1-col.txt --trc {}.6 -S .. -v echo ">"{}.6 ::: './ ab/c"d/ef g' ' ab/c"d/efg' ./b/bar ./b/foo "./ ab /c' d/ ef\"g" ./2-col.txt './a b/cd / ef/efg'
# A few rmdir errors are OK as we have multiple files in the same dirs
find . -name '*.6'
find . -name '*.6' | sort
echo '### Test --wd /tmp/newtempdir/newdir/tmp/ with space dirs'
ssh $SERVER2 rm -rf /tmp/newtempdir
stdout parallel -k --wd /tmp/newtempdir/newdir/tmp/ --basefile 1-col.txt --trc {}.7 -S .. -v echo ">"{}.7 ::: './ ab/c"d/ef g' ' ab/c"d/efg' ./b/bar ./b/foo "./ ab /c' d/ ef\"g" ./2-col.txt './a b/cd / ef/efg'
# A few rmdir errors are OK as we have multiple files in the same dirs
find . -name '*.7'
find . -name '*.7' | sort
echo '### Test --workdir ...'
parallel -k --workdir ... --basefile 1-col.txt --trc {}.1 -S .. echo ">"{}.1 ::: 2-col.txt
find . -name '*.1'
find . -name '*.1' | sort
echo '### Test --wd ...'
parallel -k --wd ... --basefile 1-col.txt --trc {}.2 -S .. -v echo ">"{}.2 ::: 2-col.txt
find . -name '*.2'
find . -name '*.2' | sort
echo '### Test --wd ... with space dirs'
stdout parallel -k --wd ... --basefile 1-col.txt --trc {}.3 -S .. -v echo ">"{}.3 ::: './ ab/c"d/ef g' ' ab/c"d/efg' ./b/bar ./b/foo "./ ab /c' d/ ef\"g" ./2-col.txt './a b/cd / ef/efg'
# A few rmdir errors are OK as we have multiple files in the same dirs
find . -name '*.3'
find . -name '*.3' | sort
echo '### Test --wd tmpdir'
parallel -k --wd tmpdir --basefile 1-col.txt --trc {}.4 -S .. -v echo ">"{}.4 ::: 2-col.txt
find . -name '*.4'
find . -name '*.4' | sort
echo '### Test --wd /tmp/ with space dirs'
stdout parallel -k --wd /tmp/ --basefile 1-col.txt --trc {}.5 -S .. -v echo ">"{}.5 ::: './ ab/c"d/ef g' ' ab/c"d/efg' ./b/bar ./b/foo "./ ab /c' d/ ef\"g" ./2-col.txt './a b/cd / ef/efg'
# A few rmdir errors are OK as we have multiple files in the same dirs
find . -name '*.5'
find . -name '*.5' | sort
cd ..
rm -rf tmp

2
testsuite/tests-to-run/test45.sh
View file

@ -59,7 +59,7 @@ echo '### Test --resume --joblog followed by --resume --joblog';
rm -f /tmp/joblog2;
echo '### Test --header';
printf "a\tb\n1.2\t3/4.5" | parallel --header "\n" echo {b} {a} {b.} {b/} {b//} {b/.};
printf "a\tb\n1.2\t3/4.5" | parallel --colsep "\t" --header "\n" echo {b} {a} {b.} {b/} {b//} {b/.};
echo '### 64-bit wierdness - this did not complete on a 64-bit machine';
seq 1 2 | parallel -j1 'seq 1 1 | parallel true'

4
testsuite/wanted-results/parallel-local2
View file

@ -54,4 +54,8 @@ a 2 spaces b 1
a 2 spaces b a > My brother's 12" records < b 2
a 2 spaces b a > My brother's 12" records < b 2a
### Test --env all chars except \n - single and double - no output is good
3 92\\ \92V2=\\ \92
1 92\ \92V2=\ \92
### Test --env all chars except \n - single and double --onall - no output is good
3 92\\ \92V2=\\ \92
1 92\ \92V2=\ \92

8
testsuite/wanted-results/parallel-virtualbox1 Normal file
View file

@ -0,0 +1,8 @@
### These tests requires VirtualBox running with the following images
tange@redhat9
tange@centos3
tange@centos5
### bug #37589: Red Hat 9 (Shrike) perl v5.8.0 built for i386-linux-thread-multi error
centos3.tange.dk OK_if_no_perl_warnings
centos5.tange.dk OK_if_no_perl_warnings
redhat9.tange.dk OK_if_no_perl_warnings

8
testsuite/wanted-results/sql03
View file

@ -51,13 +51,13 @@ Error:
sql [-hnr] [--table-size] [--db-size] [-p pass-through] [-s string] dburl [command]
### Test oracle with multiple arguments on the command line
'ARG2'
------------
'ARG
----
arg2
'ARG3'
------------
'ARG
----
arg3
### Test oracle with \n arguments on the command line

22
testsuite/wanted-results/test18
View file

@ -78,7 +78,7 @@ parallel: Warning: --cleanup ignored as there are no remote --sshlogin.
9
10
### Check forced number of CPUs being respected
alpha
hk
vm1
vm1
vm1
@ -88,16 +88,16 @@ vm1
vm1
vm1
vm1
alpha
alpha
alpha
alpha
alpha
alpha
alpha
alpha
alpha
alpha
hk
hk
hk
hk
hk
hk
hk
hk
hk
hk
### Check more than 9 simultaneous sshlogins
1
2

6
testsuite/wanted-results/test30
View file

@ -4,14 +4,14 @@
Computers / CPU cores / Max jobs to run
1:local / 2 / 1
0
### Test of --progress
16
### Test of --progress with no jobs
Computers / CPU cores / Max jobs to run
1:local / 2 / 1
0
### bug #34422: parallel -X --eta crashes with div by zero
1 2
@ -19,7 +19,7 @@ Computers / CPU cores / Max jobs to run
1:local / 2 / 1
Computer:jobs running/jobs completed/%of started jobs/Average seconds to complete
local:1/0/100%/0.0s
local:1/0/100%/0.0s ETA: 0s 0left 0.00avg local:0/1/100%/1.0s
### --timeout --onall on remote machines: 2*slept 1, 2 jobs failed
slept 1
slept 1

6
testsuite/wanted-results/test32
View file

@ -1,9 +1,9 @@
### Test of --retries - it should run 13 jobs in total
12
13
### Test of --retries - it should run 25 jobs in total
24
25
### Test of --retries - it should run 49 jobs in total
48
49
### Bug with --retries
8
8

32
testsuite/wanted-results/test35
View file

@ -6,13 +6,13 @@ echo >./b/foo.6
echo >./\ ab\ /c\'\ d/\ ef\"g.6
echo >./2-col.txt.6
echo >./a\ b/cd\ /\ ef/efg.6
./ ab/c"d/ef g.6
./ ab/c"d/efg.6
./b/foo.6
./b/bar.6
./ ab /c' d/ ef"g.6
./a b/cd / ef/efg.6
./ ab/c"d/ef g.6
./ ab/c"d/efg.6
./2-col.txt.6
./a b/cd / ef/efg.6
./b/bar.6
./b/foo.6
### Test --wd /tmp/newtempdir/newdir/tmp/ with space dirs
echo >./\ ab/c\"d/ef\ g.7
echo >\ ab/c\"d/efg.7
@ -21,13 +21,13 @@ echo >./b/foo.7
echo >./\ ab\ /c\'\ d/\ ef\"g.7
echo >./2-col.txt.7
echo >./a\ b/cd\ /\ ef/efg.7
./ ab /c' d/ ef"g.7
./ ab/c"d/ef g.7
./ ab/c"d/efg.7
./2-col.txt.7
./a b/cd / ef/efg.7
./b/bar.7
./b/foo.7
./ ab /c' d/ ef"g.7
./a b/cd / ef/efg.7
### Test --workdir ...
./2-col.txt.1
### Test --wd ...
@ -41,13 +41,13 @@ echo >./b/foo.3
echo >./\ ab\ /c\'\ d/\ ef\"g.3
echo >./2-col.txt.3
echo >./a\ b/cd\ /\ ef/efg.3
./ ab/c"d/ef g.3
./ ab/c"d/efg.3
./b/bar.3
./b/foo.3
./ ab /c' d/ ef"g.3
./ ab/c"d/ef g.3
./ ab/c"d/efg.3
./2-col.txt.3
./a b/cd / ef/efg.3
./b/bar.3
./b/foo.3
### Test --wd tmpdir
echo >2-col.txt.4
./2-col.txt.4
@ -59,10 +59,10 @@ echo >./b/foo.5
echo >./\ ab\ /c\'\ d/\ ef\"g.5
echo >./2-col.txt.5
echo >./a\ b/cd\ /\ ef/efg.5
./2-col.txt.5
./ ab/c"d/ef g.5
./ ab/c"d/efg.5
./b/foo.5
./b/bar.5
./ ab /c' d/ ef"g.5
./ ab/c"d/ef g.5
./ ab/c"d/efg.5
./2-col.txt.5
./a b/cd / ef/efg.5
./b/bar.5
./b/foo.5