testsuite: virtualbox added. Other tests made more portable.

src/optional/python/README

@@ -7,13 +7,14 @@ DataFrame object. See `help(gnuparallel.load)` for details.
 Installation:
 
     # python setup.py install
+    # aptitude install python-pandas
 
 Sample usage:
 
 1. Generate some results files by running parallel from the command line:
 
-    # mkdir outdir
+    # mkdir out
-    # parallel --results outdir/ echo {arg1} {arg2} ::: arg1 1 2 ::: arg2 three four
+    # parallel --header : --results out/pfx echo {arg1} {arg2} ::: arg1 1 2 ::: arg2 three four
 
 2. Load the results using the gnuparallel Python package:
 
@@ -23,18 +24,15 @@ Sample usage:
     Type "help", "copyright", "credits" or "license" for more information.
     >>> import gnuparallel
     >>> help(gnuparallel.load)
-    >>> myresults = gnuparallel.load('outdir')
+    >>> my_df = gnuparallel.load('out')
-    >>> myresults
+    >>> my_df
-          1      2  _prefix _stream                              resfile
+      _prefix _stream  arg1   arg2                          resfile
-    0     2   four      NaN  stdout     outdir/stdout\t1\t2\t2\tfour
+    0     pfx  stdout     1  three   out/pfxstdout	arg1	1	arg2	three
-    1     2  three      NaN  stdout     outdir/stdout\t1\t2\t2\tthree
+    1     pfx  stdout     1   four   out/pfxstdout	arg1	1	arg2	four
-    2     2   arg2      NaN  stdout     outdir/stdout\t1\t2\t2\targ2
+    2     pfx  stdout     2  three   out/pfxstdout	arg1	2	arg2	three
-    3     1   four      NaN  stdout     outdir/stdout\t1\t1\t2\tfour
+    3     pfx  stdout     2   four   out/pfxstdout	arg1	2	arg2	four
-    4     1  three      NaN  stdout     outdir/stdout\t1\t1\t2\tthree
+    >>> my_df.tail(1)
-    5     1   arg2      NaN  stdout     outdir/stdout\t1\t1\t2\targ2
+    3     pfx  stdout     2   four   out/pfxstdout	arg1	2	arg2	four
-    6  arg1   four      NaN  stdout     outdir/stdout\t1\targ1\t2\tfour
-    7  arg1  three      NaN  stdout     outdir/stdout\t1\targ1\t2\tthree
-    8  arg1   arg2      NaN  stdout     outdir/stdout\t1\targ1\t2\targ2
 
 See documentation for the pandas project (http://pandas.pydata.org/) for
 instructions on how to access and manipulate the loaded results.

src/parallel

@@ -606,7 +606,7 @@ sub get_options_from_array {
 sub parse_options {
     # Returns: N/A
     # Defaults:
-    $Global::version = 20121015;
+    $Global::version = 20121021;
     $Global::progname = 'parallel';
     $Global::infinity = 2**31;
     $Global::debug = 0;

testsuite/tests-to-run/parallel-local2.sh

@@ -70,8 +70,8 @@ stdout parallel -S localhost --env TWOSPACES --env THREESPACES echo 'a"$TWOSPACE
 stdout parallel -S localhost --env TWOSPACES,THREESPACES echo 'a"$TWOSPACES"b' 'a"$THREESPACES"b' ::: 2a
 
 echo '### Test --env all chars except \n - single and double - no output is good'
-perl -e 'for(1..9,9,11..255) { printf "%c%c %c%d\0",$_,$_,$_,$_ }' | stdout parallel --nice 19 -j4 -k -I // --arg-sep _ -0 V=// V2=V2=// parallel -k -j1 -S :,1/lo,1/tcsh@lo,1/csh@lo --env V,V2 echo \''"{}$V$V2"'\' ::: {#} {#} {#} {#} | uniq -c | grep -v '   4 '|grep -v xauth |grep -v X11
+perl -e 'for(1..9,9,11..159,159,161..255) { printf "%c%c %c%d\0",$_,$_,$_,$_ }' | stdout parallel --nice 19 -j4 -k -I // --arg-sep _ -0 V=// V2=V2=// parallel -k -j1 -S :,1/lo,1/tcsh@lo,1/csh@lo --env V,V2 echo \''"{}$V$V2"'\' ::: {#} {#} {#} {#} | uniq -c | grep -v '   4 '|grep -v xauth |grep -v X11
 
 echo '### Test --env all chars except \n - single and double --onall - no output is good'
-perl -e 'for(1..9,9,11..255) { printf "%c%c %c%d\0",$_,$_,$_,$_ }' | stdout parallel --nice 19 -j4 -k -I // --arg-sep _ -0 V=// V2=V2=// parallel -k -j1 -S :,1/lo,1/tcsh@lo,1/csh@lo --onall --env V,V2 echo \''"{}$V$V2"'\' ::: {#} | uniq -c | grep -v '   4 '|grep -v xauth |grep -v X11
+perl -e 'for(1..9,9,11..159,159,161..255) { printf "%c%c %c%d\0",$_,$_,$_,$_ }' | stdout parallel --nice 19 -j4 -k -I // --arg-sep _ -0 V=// V2=V2=// parallel -k -j1 -S :,1/lo,1/tcsh@lo,1/csh@lo --onall --env V,V2 echo \''"{}$V$V2"'\' ::: {#} | uniq -c | grep -v '   4 '|grep -v xauth |grep -v X11
 

testsuite/tests-to-run/parallel-local20.sh

@@ -35,4 +35,4 @@ echo '### bug #36595: silent loss of input with --pipe and --sshlogin'
 seq 10000 | xargs | parallel --pipe -S 10/localhost cat | wc
 
 echo 'bug #36707: --controlmaster eats jobs'
-seq 2 | parallel --controlmaster --sshlogin localhost echo OK{}
+seq 2 | parallel -k --controlmaster --sshlogin localhost echo OK{}

testsuite/tests-to-run/parallel-virtualbox1.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+echo "### These tests requires VirtualBox running with the following images"
+echo "tange@redhat9"
+echo "tange@centos3"
+echo "tange@centos5"
+
+echo "### bug #37589: Red Hat 9 (Shrike) perl v5.8.0 built for i386-linux-thread-multi error"
+cp `which parallel` /tmp/parallel
+stdout parallel --argsep == --basefile /tmp/parallel --tag --nonall -S redhat9.tange.dk,centos3.tange.dk,centos5.tange.dk /tmp/parallel echo ::: OK_if_no_perl_warnings | sort

testsuite/tests-to-run/sql01.sh

@@ -1,15 +1,19 @@
 #!/bin/bash
 
+MYSQL_ADMIN_DBURL=mysql://tange:tange@
+
 # Setup
-sql mysql://root@ "drop user 'sqlunittest'@'localhost'"
+sql $MYSQL_ADMIN_DBURL "drop user 'sqlunittest'@'localhost'"
-sql mysql://root@ DROP DATABASE sqlunittest;
+sql $MYSQL_ADMIN_DBURL DROP DATABASE sqlunittest;
-sql mysql://root@ CREATE DATABASE sqlunittest;
+sql $MYSQL_ADMIN_DBURL CREATE DATABASE sqlunittest;
-sql mysql://root@ "CREATE USER 'sqlunittest'@'localhost' IDENTIFIED BY 'CB5A1FFFA5A';"
+sql $MYSQL_ADMIN_DBURL "CREATE USER 'sqlunittest'@'localhost' IDENTIFIED BY 'CB5A1FFFA5A';"
-sql mysql://root@ "GRANT ALL PRIVILEGES ON sqlunittest.* TO 'sqlunittest'@'localhost';"
+sql $MYSQL_ADMIN_DBURL "GRANT ALL PRIVILEGES ON sqlunittest.* TO 'sqlunittest'@'localhost';"
+
+MYSQL_TEST_DBURL=mysql://sqlunittest:CB5A1FFFA5A@
 
 echo '### Test of #! -Y with file as input'
 cat >/tmp/shebang <<EOF
-#!/usr/local/bin/sql -Y mysql:///tange
+#!/usr/local/bin/sql -Y $MYSQL_TEST_DBURL
 
 SELECT 'Yes it does' AS 'Testing if -Y works';
 EOF
@@ -18,7 +22,7 @@ chmod 755 /tmp/shebang
 
 echo '### Test of #! --shebang with file as input'
 cat >/tmp/shebang <<EOF
-#!/usr/local/bin/sql --shebang mysql:///tange
+#!/usr/local/bin/sql --shebang $MYSQL_TEST_DBURL
 
 SELECT 'Yes it does' AS 'Testing if --shebang works';
 EOF
@@ -26,7 +30,7 @@ chmod 755 /tmp/shebang
 /tmp/shebang
 
 echo '### Test reading sql on command line'
-sql mysql:///tange "SELECT 'Yes it does' as 'Test reading SQL from command line';"
+sql $MYSQL_TEST_DBURL "SELECT 'Yes it does' as 'Test reading SQL from command line';"
 
 echo '### Test reading sql from file'
 cat >/tmp/unittest.sql <<EOF
@@ -39,12 +43,13 @@ INSERT INTO unittest VALUES (1,'abc');
 INSERT INTO unittest VALUES (3,'def');
 SELECT 'Yes it does' as 'Test reading SQL from file works';
 EOF
-sql mysql:///tange </tmp/unittest.sql
+sql $MYSQL_TEST_DBURL/sqlunittest </tmp/unittest.sql
 
 echo '### Test dburl with username password host port'
 sql mysql://sqlunittest:CB5A1FFFA5A@localhost:3306/sqlunittest </tmp/unittest.sql
 
 echo "### Test .sql/aliases"
+mkdir -p ~/.sql
 echo :sqlunittest mysql://sqlunittest:CB5A1FFFA5A@localhost:3306/sqlunittest >> ~/.sql/aliases
 perl -i -ne '$seen{$_}++ || print' ~/.sql/aliases
 sql :sqlunittest "SELECT 'Yes it does' as 'Test if .sql/aliases works';"
@@ -71,7 +76,7 @@ sql --passthrough -H :sqlunittest 'select * from unittest'
 echo
 
 echo "### Test --html";
-sql --html mysql:///tange 'select * from unittest'
+sql --html $MYSQL_TEST_DBURL/sqlunittest 'select * from unittest'
 echo
 
 echo "### Test --show-processlist|proclist|listproc";

testsuite/tests-to-run/sql03.sh

@@ -1,17 +1,21 @@
 #!/bin/bash
 
+MYSQL_ADMIN_DBURL=mysql://tange:tange@
+
 # Setup
-sql mysql://root@ "drop user 'sqlunittest'@'localhost'"
+sql $MYSQL_ADMIN_DBURL "drop user 'sqlunittest'@'localhost'"
-sql mysql://root@ DROP DATABASE sqlunittest;
+sql $MYSQL_ADMIN_DBURL DROP DATABASE sqlunittest;
-sql mysql://root@ CREATE DATABASE sqlunittest;
+sql $MYSQL_ADMIN_DBURL CREATE DATABASE sqlunittest;
-sql mysql://root@ "CREATE USER 'sqlunittest'@'localhost' IDENTIFIED BY 'CB5A1FFFA5A';"
+sql $MYSQL_ADMIN_DBURL "CREATE USER 'sqlunittest'@'localhost' IDENTIFIED BY 'CB5A1FFFA5A';"
-sql mysql://root@ "GRANT ALL PRIVILEGES ON sqlunittest.* TO 'sqlunittest'@'localhost';"
+sql $MYSQL_ADMIN_DBURL "GRANT ALL PRIVILEGES ON sqlunittest.* TO 'sqlunittest'@'localhost';"
+
+MYSQL_TEST_DBURL=mysql://sqlunittest:CB5A1FFFA5A@
 
 echo '### Test reading sql from url command line'
-echo | sql "mysql:///tange?SELECT 'Yes it works' as 'Test reading SQL from command line';"
+echo | sql "$MYSQL_TEST_DBURL/?SELECT 'Yes it works' as 'Test reading SQL from command line';"
 
 echo '### Test reading sql from url command line %-quoting'
-echo | sql "mysql:///tange?SELECT 'Yes it%20works' as 'Test%20%-quoting%20SQL from command line';"
+echo | sql "$MYSQL_TEST_DBURL/?SELECT 'Yes it%20works' as 'Test%20%-quoting%20SQL from command line';"
 
 echo "### Test .sql/aliases with url on commandline"
 echo :sqlunittest mysql://sqlunittest:CB5A1FFFA5A@localhost:3306/sqlunittest >> ~/.sql/aliases

testsuite/tests-to-run/test35.sh

@@ -11,35 +11,35 @@ echo '### Test --wd newtempdir/newdir/tmp/ with space dirs'
 ssh $SERVER2 rm -rf newtempdir
 stdout parallel -k --wd newtempdir/newdir/tmp/ --basefile 1-col.txt --trc {}.6 -S .. -v echo ">"{}.6 ::: './ ab/c"d/ef g' ' ab/c"d/efg' ./b/bar ./b/foo "./ ab /c' d/ ef\"g" ./2-col.txt './a b/cd / ef/efg'
 # A few rmdir errors are OK as we have multiple files in the same dirs
-find . -name '*.6'
+find . -name '*.6' | sort
 
 echo '### Test --wd /tmp/newtempdir/newdir/tmp/ with space dirs'
 ssh $SERVER2 rm -rf /tmp/newtempdir
 stdout parallel -k --wd /tmp/newtempdir/newdir/tmp/ --basefile 1-col.txt --trc {}.7 -S .. -v echo ">"{}.7 ::: './ ab/c"d/ef g' ' ab/c"d/efg' ./b/bar ./b/foo "./ ab /c' d/ ef\"g" ./2-col.txt './a b/cd / ef/efg'
 # A few rmdir errors are OK as we have multiple files in the same dirs
-find . -name '*.7'
+find . -name '*.7' | sort
 
 echo '### Test --workdir ...'
 parallel -k --workdir ... --basefile 1-col.txt --trc {}.1 -S .. echo ">"{}.1 ::: 2-col.txt
-find . -name '*.1'
+find . -name '*.1' | sort
 
 echo '### Test --wd ...'
 parallel -k --wd ... --basefile 1-col.txt --trc {}.2 -S .. -v echo ">"{}.2 ::: 2-col.txt
-find . -name '*.2'
+find . -name '*.2' | sort
 
 echo '### Test --wd ... with space dirs'
 stdout parallel -k --wd ... --basefile 1-col.txt --trc {}.3 -S .. -v echo ">"{}.3 ::: './ ab/c"d/ef g' ' ab/c"d/efg' ./b/bar ./b/foo "./ ab /c' d/ ef\"g" ./2-col.txt './a b/cd / ef/efg'
 # A few rmdir errors are OK as we have multiple files in the same dirs
-find . -name '*.3'
+find . -name '*.3' | sort
 
 echo '### Test --wd tmpdir'
 parallel -k --wd tmpdir --basefile 1-col.txt --trc {}.4 -S .. -v echo ">"{}.4 ::: 2-col.txt
-find . -name '*.4'
+find . -name '*.4' | sort
 
 echo '### Test --wd /tmp/ with space dirs'
 stdout parallel -k --wd /tmp/ --basefile 1-col.txt --trc {}.5 -S .. -v echo ">"{}.5 ::: './ ab/c"d/ef g' ' ab/c"d/efg' ./b/bar ./b/foo "./ ab /c' d/ ef\"g" ./2-col.txt './a b/cd / ef/efg'
 # A few rmdir errors are OK as we have multiple files in the same dirs
-find . -name '*.5'
+find . -name '*.5' | sort
 
 cd ..
 rm -rf tmp

testsuite/tests-to-run/test45.sh

@@ -59,7 +59,7 @@ echo '### Test --resume --joblog followed by --resume --joblog';
   rm -f /tmp/joblog2;
 
 echo '### Test --header'; 
-  printf "a\tb\n1.2\t3/4.5" | parallel --header "\n" echo {b} {a} {b.} {b/} {b//} {b/.}; 
+  printf "a\tb\n1.2\t3/4.5" | parallel --colsep "\t" --header "\n" echo {b} {a} {b.} {b/} {b//} {b/.}; 
 
 echo '### 64-bit wierdness - this did not complete on a 64-bit machine'; 
   seq 1 2 | parallel -j1 'seq 1 1 | parallel true'

testsuite/wanted-results/parallel-local2

@@ -54,4 +54,8 @@ a  2  spaces  b 1
 a  2  spaces  b a >  My brother's 12" records  < b 2
 a  2  spaces  b a >  My brother's 12" records  < b 2a
 ### Test --env all chars except \n - single and double - no output is good
+      3 92\\ \92V2=\\ \92
+      1 92\ \92V2=\ \92
 ### Test --env all chars except \n - single and double --onall - no output is good
+      3 92\\ \92V2=\\ \92
+      1 92\ \92V2=\ \92

testsuite/wanted-results/parallel-virtualbox1

@@ -0,0 +1,8 @@
+### These tests requires VirtualBox running with the following images
+tange@redhat9
+tange@centos3
+tange@centos5
+### bug #37589: Red Hat 9 (Shrike) perl v5.8.0 built for i386-linux-thread-multi error
+centos3.tange.dk	OK_if_no_perl_warnings
+centos5.tange.dk	OK_if_no_perl_warnings
+redhat9.tange.dk	OK_if_no_perl_warnings

testsuite/wanted-results/sql03

@@ -51,13 +51,13 @@ Error:
 sql [-hnr] [--table-size] [--db-size] [-p pass-through] [-s string] dburl [command]
 ### Test oracle with multiple arguments on the command line
 
-'ARG2'
+'ARG
-------------
+----
 arg2
 
 
-'ARG3'
+'ARG
-------------
+----
 arg3
 
 ### Test oracle with \n arguments on the command line

testsuite/wanted-results/test18

@@ -78,7 +78,7 @@ parallel: Warning: --cleanup ignored as there are no remote --sshlogin.
 9
 10
 ### Check forced number of CPUs being respected
-alpha
+hk
 vm1
 vm1
 vm1
@@ -88,16 +88,16 @@ vm1
 vm1
 vm1
 vm1
-alpha
+hk
-alpha
+hk
-alpha
+hk
-alpha
+hk
-alpha
+hk
-alpha
+hk
-alpha
+hk
-alpha
+hk
-alpha
+hk
-alpha
+hk
 ### Check more than 9 simultaneous sshlogins
 1
 2

testsuite/wanted-results/test30

@@ -4,14 +4,14 @@
 
 Computers / CPU cores / Max jobs to run
 1:local / 2 / 1
-
+
0
 ### Test of --progress
 16
 ### Test of --progress with no jobs
 
 Computers / CPU cores / Max jobs to run
 1:local / 2 / 1
-
+
0
 ### bug #34422: parallel -X --eta crashes with div by zero
 1 2
 
@@ -19,7 +19,7 @@ Computers / CPU cores / Max jobs to run
 1:local / 2 / 1
 
 Computer:jobs running/jobs completed/%of started jobs/Average seconds to complete
-
local:1/0/100%/0.0s 
+
local:1/0/100%/0.0s 
ETA: 0s 0left 0.00avg  local:0/1/100%/1.0s 
 ### --timeout --onall on remote machines: 2*slept 1, 2 jobs failed
 slept 1
 slept 1

testsuite/wanted-results/test32

@@ -1,9 +1,9 @@
 ### Test of --retries - it should run 13 jobs in total
-12
+13
 ### Test of --retries - it should run 25 jobs in total
-24
+25
 ### Test of --retries - it should run 49 jobs in total
-48
+49
 ### Bug with --retries
 8
 8

testsuite/wanted-results/test35

@@ -6,13 +6,13 @@ echo >./b/foo.6
 echo >./\ ab\ /c\'\ d/\ ef\"g.6
 echo >./2-col.txt.6
 echo >./a\ b/cd\ /\ ef/efg.6
-./ ab/c"d/ef g.6
-./ ab/c"d/efg.6
-./b/foo.6
-./b/bar.6
 ./ ab /c' d/ ef"g.6
-./a b/cd / ef/efg.6
+./ ab/c"d/ef g.6
+./ ab/c"d/efg.6
 ./2-col.txt.6
+./a b/cd / ef/efg.6
+./b/bar.6
+./b/foo.6
 ### Test --wd /tmp/newtempdir/newdir/tmp/ with space dirs
 echo >./\ ab/c\"d/ef\ g.7
 echo >\ ab/c\"d/efg.7
@@ -21,13 +21,13 @@ echo >./b/foo.7
 echo >./\ ab\ /c\'\ d/\ ef\"g.7
 echo >./2-col.txt.7
 echo >./a\ b/cd\ /\ ef/efg.7
+./ ab /c' d/ ef"g.7
 ./ ab/c"d/ef g.7
 ./ ab/c"d/efg.7
 ./2-col.txt.7
+./a b/cd / ef/efg.7
 ./b/bar.7
 ./b/foo.7
-./ ab /c' d/ ef"g.7
-./a b/cd / ef/efg.7
 ### Test --workdir ...
 ./2-col.txt.1
 ### Test --wd ...
@@ -41,13 +41,13 @@ echo >./b/foo.3
 echo >./\ ab\ /c\'\ d/\ ef\"g.3
 echo >./2-col.txt.3
 echo >./a\ b/cd\ /\ ef/efg.3
-./ ab/c"d/ef g.3
-./ ab/c"d/efg.3
-./b/bar.3
-./b/foo.3
 ./ ab /c' d/ ef"g.3
+./ ab/c"d/ef g.3
+./ ab/c"d/efg.3
 ./2-col.txt.3
 ./a b/cd / ef/efg.3
+./b/bar.3
+./b/foo.3
 ### Test --wd tmpdir
 echo >2-col.txt.4
 ./2-col.txt.4
@@ -59,10 +59,10 @@ echo >./b/foo.5
 echo >./\ ab\ /c\'\ d/\ ef\"g.5
 echo >./2-col.txt.5
 echo >./a\ b/cd\ /\ ef/efg.5
-./2-col.txt.5
-./ ab/c"d/ef g.5
-./ ab/c"d/efg.5
-./b/foo.5
-./b/bar.5
 ./ ab /c' d/ ef"g.5
+./ ab/c"d/ef g.5
+./ ab/c"d/efg.5
+./2-col.txt.5
 ./a b/cd / ef/efg.5
+./b/bar.5
+./b/foo.5