Unittest adapted.

doc/release_new_version

@@ -171,48 +171,40 @@ cc:Sandro Cazzaniga <kharec@mandriva.org>,
    Ryoichiro Suzuki <ryoichiro.suzuki@gmail.com>,
    Jesse Alama <jesse.alama@gmail.com>
 
-Subject: GNU Parallel 20121022 ('Hanne Vilmann') released
+Subject: GNU Parallel 20121122 ('Barack') released
 
-GNU Parallel 20121022 ('Hanne Vilmann') has been released. It is
+GNU Parallel 20121122 ('Barack') has been released. It is
 available for download at: http://ftp.gnu.org/gnu/parallel/
 
 New in this release:
 
-* --results structures output files so they can be imported using
+* GNU Parallel was part of the justification for Nordic Free Software
-  Pandas (http://pandas.pydata.org/). Useful when doing simulations to
+  Award 2012
-  select output from some arguments.
+  https://fsfe.org/news/2012/news-20121112-01.html
 
-* Use --env to transfer environment variables through ssh to remote
+* Dateien schneller packen und konvertieren mit GNU parallel
-  machines.
+  http://www.linux-community.de/Archiv/Tipp-der-Woche/Dateien-schneller-packen-und-konvertieren-mit-GNU-parallel
 
-* GNU Parallel is accepted for Ubuntu Quantal.
+* Processing images from Lunar Reconnaissance Orbiter Narrow Angle
-  http://packages.ubuntu.com/quantal/parallel
+  Camera http://lunokhod.org/?p=774
 
-* GNU Parallel was requested for OpenSUSE (go vote!)
+* GNU Parallel is used in types2: Type and Hapax Accumulation Curves
-  https://features.opensuse.org/314461
+  http://www.cs.helsinki.fi/u/josuomel/types2/
 
-* Kneth's Korner: Map/Reduce and GNU Parallel
+* GNU Parallel used in: A Hybrid Factor Analysis and Probabilistic
-  http://kenneth.geisshirt.dk/2012/10/mapreduce-and-gnu-parallel.html
+  PCA-based system for Dictionary Learning and Encoding for Robust
+  Speaker Recognition
+  http://www.superlectures.com/odyssey2012/lecture.php?id=8&lang=en
+  http://www.cse.iitm.ac.in/~mrsri/ppca.pdf
 
-* Blog post in Japanese by Siguniang.
+* Mapreduce avec parallel, cat et une redirection
-  https://siguniang.wordpress.com/2012/09/09/notes-on-gnu-parallel-the-command-line-power-tool/
+  http://datacratic.com/site/blog/mapreduce-avec-parallel-cat-et-une-redirection
-  (Siguniang mentions the logo is called 'The Cafe Wall Illusion')
-  http://www.michaelbach.de/ot/ang_cafewall/index.html
 
-* GNU Parallel, where have you been all my life?
+* GNU Parallel for Large Batches of Small Jobs
-  http://ssdigit.nothingisreal.com/2012/09/gnu-parallel-where-have-you-been-all-my.html
+  https://support.scinet.utoronto.ca/wiki/images/7/7b/Tech-talk-gnu-parallel.pdf
 
-* Reference-free ddRADseq analysis tools uses GNU Parallel.
+* GNU Parallel used in black belt tip (8:50-10:12)
-  https://github.com/brantp/rtd#readme
+  https://aws.amazon.com/apac/awspodcast/episode20/
-
-* When bash just isn’t quick enough.
-  https://soimasysadmin.wordpress.com/2012/08/15/gnu-parallel-when-bash-just-isnt-quick-enough/
-
-* Using GNU Parallel to Package Multiple Jobs in a Single PBS Job.
-  http://www.nas.nasa.gov/hecc/support/kb/Using-GNU-Parallel-to-Package-Multiple-Jobs-in-a-Single-PBS-Job_303.html
-
-* GNU Parallel Notes.
-  http://xed.ch/help/parallel.html
 
 * Bug fixes and man page updates.
 

packager/obs/home:tange/parallel/parallel.spec

@@ -1,6 +1,6 @@
 Summary:	Shell tool for executing jobs in parallel
 Name: 		parallel
-Version: 	20120822
+Version: 	20121122
 Release: 	1
 License: 	GPL
 Group: 		Productivity/File utilities

src/parallel

@@ -606,7 +606,7 @@ sub get_options_from_array {
 sub parse_options {
     # Returns: N/A
     # Defaults:
-    $Global::version = 20121023;
+    $Global::version = 20121122;
     $Global::progname = 'parallel';
     $Global::infinity = 2**31;
     $Global::debug = 0;

testsuite/Start.sh

@@ -1,13 +1,21 @@
 #!/bin/bash
 
+# Argument can be substring of tests (such as 'local')
+
 export LANG=C
 SHFILE=/tmp/unittest-parallel.sh
 
-# Try a failing test twice.
+# Run a failing test once
 ls -t tests-to-run/*${1}*.sh \
-| perl -pe 's:(.*/(.*)).sh:bash $1.sh > actual-results/$2; diff -Naur wanted-results/$2 actual-results/$2 >/dev/null || bash $1.sh > actual-results/$2; diff -Naur wanted-results/$2 actual-results/$2:' \
+| perl -pe 's:(.*/(.*)).sh:bash $1.sh > actual-results/$2; diff -Naur wanted-results/$2 actual-results/$2: ' \
 >$SHFILE
 
+#  # Try a failing test thrice
+#  ls -t tests-to-run/*${1}*.sh \
+#  | perl -pe 's:(.*/(.*)).sh:bash $1.sh > actual-results/$2; diff -Naur wanted-results/$2 actual-results/$2 >/dev/null || bash $1.sh > actual-results/$2; diff -Naur wanted-results/$2 actual-results/$2 >/dev/null || bash $1.sh > actual-results/$2; diff -Naur wanted-results/$2 actual-results/$2: ' \
+#  >$SHFILE
+
+
 mkdir -p actual-results
 stdout sh -x $SHFILE | tee testsuite.log
 rm $SHFILE

testsuite/tests-to-run/niceload01.sh

@@ -8,7 +8,7 @@ echo
 MEMAVAIL=$(free | perl -ane '/buffers.cache:/ and print $F[3]')
 while [ $MEMAVAIL -gt 1000000 ] ; do
   BS=$(echo $MEMAVAIL/10 | bc)
-  (seq 1 10 | parallel -j0 -N0 timeout 10 nice dd if=/dev/zero bs=${BS}k '|' wc -c >/dev/null &)
+  (seq 1 10 | parallel -j0 -N0 timeout 15 nice dd if=/dev/zero bs=${BS}k '|' wc -c >/dev/null &)
   sleep 2
   MEMAVAIL=$(free | perl -ane '/buffers.cache:/ and print $F[3]')
 done

testsuite/tests-to-run/parallel-local114.sh

@@ -49,32 +49,32 @@ echo "bug #34958: --pipe with record size measured in lines";
 
 echo "### Test --results"; 
   mkdir -p /tmp/parallel_results_test; 
-  parallel -k --results /tmp/parallel_results_test/testA echo {1} {2} ::: I II ::: III IIII;
+  parallel -k --results /tmp/parallel_results_test/testA echo {1} {2} ::: I II ::: III IIII; 
   ls /tmp/parallel_results_test/testA*; rm /tmp/parallel_results_test/testA*
 
 echo "### Test --res"; 
   mkdir -p /tmp/parallel_results_test; 
-  parallel -k --results /tmp/parallel_results_test/testD echo {1} {2} ::: I II ::: III IIII;
+  parallel -k --results /tmp/parallel_results_test/testD echo {1} {2} ::: I II ::: III IIII; 
   ls /tmp/parallel_results_test/testD*; rm /tmp/parallel_results_test/testD*
 
 echo "### Test --result"; 
   mkdir -p /tmp/parallel_results_test; 
-  parallel -k --result /tmp/parallel_results_test/testE echo {1} {2} ::: I II ::: III IIII;
+  parallel -k --result /tmp/parallel_results_test/testE echo {1} {2} ::: I II ::: III IIII; 
   ls /tmp/parallel_results_test/testE*; rm /tmp/parallel_results_test/testE*
 
 echo "### Test --results --header :"; 
   mkdir -p /tmp/parallel_results_test; 
-  parallel -k --header : --results /tmp/parallel_results_test/testB echo {1} {2} ::: a I II ::: b III IIII
+  parallel -k --header : --results /tmp/parallel_results_test/testB echo {1} {2} ::: a I II ::: b III IIII; 
   ls /tmp/parallel_results_test/testB*; rm /tmp/parallel_results_test/testB*
 
-echo "### Test --results --header :"; 
+echo "### Test --results --header : named"; 
   mkdir -p /tmp/parallel_results_test; 
-  parallel -k --header : --results /tmp/parallel_results_test/testC echo {a} {b} ::: a I II ::: b III IIII
+  parallel -k --header : --results /tmp/parallel_results_test/testC echo {a} {b} ::: a I II ::: b III IIII; 
   ls /tmp/parallel_results_test/testC*; rm /tmp/parallel_results_test/testC*
 
 echo "### Test --results --header : piped"; 
   mkdir -p /tmp/parallel_results_test; 
-  (echo Col; perl -e 'print "backslash\\tab\tslash/null\0eof\n"') | parallel  --header : --result /tmp/parallel_results_test/testF_ true;
+  (echo Col; perl -e 'print "backslash\\tab\tslash/null\0eof\n"') | parallel  --header : --result /tmp/parallel_results_test/testF_ true; 
   ls /tmp/parallel_results_test/testF*; rm /tmp/parallel_results_test/testF*
  
 EOF

testsuite/tests-to-run/parallel-local16.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+echo '### Test --env all chars except \n,\92,\160 - single and double - no output is good'
+# 92 and 160 are special for csh
+perl -e 'for(1..9,9,11..91,91,93..159,159,161..255) { printf "%c%c %c%d\0",$_,$_,$_,$_ }' | stdout parallel --nice 10 -j4 -k -I // --arg-sep _ -0 V=// V2=V2=// parallel --retries 3 -k -j1 -S :,1/lo,1/tcsh@lo,1/csh@lo --env V,V2 echo \''"{}$V$V2"'\' ::: {#} {#} {#} {#} | uniq -c | grep -v '   4 '|grep -v xauth |grep -v X11

testsuite/tests-to-run/parallel-local2.sh

@@ -3,68 +3,11 @@
 echo '### Test slow arguments generation - https://savannah.gnu.org/bugs/?32834'
 seq 1 3 | parallel -j1 "sleep 2; echo {}" | parallel -kj2 echo
 
-echo '### Test --env  - https://savannah.gnu.org/bugs/?37351'
-export TWOSPACES='  2  spaces  '
-export THREESPACES=" >  My brother's 12\" records  < "
-echo a"$TWOSPACES"b 1
-parallel --env TWOSPACES echo 'a"$TWOSPACES"b' ::: 1
-parallel -S localhost --env TWOSPACES echo 'a"$TWOSPACES"b' ::: 1
-parallel -S csh@localhost --env TWOSPACES echo 'a"$TWOSPACES"b' ::: 1
-parallel -S tcsh@localhost --env TWOSPACES echo 'a"$TWOSPACES"b' ::: 1
-
-echo a"$TWOSPACES"b a"$THREESPACES"b 2
-parallel --env TWOSPACES --env THREESPACES echo 'a"$TWOSPACES"b' 'a"$THREESPACES"b' ::: 2
-parallel -S localhost --env TWOSPACES --env THREESPACES echo 'a"$TWOSPACES"b' 'a"$THREESPACES"b' ::: 2
-parallel -S csh@localhost --env TWOSPACES --env THREESPACES echo 'a"$TWOSPACES"b' 'a"$THREESPACES"b' ::: 2
-parallel -S tcsh@localhost --env TWOSPACES --env THREESPACES echo 'a"$TWOSPACES"b' 'a"$THREESPACES"b' ::: 2
-
-echo a"$TWOSPACES"b a"$THREESPACES"b 3
-parallel --env TWOSPACES,THREESPACES echo 'a"$TWOSPACES"b' 'a"$THREESPACES"b' ::: 3
-parallel -S localhost --env TWOSPACES,THREESPACES echo 'a"$TWOSPACES"b' 'a"$THREESPACES"b' ::: 3
-parallel -S csh@localhost --env TWOSPACES,THREESPACES echo 'a"$TWOSPACES"b' 'a"$THREESPACES"b' ::: 3
-parallel -S tcsh@localhost --env TWOSPACES,THREESPACES echo 'a"$TWOSPACES"b' 'a"$THREESPACES"b' ::: 3
-
-export MIN="  \'\""
-echo a"$MIN"b 4
-parallel --env MIN echo 'a"$MIN"b' ::: 4
-parallel -S localhost --env MIN echo 'a"$MIN"b' ::: 4
-parallel -S csh@localhost --env MIN echo 'a"$MIN"b' ::: 4
-parallel -S tcsh@localhost --env MIN echo 'a"$MIN"b' ::: 4
-
-export SPC="'"'   * ? >o  <i*? ][\!#¤%=( ) | }'
-echo a"$SPC"b 5
-parallel --env SPC echo 'a"$SPC"b' ::: 5
-parallel -S localhost --env SPC echo 'a"$SPC"b' ::: 5
-parallel -S csh@localhost --env SPC echo 'a"$SPC"b' ::: 5
-parallel -S tcsh@localhost --env SPC echo 'a"$SPC"b' ::: 5
-
-echo '### Test --env for \n and \\ - single and double - no output is good'
-perl -e 'for(10,92) { printf "%c%c %c%d\0",$_,$_,$_,$_ }' | stdout parallel --nice 19 -j4 -k -I // --arg-sep _ -0 V=// V2=V2=// parallel -k -j1 -S :,1/lo,1/tcsh@lo,1/csh@lo --env V,V2 echo \''"{}$V$V2"'\' ::: {#} {#} {#} {#} | uniq -c | grep -v '   4 '|grep -v xauth |grep -v X11
-
-echo '### Test --env for \n and \\ - single and double --onall - no output is good'
-perl -e 'for(10,92) { printf "%c%c %c%d\0",$_,$_,$_,$_ }' | stdout parallel --nice 19 -j4 -k -I // --arg-sep _ -0 V=// V2=V2=// parallel -k -j1 -S :,1/lo,1/tcsh@lo,1/csh@lo --onall --env V,V2 echo \''"{}$V$V2"'\' ::: {#} | uniq -c | grep -v '   4 '|grep -v xauth |grep -v X11
-
-echo '### Test --env for \160 - which kills csh - single and double - no output is good'
-perl -e 'for(160) { printf "%c%c %c%d\0",$_,$_,$_,$_ }' | stdout parallel --nice 19 -j4 -k -I // --arg-sep _ -0 V=// V2=V2=// parallel -k -j1 -S :,1/lo,1/tcsh@lo --env V,V2 echo \''"{}$V$V2"'\' ::: {#} {#} {#} | uniq -c | grep -v '   3 '|grep -v xauth |grep -v X11
-
-echo '### Test --env for \160  - which kills csh - single and double --onall - no output is good'
-perl -e 'for(160) { printf "%c%c %c%d\0",$_,$_,$_,$_ }' | stdout parallel --nice 19 -j4 -k -I // --arg-sep _ -0 V=// V2=V2=// parallel -k -j1 -S :,1/lo,1/tcsh@lo --onall --env V,V2 echo \''"{}$V$V2"'\' ::: {#} | uniq -c | grep -v '   3 '|grep -v xauth |grep -v X11
-
 echo '### Test too slow spawning'
 killall -9 burnP6 2>/dev/null
-seq `parallel --number-of-cores` | parallel -j100% -N0 timeout -k 25 26 burnP6 &
+seq `parallel --number-of-cores` | parallel -j200% -N0 timeout -k 25 26 burnP6 &
 sleep 1
 seq 1 1000 |
 stdout nice nice  parallel -s 100 -uj0 true |
 perl -pe '/parallel: Warning: Starting \d+ processes took/ and do {close STDIN; `killall -9 burnP6`; print "OK\n"; exit }'
 killall -9 burnP6 2>/dev/null
-
-echo '### Test --env  - https://savannah.gnu.org/bugs/?37351'
-export TWOSPACES='  2  spaces  '
-export THREESPACES=" >  My brother's 12\" records  < "
-stdout parallel --env TWOSPACES echo 'a"$TWOSPACES"b' ::: 1
-stdout parallel --env TWOSPACES --env THREESPACES echo 'a"$TWOSPACES"b' 'a"$THREESPACES"b' ::: 2
-stdout parallel --env TWOSPACES,THREESPACES echo 'a"$TWOSPACES"b' 'a"$THREESPACES"b' ::: 2a
-stdout parallel -S localhost --env TWOSPACES echo 'a"$TWOSPACES"b' ::: 1
-stdout parallel -S localhost --env TWOSPACES --env THREESPACES echo 'a"$TWOSPACES"b' 'a"$THREESPACES"b' ::: 2
-stdout parallel -S localhost --env TWOSPACES,THREESPACES echo 'a"$TWOSPACES"b' 'a"$THREESPACES"b' ::: 2a

testsuite/tests-to-run/parallel-local8.sh

@@ -1,20 +1,21 @@
 #!/bin/bash
 
 echo "### Test : as delimiter. This can be confusing for uptime ie. --load";
-parallel -k --load 100% -d : echo ::: a:b:c
+parallel -k --load 300% -d : echo ::: a:b:c
 
-export PARALLEL="--load 100%"
+export PARALLEL="--load 300%"
 echo PARALLEL=$PARALLEL
 
 for i in $(seq 2 10); do
   i2=$[i*i]
-  seq $i2 | parallel -j0 --load 100% -kX echo {} |wc
+  seq $i2 | parallel -j0 --load 300% -kX echo {} |wc
   seq 1 ${i2}0000 | nice parallel -kj20 --recend "\n" --spreadstdin gzip -9 | zcat | sort -n | md5sum
 done
 
 echo "### Test if --load blocks. Bug.";
-seq 1 1000 | parallel -kj2 --load 100% --recend "\n" --spreadstdin gzip -9 | zcat | sort -n | md5sum
+seq 1 1000 | parallel -kj2 --load 300% --recend "\n" --spreadstdin gzip -9 | zcat | sort -n | md5sum
-seq 1 1000 | parallel -kj0 --load 100% --recend "\n" --spreadstdin gzip -9 | zcat | sort -n | md5sum
+seq 1 1000 | parallel -kj0 --load 300% --recend "\n" --spreadstdin gzip -9 | zcat | sort -n | md5sum
 
 seq 1 1000000 | parallel -kj0 --recend "\n" --spreadstdin gzip -9 | zcat | sort -n | md5sum
 seq 1 1000000 | nice parallel -kj20 --recend "\n" --spreadstdin gzip -9 | zcat | sort -n | md5sum
+ 

testsuite/tests-to-run/parallel-local9.sh

@@ -12,7 +12,7 @@ echo '### Test --spreadstdin - more procs than args';
   cat /tmp/parallel.ss.*;
 
 echo '### Test --spreadstdin - more args than procs'; 
-  rm /tmp/parallel.ss2.*; 
+  rm -f /tmp/parallel.ss2.*; 
   seq 1 10 | stdout parallel -j 5 --spreadstdin 'cat >/tmp/parallel.ss2.$PARALLEL_SEQ' >/dev/null; 
   cat /tmp/parallel.ss2.*
 

testsuite/tests-to-run/sem01.sh

@@ -6,12 +6,6 @@ parallel -u --semaphore seq 11 20 '|' pv -qL 100
 parallel --semaphore --wait
 echo done
 
-echo '### Test default id = --id `tty` and --semaphorename'
-parallel --id `tty` -u --semaphore seq 1 10 '|' pv -qL 20
-parallel -u --semaphore seq 11 20 '|' pv -qL 100
-parallel --semaphorename `tty` --semaphore --wait
-echo done
-
 echo '### Test semaphore 2 jobs running simultaneously'
 parallel -u -j2 --semaphore 'echo job1a 1; sleep 1; echo job1b 3'
 sleep 0.2
@@ -33,6 +27,7 @@ echo done
 echo '### Test similar example as from man page - run 2 jobs simultaneously'
 echo 'Expect done: 1 2 5 3 4'
 for i in 5 1 2 3 4 ; do
+  sleep 0.2
   echo Scheduling $i
   sem -j2 -u echo starting $i ";" sleep $i ";" echo done $i
 done

testsuite/wanted-results/parallel-local114

@@ -168,7 +168,7 @@ II IIII
 /tmp/parallel_results_test/testBstdout	a	I	b	IIII
 /tmp/parallel_results_test/testBstdout	a	II	b	III
 /tmp/parallel_results_test/testBstdout	a	II	b	IIII
-### Test --results --header :
+### Test --results --header : named
 I III
 I IIII
 II III

testsuite/wanted-results/parallel-local2

@@ -2,60 +2,5 @@
 1
 2
 3
-### Test --env  - https://savannah.gnu.org/bugs/?37351
-a  2  spaces  b 1
-a  2  spaces  b 1
-a  2  spaces  b 1
-a  2  spaces  b 1
-a  2  spaces  b 1
-a  2  spaces  b a >  My brother's 12" records  < b 2
-a  2  spaces  b a >  My brother's 12" records  < b 2
-a  2  spaces  b a >  My brother's 12" records  < b 2
-a  2  spaces  b a >  My brother's 12" records  < b 2
-a  2  spaces  b a >  My brother's 12" records  < b 2
-a  2  spaces  b a >  My brother's 12" records  < b 3
-a  2  spaces  b a >  My brother's 12" records  < b 3
-a  2  spaces  b a >  My brother's 12" records  < b 3
-a  2  spaces  b a >  My brother's 12" records  < b 3
-a  2  spaces  b a >  My brother's 12" records  < b 3
-a  \'"b 4
-a  \'"b 4
-a  \'"b 4
-a  \'"b 4
-a  \'"b 4
-a'   * ? >o  <i*? ][\!#¤%=( ) | }b 5
-a'   * ? >o  <i*? ][\!#¤%=( ) | }b 5
-a'   * ? >o  <i*? ][\!#¤%=( ) | }b 5
-a'   * ? >o  <i*? ][\!#¤%=( ) | }b 5
-a'   * ? >o  <i*? ][\!#¤%=( ) | }b 5
-### Test --env for \n and \\ - single and double - no output is good
-      2 setenv: Too many arguments.
-      1 export: Command not found.
-      1 V: Undefined variable.
-      2 1 10V2= 10
-      3 2\\ \92V2=\\ \92
-      1 2\ \92V2=\ \92
-### Test --env for \n and \\ - single and double --onall - no output is good
-      2 setenv: Too many arguments.
-      1 export: Command not found.
-      1 V: Undefined variable.
-      2 1 10V2= 10
-      3 2\\ \92V2=\\ \92
-      1 2\ \92V2=\ \92
-### Test --env for \160 - which kills csh - single and double - no output is good
-### Test --env for \160  - which kills csh - single and double --onall - no output is good
 ### Test too slow spawning
 OK
-### Test --env  - https://savannah.gnu.org/bugs/?37351
-a  2  spaces  b 1
-a  2  spaces  b a >  My brother's 12" records  < b 2
-a  2  spaces  b a >  My brother's 12" records  < b 2a
-a  2  spaces  b 1
-a  2  spaces  b a >  My brother's 12" records  < b 2
-a  2  spaces  b a >  My brother's 12" records  < b 2a
-### Test --env all chars except \n - single and double - no output is good
-      3 92\\ \92V2=\\ \92
-      1 92\ \92V2=\ \92
-### Test --env all chars except \n - single and double --onall - no output is good
-      3 92\\ \92V2=\\ \92
-      1 92\ \92V2=\ \92

testsuite/wanted-results/parallel-local8

@@ -2,7 +2,7 @@
 a
 b
 c
-PARALLEL=--load 100%
+PARALLEL=--load 300%
       4       4       8
 1c0f34fee7176dc367bead8f96cba6bc  -
       9       9      18

testsuite/wanted-results/sem01

@@ -20,28 +20,6 @@
 19
 20
 done
-### Test default id = --id `tty` and --semaphorename
-1
-2
-3
-4
-5
-6
-7
-8
-9
-10
-11
-12
-13
-14
-15
-16
-17
-18
-19
-20
-done
 ### Test semaphore 2 jobs running simultaneously
 job1a 1
 job2a 2
@@ -61,16 +39,16 @@ done
 ### Test similar example as from man page - run 2 jobs simultaneously
 Expect done: 1 2 5 3 4
 Scheduling 5
-Scheduling 1
 starting 5
-Scheduling 2
+Scheduling 1
 starting 1
+Scheduling 2
 done 1
-Scheduling 3
 starting 2
+Scheduling 3
 done 2
-Scheduling 4
 starting 3
+Scheduling 4
 done 5
 starting 4
 done 3