testsuite: niceload impoved. Fewer --env tests.

NEWS

@@ -1,3 +1,44 @@
+20121022
+
+* --results structures output files so they can be imported using
+  Pandas (http://pandas.pydata.org/). Useful when doing simulations to
+  select output from some arguments.
+
+* Use --env to transfer environment variables through ssh to remote
+  machines.
+
+* GNU Parallel is accepted for Ubuntu Quantal.
+  http://packages.ubuntu.com/quantal/parallel
+
+* GNU Parallel was requested for OpenSUSE (go vote!)
+  https://features.opensuse.org/314461
+
+* Kneth's Korner: Map/Reduce and GNU Parallel
+  http://kenneth.geisshirt.dk/2012/10/mapreduce-and-gnu-parallel.html
+
+* Blog post in Japanese by Siguniang.
+  https://siguniang.wordpress.com/2012/09/09/notes-on-gnu-parallel-the-command-line-power-tool/
+  (Siguniang mentions the logo is called 'The Cafe Wall Illusion')
+  http://www.michaelbach.de/ot/ang_cafewall/index.html
+
+* GNU Parallel, where have you been all my life?
+  http://ssdigit.nothingisreal.com/2012/09/gnu-parallel-where-have-you-been-all-my.html
+
+* Reference-free ddRADseq analysis tools uses GNU Parallel.
+  https://github.com/brantp/rtd#readme
+
+* When bash just isn’t quick enough.
+  https://soimasysadmin.wordpress.com/2012/08/15/gnu-parallel-when-bash-just-isnt-quick-enough/
+
+* Using GNU Parallel to Package Multiple Jobs in a Single PBS Job.
+  http://www.nas.nasa.gov/hecc/support/kb/Using-GNU-Parallel-to-Package-Multiple-Jobs-in-a-Single-PBS-Job_303.html
+
+* GNU Parallel Notes.
+  http://xed.ch/help/parallel.html
+
+* Bug fixes and man page updates.
+
+
 20120822
 
 * Due to a bugfix the main flow has changed. Making this release beta

doc/release_new_version

@@ -178,7 +178,8 @@ available for download at: http://ftp.gnu.org/gnu/parallel/
 
 New in this release:
 
-* When using GNU Parallel for simulations --results makes it easy to
+* --results structures output files so they can be imported using
+  Pandas (http://pandas.pydata.org/). Useful when doing simulations to
   select output from some arguments.
 
 * Use --env to transfer environment variables through ssh to remote

src/parallel.pod

@@ -445,7 +445,7 @@ If I<eof-str> is omitted, there is no end of file string.  If neither
 B<-E> nor B<-e> is used, no end of file string is used.
 
 
-=item B<--env> I<var>
+=item B<--env> I<var> (alpha testing)
 
 Copy environment variable I<var>. This will copy I<var> to the
 environment that the command is run in. This is especially useful for
@@ -472,7 +472,7 @@ See also: B<--bg>, B<man sem>
 Implies B<--semaphore>.
 
 
-=item B<--filter-hosts>
+=item B<--filter-hosts> (alpha testing)
 
 Remove down hosts. For each remote host: check that login through ssh
 works. If not: do not use this host.
@@ -657,7 +657,7 @@ to see the difference:
 
 =item B<-L> I<max-lines>
 
-When used with B<--pipe>: Read records of I<max-lines>.
+When used with B<--pipe>: Read records of I<max-lines> (alpha testing).
 
 When used otherwise: Use at most I<max-lines> nonblank input lines per
 command line.  Trailing blanks cause an input line to be logically
@@ -673,7 +673,7 @@ Implies B<-X> unless B<-m>, B<--xargs>, or B<--pipe> is set.
 
 =item B<-l>[I<max-lines>]
 
-When used with B<--pipe>: Read records of I<max-lines>.
+When used with B<--pipe>: Read records of I<max-lines> (alpha testing).
 
 When used otherwise: Synonym for the B<-L> option.  Unlike B<-L>, the
 I<max-lines> argument is optional.  If I<max-lines> is not specified,

testsuite/tests-to-run/niceload01.sh

@@ -4,55 +4,53 @@ echo '### Test niceload -q'
 niceload -q perl -e '$a = "works";$b="This $a\n"; print($b);'
 echo 
 
-# Force less than 1 GB buffer+cache
-MEMAVAIL=$(free | perl -ane '/buffers.cache:/ and print $F[3]')
-while [ $MEMAVAIL -gt 1000000 ] ; do
-  BS=$(echo $MEMAVAIL/10 | bc)
-  (seq 1 5 | parallel -j0 -N0 timeout 10 nice dd if=/dev/zero of=/dev/null bs=${BS}k &)
-  sleep 2;
-  MEMAVAIL=$(free | perl -ane '/buffers.cache:/ and print $F[3]')
-done
-
-echo '### --rm and --runmem'
-niceload  -H --rm 1g free -g | perl -ane '/buffers.cache:/ and print $F[3],"\n"' | grep '[1-9]' >/dev/null && echo OK &
-niceload  -H --runmem 1g free -g | perl -ane '/buffers.cache:/ and print $F[3],"\n"' | grep '[1-9]' >/dev/null && echo OK &
-wait
-
-
 # Force swapping
 MEMAVAIL=$(free | perl -ane '/buffers.cache:/ and print $F[3]')
 while [ $MEMAVAIL -gt 1000000 ] ; do
   BS=$(echo $MEMAVAIL/10 | bc)
-  (seq 1 10 | parallel -j0 -N0 timeout 10 nice dd if=/dev/zero of=/dev/null bs=${BS}k &)
-  sleep 2;
+  (seq 1 10 | parallel -j0 -N0 timeout 10 nice dd if=/dev/zero bs=${BS}k '|' wc -c >/dev/null &)
+  sleep 2
   MEMAVAIL=$(free | perl -ane '/buffers.cache:/ and print $F[3]')
 done
 
-echo '### -N and --noswap '
-niceload -H -N vmstat 1 2 | tail -n1 | awk '{print $7*$8}' &
-niceload -H --noswap vmstat 1 2 | tail -n1 | awk '{print $7*$8}' &
-wait
+#echo 1 | parallel --timeout 20 'seq 10000{} | gzip -1 | perl -e '\'\$a=join\"\",\<\>\;\ while\(1\)\{push\ @a,\$a\}\'
+
+
+# niceload -q -l 5 perl -e '$a=join"",<>; while(1){push @a,$a}' &
+
+cat <<'EOF' | stdout parallel -j0 -k -L1
+echo '### --rm and --runmem'
+  niceload  -H --rm 1g free -g | perl -ane '/buffers.cache:/ and print $F[3],"\n"' | grep '[1-9]' >/dev/null && echo OK--rm
+  niceload  -H --runmem 1g free -g | perl -ane '/buffers.cache:/ and print $F[3],"\n"' | grep '[1-9]' >/dev/null && echo OK--runmem
+
+echo '### -N and --noswap. Must give 0'
+  niceload -H -N vmstat 1 2 | tail -n1 | awk '{print "-N " $7*$8}'
+  niceload -H --noswap vmstat 1 2 | tail -n1 | awk '{print "--noswap " $7*$8}'
+EOF
 
 # force load > 10
 while uptime | grep -v age:.[1-9][0-9].[0-9][0-9] >/dev/null ; do (timeout 5 nice burnP6 2>/dev/null &) done
 
+cat <<'EOF' | stdout parallel -j0 -L1
 echo '### --soft -f and test if child is actually suspended and thus takes longer'
-niceload --soft -t 0.2 -f 0.5 'seq 1000000 | wc;echo This should finish last' &
-(sleep 1; seq 1000000 | wc;echo This should finish first) &
-wait
+  niceload --soft -t 0.2 -f 0.5 'seq 1000000 | wc;echo This should finish last'
+  (sleep 1; seq 1000000 | wc;echo This should finish first)
+EOF
 
 # force load > 10
 while uptime | grep -v age:.[1-9][0-9].[0-9][0-9] >/dev/null ; do (timeout 5 nice burnP6 2>/dev/null &) done
 
+cat <<'EOF' | stdout parallel -j0 -k -L1
 echo '### -H and --hard'
-niceload  -H -l 9.9 uptime | grep ':.[1-9][0-9].[0-9][0-9],' || echo OK
-niceload  --hard -l 9 uptime | grep ':.[1-9][0-9].[0-9][0-9],' || echo OK
+  niceload  -H -l 9.9 uptime | grep ':.[1-9][0-9].[0-9][0-9],' || echo OK-l9.9
+  niceload  --hard -l 9 uptime | grep ':.[1-9][0-9].[0-9][0-9],' || echo OK-l9
+EOF
 
+cat <<'EOF' | stdout parallel -j0 -L1
 echo '### -f and --factor'
-niceload -H -f 0.01 -l6 echo f 0.1 first &
-niceload -H --factor 10 -l6 echo factor 10 last &
-wait
-
+  niceload -H --factor 10 -l6 echo factor 10 finish last
+  niceload -H -f 0.01 -l6 echo f 0.1 finish first
+EOF
 
 #echo '### Test niceload -p'
 #sleep 3 &

testsuite/tests-to-run/parallel-local2.sh

@@ -68,10 +68,3 @@ stdout parallel --env TWOSPACES,THREESPACES echo 'a"$TWOSPACES"b' 'a"$THREESPACE
 stdout parallel -S localhost --env TWOSPACES echo 'a"$TWOSPACES"b' ::: 1
 stdout parallel -S localhost --env TWOSPACES --env THREESPACES echo 'a"$TWOSPACES"b' 'a"$THREESPACES"b' ::: 2
 stdout parallel -S localhost --env TWOSPACES,THREESPACES echo 'a"$TWOSPACES"b' 'a"$THREESPACES"b' ::: 2a
-
-echo '### Test --env all chars except \n - single and double - no output is good'
-perl -e 'for(1..9,9,11..159,159,161..255) { printf "%c%c %c%d\0",$_,$_,$_,$_ }' | stdout parallel --nice 19 -j4 -k -I // --arg-sep _ -0 V=// V2=V2=// parallel -k -j1 -S :,1/lo,1/tcsh@lo,1/csh@lo --env V,V2 echo \''"{}$V$V2"'\' ::: {#} {#} {#} {#} | uniq -c | grep -v '   4 '|grep -v xauth |grep -v X11
-
-echo '### Test --env all chars except \n - single and double --onall - no output is good'
-perl -e 'for(1..9,9,11..159,159,161..255) { printf "%c%c %c%d\0",$_,$_,$_,$_ }' | stdout parallel --nice 19 -j4 -k -I // --arg-sep _ -0 V=// V2=V2=// parallel -k -j1 -S :,1/lo,1/tcsh@lo,1/csh@lo --onall --env V,V2 echo \''"{}$V$V2"'\' ::: {#} | uniq -c | grep -v '   4 '|grep -v xauth |grep -v X11
-

testsuite/tests-to-run/parallel-local6.sh

@@ -13,8 +13,8 @@ rm /tmp/parallel_joblog_exitval /tmp/parallel_joblog_signal
 
 echo '### Test --env all chars except \n,\92,\160 - single and double - no output is good'
 # 92 and 160 are special for csh
-perl -e 'for(1..9,9,11..91,91,93..159,159,161..255) { printf "%c%c %c%d\0",$_,$_,$_,$_ }' | stdout parallel --nice 19 -j4 -k -I // --arg-sep _ -0 V=// V2=V2=// parallel -k -j1 -S :,1/lo,1/tcsh@lo,1/csh@lo --env V,V2 echo \''"{}$V$V2"'\' ::: {#} {#} {#} {#} | uniq -c | grep -v '   4 '|grep -v xauth |grep -v X11
+perl -e 'for(1..9,9,11..91,91,93..159,159,161..255) { printf "%c%c %c%d\0",$_,$_,$_,$_ }' | stdout parallel --nice 10 -j4 -k -I // --arg-sep _ -0 V=// V2=V2=// parallel -k -j1 -S :,1/lo,1/tcsh@lo,1/csh@lo --env V,V2 echo \''"{}$V$V2"'\' ::: {#} {#} {#} {#} | uniq -c | grep -v '   4 '|grep -v xauth |grep -v X11
 
 echo '### Test --env all chars except \n,\92,\160 - single and double --onall - no output is good'
 # 92 and 160 are special for csh
-perl -e 'for(1..9,9,11..91,91,93..159,159,161..255) { printf "%c%c %c%d\0",$_,$_,$_,$_ }' | stdout parallel --nice 19 -j4 -k -I // --arg-sep _ -0 V=// V2=V2=// parallel -k -j1 -S :,1/lo,1/tcsh@lo,1/csh@lo --onall --env V,V2 echo \''"{}$V$V2"'\' ::: {#} | uniq -c | grep -v '   4 '|grep -v xauth |grep -v X11
+perl -e 'for(1..9,9,11..91,91,93..159,159,161..255) { printf "%c%c %c%d\0",$_,$_,$_,$_ }' | stdout parallel --nice 10 -j4 -k -I // --arg-sep _ -0 V=// V2=V2=// parallel -k -j1 -S :,1/lo,1/tcsh@lo,1/csh@lo --onall --env V,V2 echo \''"{}$V$V2"'\' ::: {#} | uniq -c | grep -v '   4 '|grep -v xauth |grep -v X11

testsuite/wanted-results/niceload01

@@ -2,19 +2,19 @@
 This works
 
 ### --rm and --runmem
-OK
-OK
-### -N and --noswap 
-0
-0
+OK--rm
+OK--runmem
+### -N and --noswap. Must give 0
+-N 0
+--noswap 0
 ### --soft -f and test if child is actually suspended and thus takes longer
 1000000 1000000 6888896
 This should finish first
 1000000 1000000 6888896
 This should finish last
 ### -H and --hard
-OK
-OK
+OK-l9.9
+OK-l9
 ### -f and --factor
-f 0.1 first
-factor 10 last
+f 0.1 finish first
+factor 10 finish last