diff --git a/NEWS b/NEWS index 9b0372a9..f236e5c3 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,44 @@ +20121022 + +* --results structures output files so they can be imported using + Pandas (http://pandas.pydata.org/). Useful when doing simulations to + select output from some arguments. + +* Use --env to transfer environment variables through ssh to remote + machines. + +* GNU Parallel is accepted for Ubuntu Quantal. + http://packages.ubuntu.com/quantal/parallel + +* GNU Parallel was requested for OpenSUSE (go vote!) + https://features.opensuse.org/314461 + +* Kneth's Korner: Map/Reduce and GNU Parallel + http://kenneth.geisshirt.dk/2012/10/mapreduce-and-gnu-parallel.html + +* Blog post in Japanese by Siguniang. + https://siguniang.wordpress.com/2012/09/09/notes-on-gnu-parallel-the-command-line-power-tool/ + (Siguniang mentions the logo is called 'The Cafe Wall Illusion') + http://www.michaelbach.de/ot/ang_cafewall/index.html + +* GNU Parallel, where have you been all my life? + http://ssdigit.nothingisreal.com/2012/09/gnu-parallel-where-have-you-been-all-my.html + +* Reference-free ddRADseq analysis tools uses GNU Parallel. + https://github.com/brantp/rtd#readme + +* When bash just isn’t quick enough. + https://soimasysadmin.wordpress.com/2012/08/15/gnu-parallel-when-bash-just-isnt-quick-enough/ + +* Using GNU Parallel to Package Multiple Jobs in a Single PBS Job. + http://www.nas.nasa.gov/hecc/support/kb/Using-GNU-Parallel-to-Package-Multiple-Jobs-in-a-Single-PBS-Job_303.html + +* GNU Parallel Notes. + http://xed.ch/help/parallel.html + +* Bug fixes and man page updates. + + 20120822 * Due to a bugfix the main flow has changed. Making this release beta diff --git a/doc/release_new_version b/doc/release_new_version index 3b7da6e1..d5cd6597 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -178,7 +178,8 @@ available for download at: http://ftp.gnu.org/gnu/parallel/ New in this release: -* When using GNU Parallel for simulations --results makes it easy to +* --results structures output files so they can be imported using + Pandas (http://pandas.pydata.org/). Useful when doing simulations to select output from some arguments. * Use --env to transfer environment variables through ssh to remote diff --git a/src/parallel.pod b/src/parallel.pod index 2f14aae8..30aaf145 100644 --- a/src/parallel.pod +++ b/src/parallel.pod @@ -445,7 +445,7 @@ If I is omitted, there is no end of file string. If neither B<-E> nor B<-e> is used, no end of file string is used. -=item B<--env> I +=item B<--env> I (alpha testing) Copy environment variable I. This will copy I to the environment that the command is run in. This is especially useful for @@ -472,7 +472,7 @@ See also: B<--bg>, B Implies B<--semaphore>. -=item B<--filter-hosts> +=item B<--filter-hosts> (alpha testing) Remove down hosts. For each remote host: check that login through ssh works. If not: do not use this host. @@ -657,7 +657,7 @@ to see the difference: =item B<-L> I -When used with B<--pipe>: Read records of I. +When used with B<--pipe>: Read records of I (alpha testing). When used otherwise: Use at most I nonblank input lines per command line. Trailing blanks cause an input line to be logically @@ -673,7 +673,7 @@ Implies B<-X> unless B<-m>, B<--xargs>, or B<--pipe> is set. =item B<-l>[I] -When used with B<--pipe>: Read records of I. +When used with B<--pipe>: Read records of I (alpha testing). When used otherwise: Synonym for the B<-L> option. Unlike B<-L>, the I argument is optional. If I is not specified, diff --git a/testsuite/tests-to-run/niceload01.sh b/testsuite/tests-to-run/niceload01.sh index 6dfdcb45..3771aadb 100644 --- a/testsuite/tests-to-run/niceload01.sh +++ b/testsuite/tests-to-run/niceload01.sh @@ -4,55 +4,53 @@ echo '### Test niceload -q' niceload -q perl -e '$a = "works";$b="This $a\n"; print($b);' echo -# Force less than 1 GB buffer+cache -MEMAVAIL=$(free | perl -ane '/buffers.cache:/ and print $F[3]') -while [ $MEMAVAIL -gt 1000000 ] ; do - BS=$(echo $MEMAVAIL/10 | bc) - (seq 1 5 | parallel -j0 -N0 timeout 10 nice dd if=/dev/zero of=/dev/null bs=${BS}k &) - sleep 2; - MEMAVAIL=$(free | perl -ane '/buffers.cache:/ and print $F[3]') -done - -echo '### --rm and --runmem' -niceload -H --rm 1g free -g | perl -ane '/buffers.cache:/ and print $F[3],"\n"' | grep '[1-9]' >/dev/null && echo OK & -niceload -H --runmem 1g free -g | perl -ane '/buffers.cache:/ and print $F[3],"\n"' | grep '[1-9]' >/dev/null && echo OK & -wait - - # Force swapping MEMAVAIL=$(free | perl -ane '/buffers.cache:/ and print $F[3]') while [ $MEMAVAIL -gt 1000000 ] ; do BS=$(echo $MEMAVAIL/10 | bc) - (seq 1 10 | parallel -j0 -N0 timeout 10 nice dd if=/dev/zero of=/dev/null bs=${BS}k &) - sleep 2; + (seq 1 10 | parallel -j0 -N0 timeout 10 nice dd if=/dev/zero bs=${BS}k '|' wc -c >/dev/null &) + sleep 2 MEMAVAIL=$(free | perl -ane '/buffers.cache:/ and print $F[3]') done -echo '### -N and --noswap ' -niceload -H -N vmstat 1 2 | tail -n1 | awk '{print $7*$8}' & -niceload -H --noswap vmstat 1 2 | tail -n1 | awk '{print $7*$8}' & -wait +#echo 1 | parallel --timeout 20 'seq 10000{} | gzip -1 | perl -e '\'\$a=join\"\",\<\>\;\ while\(1\)\{push\ @a,\$a\}\' + + +# niceload -q -l 5 perl -e '$a=join"",<>; while(1){push @a,$a}' & + +cat <<'EOF' | stdout parallel -j0 -k -L1 +echo '### --rm and --runmem' + niceload -H --rm 1g free -g | perl -ane '/buffers.cache:/ and print $F[3],"\n"' | grep '[1-9]' >/dev/null && echo OK--rm + niceload -H --runmem 1g free -g | perl -ane '/buffers.cache:/ and print $F[3],"\n"' | grep '[1-9]' >/dev/null && echo OK--runmem + +echo '### -N and --noswap. Must give 0' + niceload -H -N vmstat 1 2 | tail -n1 | awk '{print "-N " $7*$8}' + niceload -H --noswap vmstat 1 2 | tail -n1 | awk '{print "--noswap " $7*$8}' +EOF # force load > 10 while uptime | grep -v age:.[1-9][0-9].[0-9][0-9] >/dev/null ; do (timeout 5 nice burnP6 2>/dev/null &) done +cat <<'EOF' | stdout parallel -j0 -L1 echo '### --soft -f and test if child is actually suspended and thus takes longer' -niceload --soft -t 0.2 -f 0.5 'seq 1000000 | wc;echo This should finish last' & -(sleep 1; seq 1000000 | wc;echo This should finish first) & -wait + niceload --soft -t 0.2 -f 0.5 'seq 1000000 | wc;echo This should finish last' + (sleep 1; seq 1000000 | wc;echo This should finish first) +EOF # force load > 10 while uptime | grep -v age:.[1-9][0-9].[0-9][0-9] >/dev/null ; do (timeout 5 nice burnP6 2>/dev/null &) done +cat <<'EOF' | stdout parallel -j0 -k -L1 echo '### -H and --hard' -niceload -H -l 9.9 uptime | grep ':.[1-9][0-9].[0-9][0-9],' || echo OK -niceload --hard -l 9 uptime | grep ':.[1-9][0-9].[0-9][0-9],' || echo OK + niceload -H -l 9.9 uptime | grep ':.[1-9][0-9].[0-9][0-9],' || echo OK-l9.9 + niceload --hard -l 9 uptime | grep ':.[1-9][0-9].[0-9][0-9],' || echo OK-l9 +EOF +cat <<'EOF' | stdout parallel -j0 -L1 echo '### -f and --factor' -niceload -H -f 0.01 -l6 echo f 0.1 first & -niceload -H --factor 10 -l6 echo factor 10 last & -wait - + niceload -H --factor 10 -l6 echo factor 10 finish last + niceload -H -f 0.01 -l6 echo f 0.1 finish first +EOF #echo '### Test niceload -p' #sleep 3 & diff --git a/testsuite/tests-to-run/parallel-local2.sh b/testsuite/tests-to-run/parallel-local2.sh index 225b3d74..84054958 100644 --- a/testsuite/tests-to-run/parallel-local2.sh +++ b/testsuite/tests-to-run/parallel-local2.sh @@ -68,10 +68,3 @@ stdout parallel --env TWOSPACES,THREESPACES echo 'a"$TWOSPACES"b' 'a"$THREESPACE stdout parallel -S localhost --env TWOSPACES echo 'a"$TWOSPACES"b' ::: 1 stdout parallel -S localhost --env TWOSPACES --env THREESPACES echo 'a"$TWOSPACES"b' 'a"$THREESPACES"b' ::: 2 stdout parallel -S localhost --env TWOSPACES,THREESPACES echo 'a"$TWOSPACES"b' 'a"$THREESPACES"b' ::: 2a - -echo '### Test --env all chars except \n - single and double - no output is good' -perl -e 'for(1..9,9,11..159,159,161..255) { printf "%c%c %c%d\0",$_,$_,$_,$_ }' | stdout parallel --nice 19 -j4 -k -I // --arg-sep _ -0 V=// V2=V2=// parallel -k -j1 -S :,1/lo,1/tcsh@lo,1/csh@lo --env V,V2 echo \''"{}$V$V2"'\' ::: {#} {#} {#} {#} | uniq -c | grep -v ' 4 '|grep -v xauth |grep -v X11 - -echo '### Test --env all chars except \n - single and double --onall - no output is good' -perl -e 'for(1..9,9,11..159,159,161..255) { printf "%c%c %c%d\0",$_,$_,$_,$_ }' | stdout parallel --nice 19 -j4 -k -I // --arg-sep _ -0 V=// V2=V2=// parallel -k -j1 -S :,1/lo,1/tcsh@lo,1/csh@lo --onall --env V,V2 echo \''"{}$V$V2"'\' ::: {#} | uniq -c | grep -v ' 4 '|grep -v xauth |grep -v X11 - diff --git a/testsuite/tests-to-run/parallel-local6.sh b/testsuite/tests-to-run/parallel-local6.sh index 6d591098..3bd27fa2 100755 --- a/testsuite/tests-to-run/parallel-local6.sh +++ b/testsuite/tests-to-run/parallel-local6.sh @@ -13,8 +13,8 @@ rm /tmp/parallel_joblog_exitval /tmp/parallel_joblog_signal echo '### Test --env all chars except \n,\92,\160 - single and double - no output is good' # 92 and 160 are special for csh -perl -e 'for(1..9,9,11..91,91,93..159,159,161..255) { printf "%c%c %c%d\0",$_,$_,$_,$_ }' | stdout parallel --nice 19 -j4 -k -I // --arg-sep _ -0 V=// V2=V2=// parallel -k -j1 -S :,1/lo,1/tcsh@lo,1/csh@lo --env V,V2 echo \''"{}$V$V2"'\' ::: {#} {#} {#} {#} | uniq -c | grep -v ' 4 '|grep -v xauth |grep -v X11 +perl -e 'for(1..9,9,11..91,91,93..159,159,161..255) { printf "%c%c %c%d\0",$_,$_,$_,$_ }' | stdout parallel --nice 10 -j4 -k -I // --arg-sep _ -0 V=// V2=V2=// parallel -k -j1 -S :,1/lo,1/tcsh@lo,1/csh@lo --env V,V2 echo \''"{}$V$V2"'\' ::: {#} {#} {#} {#} | uniq -c | grep -v ' 4 '|grep -v xauth |grep -v X11 echo '### Test --env all chars except \n,\92,\160 - single and double --onall - no output is good' # 92 and 160 are special for csh -perl -e 'for(1..9,9,11..91,91,93..159,159,161..255) { printf "%c%c %c%d\0",$_,$_,$_,$_ }' | stdout parallel --nice 19 -j4 -k -I // --arg-sep _ -0 V=// V2=V2=// parallel -k -j1 -S :,1/lo,1/tcsh@lo,1/csh@lo --onall --env V,V2 echo \''"{}$V$V2"'\' ::: {#} | uniq -c | grep -v ' 4 '|grep -v xauth |grep -v X11 +perl -e 'for(1..9,9,11..91,91,93..159,159,161..255) { printf "%c%c %c%d\0",$_,$_,$_,$_ }' | stdout parallel --nice 10 -j4 -k -I // --arg-sep _ -0 V=// V2=V2=// parallel -k -j1 -S :,1/lo,1/tcsh@lo,1/csh@lo --onall --env V,V2 echo \''"{}$V$V2"'\' ::: {#} | uniq -c | grep -v ' 4 '|grep -v xauth |grep -v X11 diff --git a/testsuite/wanted-results/niceload01 b/testsuite/wanted-results/niceload01 index 48e374b2..3b8cd5f6 100644 --- a/testsuite/wanted-results/niceload01 +++ b/testsuite/wanted-results/niceload01 @@ -2,19 +2,19 @@ This works ### --rm and --runmem -OK -OK -### -N and --noswap -0 -0 +OK--rm +OK--runmem +### -N and --noswap. Must give 0 +-N 0 +--noswap 0 ### --soft -f and test if child is actually suspended and thus takes longer 1000000 1000000 6888896 This should finish first 1000000 1000000 6888896 This should finish last ### -H and --hard -OK -OK +OK-l9.9 +OK-l9 ### -f and --factor -f 0.1 first -factor 10 last +f 0.1 finish first +factor 10 finish last