diff --git a/doc/release_new_version b/doc/release_new_version index d5cd6597..eb60055c 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -171,48 +171,40 @@ cc:Sandro Cazzaniga , Ryoichiro Suzuki , Jesse Alama -Subject: GNU Parallel 20121022 ('Hanne Vilmann') released +Subject: GNU Parallel 20121122 ('Barack') released -GNU Parallel 20121022 ('Hanne Vilmann') has been released. It is +GNU Parallel 20121122 ('Barack') has been released. It is available for download at: http://ftp.gnu.org/gnu/parallel/ New in this release: -* --results structures output files so they can be imported using - Pandas (http://pandas.pydata.org/). Useful when doing simulations to - select output from some arguments. +* GNU Parallel was part of the justification for Nordic Free Software + Award 2012 + https://fsfe.org/news/2012/news-20121112-01.html -* Use --env to transfer environment variables through ssh to remote - machines. +* Dateien schneller packen und konvertieren mit GNU parallel + http://www.linux-community.de/Archiv/Tipp-der-Woche/Dateien-schneller-packen-und-konvertieren-mit-GNU-parallel -* GNU Parallel is accepted for Ubuntu Quantal. - http://packages.ubuntu.com/quantal/parallel +* Processing images from Lunar Reconnaissance Orbiter Narrow Angle + Camera http://lunokhod.org/?p=774 -* GNU Parallel was requested for OpenSUSE (go vote!) - https://features.opensuse.org/314461 +* GNU Parallel is used in types2: Type and Hapax Accumulation Curves + http://www.cs.helsinki.fi/u/josuomel/types2/ -* Kneth's Korner: Map/Reduce and GNU Parallel - http://kenneth.geisshirt.dk/2012/10/mapreduce-and-gnu-parallel.html +* GNU Parallel used in: A Hybrid Factor Analysis and Probabilistic + PCA-based system for Dictionary Learning and Encoding for Robust + Speaker Recognition + http://www.superlectures.com/odyssey2012/lecture.php?id=8&lang=en + http://www.cse.iitm.ac.in/~mrsri/ppca.pdf -* Blog post in Japanese by Siguniang. - https://siguniang.wordpress.com/2012/09/09/notes-on-gnu-parallel-the-command-line-power-tool/ - (Siguniang mentions the logo is called 'The Cafe Wall Illusion') - http://www.michaelbach.de/ot/ang_cafewall/index.html +* Mapreduce avec parallel, cat et une redirection + http://datacratic.com/site/blog/mapreduce-avec-parallel-cat-et-une-redirection -* GNU Parallel, where have you been all my life? - http://ssdigit.nothingisreal.com/2012/09/gnu-parallel-where-have-you-been-all-my.html +* GNU Parallel for Large Batches of Small Jobs + https://support.scinet.utoronto.ca/wiki/images/7/7b/Tech-talk-gnu-parallel.pdf -* Reference-free ddRADseq analysis tools uses GNU Parallel. - https://github.com/brantp/rtd#readme - -* When bash just isn’t quick enough. - https://soimasysadmin.wordpress.com/2012/08/15/gnu-parallel-when-bash-just-isnt-quick-enough/ - -* Using GNU Parallel to Package Multiple Jobs in a Single PBS Job. - http://www.nas.nasa.gov/hecc/support/kb/Using-GNU-Parallel-to-Package-Multiple-Jobs-in-a-Single-PBS-Job_303.html - -* GNU Parallel Notes. - http://xed.ch/help/parallel.html +* GNU Parallel used in black belt tip (8:50-10:12) + https://aws.amazon.com/apac/awspodcast/episode20/ * Bug fixes and man page updates. diff --git a/packager/obs/home:tange/parallel/parallel.spec b/packager/obs/home:tange/parallel/parallel.spec index b52c283c..b521042f 100644 --- a/packager/obs/home:tange/parallel/parallel.spec +++ b/packager/obs/home:tange/parallel/parallel.spec @@ -1,6 +1,6 @@ Summary: Shell tool for executing jobs in parallel Name: parallel -Version: 20120822 +Version: 20121122 Release: 1 License: GPL Group: Productivity/File utilities diff --git a/src/parallel b/src/parallel index 0020220a..821e45ad 100755 --- a/src/parallel +++ b/src/parallel @@ -606,7 +606,7 @@ sub get_options_from_array { sub parse_options { # Returns: N/A # Defaults: - $Global::version = 20121023; + $Global::version = 20121122; $Global::progname = 'parallel'; $Global::infinity = 2**31; $Global::debug = 0; diff --git a/testsuite/Start.sh b/testsuite/Start.sh index 783c1013..bc255221 100644 --- a/testsuite/Start.sh +++ b/testsuite/Start.sh @@ -1,13 +1,21 @@ #!/bin/bash +# Argument can be substring of tests (such as 'local') + export LANG=C SHFILE=/tmp/unittest-parallel.sh -# Try a failing test twice. +# Run a failing test once ls -t tests-to-run/*${1}*.sh \ -| perl -pe 's:(.*/(.*)).sh:bash $1.sh > actual-results/$2; diff -Naur wanted-results/$2 actual-results/$2 >/dev/null || bash $1.sh > actual-results/$2; diff -Naur wanted-results/$2 actual-results/$2:' \ +| perl -pe 's:(.*/(.*)).sh:bash $1.sh > actual-results/$2; diff -Naur wanted-results/$2 actual-results/$2: ' \ >$SHFILE +# # Try a failing test thrice +# ls -t tests-to-run/*${1}*.sh \ +# | perl -pe 's:(.*/(.*)).sh:bash $1.sh > actual-results/$2; diff -Naur wanted-results/$2 actual-results/$2 >/dev/null || bash $1.sh > actual-results/$2; diff -Naur wanted-results/$2 actual-results/$2 >/dev/null || bash $1.sh > actual-results/$2; diff -Naur wanted-results/$2 actual-results/$2: ' \ +# >$SHFILE + + mkdir -p actual-results stdout sh -x $SHFILE | tee testsuite.log rm $SHFILE diff --git a/testsuite/tests-to-run/niceload01.sh b/testsuite/tests-to-run/niceload01.sh index 84ebee91..222445ec 100644 --- a/testsuite/tests-to-run/niceload01.sh +++ b/testsuite/tests-to-run/niceload01.sh @@ -8,7 +8,7 @@ echo MEMAVAIL=$(free | perl -ane '/buffers.cache:/ and print $F[3]') while [ $MEMAVAIL -gt 1000000 ] ; do BS=$(echo $MEMAVAIL/10 | bc) - (seq 1 10 | parallel -j0 -N0 timeout 10 nice dd if=/dev/zero bs=${BS}k '|' wc -c >/dev/null &) + (seq 1 10 | parallel -j0 -N0 timeout 15 nice dd if=/dev/zero bs=${BS}k '|' wc -c >/dev/null &) sleep 2 MEMAVAIL=$(free | perl -ane '/buffers.cache:/ and print $F[3]') done diff --git a/testsuite/tests-to-run/parallel-local114.sh b/testsuite/tests-to-run/parallel-local114.sh index 23f2e2d5..78493dd3 100755 --- a/testsuite/tests-to-run/parallel-local114.sh +++ b/testsuite/tests-to-run/parallel-local114.sh @@ -49,32 +49,32 @@ echo "bug #34958: --pipe with record size measured in lines"; echo "### Test --results"; mkdir -p /tmp/parallel_results_test; - parallel -k --results /tmp/parallel_results_test/testA echo {1} {2} ::: I II ::: III IIII; + parallel -k --results /tmp/parallel_results_test/testA echo {1} {2} ::: I II ::: III IIII; ls /tmp/parallel_results_test/testA*; rm /tmp/parallel_results_test/testA* echo "### Test --res"; mkdir -p /tmp/parallel_results_test; - parallel -k --results /tmp/parallel_results_test/testD echo {1} {2} ::: I II ::: III IIII; + parallel -k --results /tmp/parallel_results_test/testD echo {1} {2} ::: I II ::: III IIII; ls /tmp/parallel_results_test/testD*; rm /tmp/parallel_results_test/testD* echo "### Test --result"; mkdir -p /tmp/parallel_results_test; - parallel -k --result /tmp/parallel_results_test/testE echo {1} {2} ::: I II ::: III IIII; + parallel -k --result /tmp/parallel_results_test/testE echo {1} {2} ::: I II ::: III IIII; ls /tmp/parallel_results_test/testE*; rm /tmp/parallel_results_test/testE* echo "### Test --results --header :"; mkdir -p /tmp/parallel_results_test; - parallel -k --header : --results /tmp/parallel_results_test/testB echo {1} {2} ::: a I II ::: b III IIII + parallel -k --header : --results /tmp/parallel_results_test/testB echo {1} {2} ::: a I II ::: b III IIII; ls /tmp/parallel_results_test/testB*; rm /tmp/parallel_results_test/testB* -echo "### Test --results --header :"; +echo "### Test --results --header : named"; mkdir -p /tmp/parallel_results_test; - parallel -k --header : --results /tmp/parallel_results_test/testC echo {a} {b} ::: a I II ::: b III IIII + parallel -k --header : --results /tmp/parallel_results_test/testC echo {a} {b} ::: a I II ::: b III IIII; ls /tmp/parallel_results_test/testC*; rm /tmp/parallel_results_test/testC* echo "### Test --results --header : piped"; mkdir -p /tmp/parallel_results_test; - (echo Col; perl -e 'print "backslash\\tab\tslash/null\0eof\n"') | parallel --header : --result /tmp/parallel_results_test/testF_ true; + (echo Col; perl -e 'print "backslash\\tab\tslash/null\0eof\n"') | parallel --header : --result /tmp/parallel_results_test/testF_ true; ls /tmp/parallel_results_test/testF*; rm /tmp/parallel_results_test/testF* EOF diff --git a/testsuite/tests-to-run/parallel-local16.sh b/testsuite/tests-to-run/parallel-local16.sh new file mode 100755 index 00000000..aef5ee12 --- /dev/null +++ b/testsuite/tests-to-run/parallel-local16.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +echo '### Test --env all chars except \n,\92,\160 - single and double - no output is good' +# 92 and 160 are special for csh +perl -e 'for(1..9,9,11..91,91,93..159,159,161..255) { printf "%c%c %c%d\0",$_,$_,$_,$_ }' | stdout parallel --nice 10 -j4 -k -I // --arg-sep _ -0 V=// V2=V2=// parallel --retries 3 -k -j1 -S :,1/lo,1/tcsh@lo,1/csh@lo --env V,V2 echo \''"{}$V$V2"'\' ::: {#} {#} {#} {#} | uniq -c | grep -v ' 4 '|grep -v xauth |grep -v X11 diff --git a/testsuite/tests-to-run/parallel-local2.sh b/testsuite/tests-to-run/parallel-local2.sh index 84054958..9394fe44 100644 --- a/testsuite/tests-to-run/parallel-local2.sh +++ b/testsuite/tests-to-run/parallel-local2.sh @@ -3,68 +3,11 @@ echo '### Test slow arguments generation - https://savannah.gnu.org/bugs/?32834' seq 1 3 | parallel -j1 "sleep 2; echo {}" | parallel -kj2 echo -echo '### Test --env - https://savannah.gnu.org/bugs/?37351' -export TWOSPACES=' 2 spaces ' -export THREESPACES=" > My brother's 12\" records < " -echo a"$TWOSPACES"b 1 -parallel --env TWOSPACES echo 'a"$TWOSPACES"b' ::: 1 -parallel -S localhost --env TWOSPACES echo 'a"$TWOSPACES"b' ::: 1 -parallel -S csh@localhost --env TWOSPACES echo 'a"$TWOSPACES"b' ::: 1 -parallel -S tcsh@localhost --env TWOSPACES echo 'a"$TWOSPACES"b' ::: 1 - -echo a"$TWOSPACES"b a"$THREESPACES"b 2 -parallel --env TWOSPACES --env THREESPACES echo 'a"$TWOSPACES"b' 'a"$THREESPACES"b' ::: 2 -parallel -S localhost --env TWOSPACES --env THREESPACES echo 'a"$TWOSPACES"b' 'a"$THREESPACES"b' ::: 2 -parallel -S csh@localhost --env TWOSPACES --env THREESPACES echo 'a"$TWOSPACES"b' 'a"$THREESPACES"b' ::: 2 -parallel -S tcsh@localhost --env TWOSPACES --env THREESPACES echo 'a"$TWOSPACES"b' 'a"$THREESPACES"b' ::: 2 - -echo a"$TWOSPACES"b a"$THREESPACES"b 3 -parallel --env TWOSPACES,THREESPACES echo 'a"$TWOSPACES"b' 'a"$THREESPACES"b' ::: 3 -parallel -S localhost --env TWOSPACES,THREESPACES echo 'a"$TWOSPACES"b' 'a"$THREESPACES"b' ::: 3 -parallel -S csh@localhost --env TWOSPACES,THREESPACES echo 'a"$TWOSPACES"b' 'a"$THREESPACES"b' ::: 3 -parallel -S tcsh@localhost --env TWOSPACES,THREESPACES echo 'a"$TWOSPACES"b' 'a"$THREESPACES"b' ::: 3 - -export MIN=" \'\"" -echo a"$MIN"b 4 -parallel --env MIN echo 'a"$MIN"b' ::: 4 -parallel -S localhost --env MIN echo 'a"$MIN"b' ::: 4 -parallel -S csh@localhost --env MIN echo 'a"$MIN"b' ::: 4 -parallel -S tcsh@localhost --env MIN echo 'a"$MIN"b' ::: 4 - -export SPC="'"' * ? >o /dev/null -seq `parallel --number-of-cores` | parallel -j100% -N0 timeout -k 25 26 burnP6 & +seq `parallel --number-of-cores` | parallel -j200% -N0 timeout -k 25 26 burnP6 & sleep 1 seq 1 1000 | stdout nice nice parallel -s 100 -uj0 true | perl -pe '/parallel: Warning: Starting \d+ processes took/ and do {close STDIN; `killall -9 burnP6`; print "OK\n"; exit }' killall -9 burnP6 2>/dev/null - -echo '### Test --env - https://savannah.gnu.org/bugs/?37351' -export TWOSPACES=' 2 spaces ' -export THREESPACES=" > My brother's 12\" records < " -stdout parallel --env TWOSPACES echo 'a"$TWOSPACES"b' ::: 1 -stdout parallel --env TWOSPACES --env THREESPACES echo 'a"$TWOSPACES"b' 'a"$THREESPACES"b' ::: 2 -stdout parallel --env TWOSPACES,THREESPACES echo 'a"$TWOSPACES"b' 'a"$THREESPACES"b' ::: 2a -stdout parallel -S localhost --env TWOSPACES echo 'a"$TWOSPACES"b' ::: 1 -stdout parallel -S localhost --env TWOSPACES --env THREESPACES echo 'a"$TWOSPACES"b' 'a"$THREESPACES"b' ::: 2 -stdout parallel -S localhost --env TWOSPACES,THREESPACES echo 'a"$TWOSPACES"b' 'a"$THREESPACES"b' ::: 2a diff --git a/testsuite/tests-to-run/parallel-local8.sh b/testsuite/tests-to-run/parallel-local8.sh index 9e3357e2..92fa81cd 100644 --- a/testsuite/tests-to-run/parallel-local8.sh +++ b/testsuite/tests-to-run/parallel-local8.sh @@ -1,20 +1,21 @@ #!/bin/bash echo "### Test : as delimiter. This can be confusing for uptime ie. --load"; -parallel -k --load 100% -d : echo ::: a:b:c +parallel -k --load 300% -d : echo ::: a:b:c -export PARALLEL="--load 100%" +export PARALLEL="--load 300%" echo PARALLEL=$PARALLEL for i in $(seq 2 10); do i2=$[i*i] - seq $i2 | parallel -j0 --load 100% -kX echo {} |wc + seq $i2 | parallel -j0 --load 300% -kX echo {} |wc seq 1 ${i2}0000 | nice parallel -kj20 --recend "\n" --spreadstdin gzip -9 | zcat | sort -n | md5sum done echo "### Test if --load blocks. Bug."; -seq 1 1000 | parallel -kj2 --load 100% --recend "\n" --spreadstdin gzip -9 | zcat | sort -n | md5sum -seq 1 1000 | parallel -kj0 --load 100% --recend "\n" --spreadstdin gzip -9 | zcat | sort -n | md5sum +seq 1 1000 | parallel -kj2 --load 300% --recend "\n" --spreadstdin gzip -9 | zcat | sort -n | md5sum +seq 1 1000 | parallel -kj0 --load 300% --recend "\n" --spreadstdin gzip -9 | zcat | sort -n | md5sum seq 1 1000000 | parallel -kj0 --recend "\n" --spreadstdin gzip -9 | zcat | sort -n | md5sum seq 1 1000000 | nice parallel -kj20 --recend "\n" --spreadstdin gzip -9 | zcat | sort -n | md5sum + \ No newline at end of file diff --git a/testsuite/tests-to-run/parallel-local9.sh b/testsuite/tests-to-run/parallel-local9.sh index 629286e1..a62a1792 100644 --- a/testsuite/tests-to-run/parallel-local9.sh +++ b/testsuite/tests-to-run/parallel-local9.sh @@ -12,7 +12,7 @@ echo '### Test --spreadstdin - more procs than args'; cat /tmp/parallel.ss.*; echo '### Test --spreadstdin - more args than procs'; - rm /tmp/parallel.ss2.*; + rm -f /tmp/parallel.ss2.*; seq 1 10 | stdout parallel -j 5 --spreadstdin 'cat >/tmp/parallel.ss2.$PARALLEL_SEQ' >/dev/null; cat /tmp/parallel.ss2.* diff --git a/testsuite/tests-to-run/sem01.sh b/testsuite/tests-to-run/sem01.sh index 79d1d177..5efb9c17 100755 --- a/testsuite/tests-to-run/sem01.sh +++ b/testsuite/tests-to-run/sem01.sh @@ -6,12 +6,6 @@ parallel -u --semaphore seq 11 20 '|' pv -qL 100 parallel --semaphore --wait echo done -echo '### Test default id = --id `tty` and --semaphorename' -parallel --id `tty` -u --semaphore seq 1 10 '|' pv -qL 20 -parallel -u --semaphore seq 11 20 '|' pv -qL 100 -parallel --semaphorename `tty` --semaphore --wait -echo done - echo '### Test semaphore 2 jobs running simultaneously' parallel -u -j2 --semaphore 'echo job1a 1; sleep 1; echo job1b 3' sleep 0.2 @@ -33,6 +27,7 @@ echo done echo '### Test similar example as from man page - run 2 jobs simultaneously' echo 'Expect done: 1 2 5 3 4' for i in 5 1 2 3 4 ; do + sleep 0.2 echo Scheduling $i sem -j2 -u echo starting $i ";" sleep $i ";" echo done $i done diff --git a/testsuite/wanted-results/parallel-local114 b/testsuite/wanted-results/parallel-local114 index 3e47332d..8276e389 100644 --- a/testsuite/wanted-results/parallel-local114 +++ b/testsuite/wanted-results/parallel-local114 @@ -168,7 +168,7 @@ II IIII /tmp/parallel_results_test/testBstdout a I b IIII /tmp/parallel_results_test/testBstdout a II b III /tmp/parallel_results_test/testBstdout a II b IIII -### Test --results --header : +### Test --results --header : named I III I IIII II III diff --git a/testsuite/wanted-results/parallel-local2 b/testsuite/wanted-results/parallel-local2 index 24088d8e..55a91f5e 100644 --- a/testsuite/wanted-results/parallel-local2 +++ b/testsuite/wanted-results/parallel-local2 @@ -2,60 +2,5 @@ 1 2 3 -### Test --env - https://savannah.gnu.org/bugs/?37351 -a 2 spaces b 1 -a 2 spaces b 1 -a 2 spaces b 1 -a 2 spaces b 1 -a 2 spaces b 1 -a 2 spaces b a > My brother's 12" records < b 2 -a 2 spaces b a > My brother's 12" records < b 2 -a 2 spaces b a > My brother's 12" records < b 2 -a 2 spaces b a > My brother's 12" records < b 2 -a 2 spaces b a > My brother's 12" records < b 2 -a 2 spaces b a > My brother's 12" records < b 3 -a 2 spaces b a > My brother's 12" records < b 3 -a 2 spaces b a > My brother's 12" records < b 3 -a 2 spaces b a > My brother's 12" records < b 3 -a 2 spaces b a > My brother's 12" records < b 3 -a \'"b 4 -a \'"b 4 -a \'"b 4 -a \'"b 4 -a \'"b 4 -a' * ? >o o o o o My brother's 12" records < b 2 -a 2 spaces b a > My brother's 12" records < b 2a -a 2 spaces b 1 -a 2 spaces b a > My brother's 12" records < b 2 -a 2 spaces b a > My brother's 12" records < b 2a -### Test --env all chars except \n - single and double - no output is good - 3 92\\ \92V2=\\ \92 - 1 92\ \92V2=\ \92 -### Test --env all chars except \n - single and double --onall - no output is good - 3 92\\ \92V2=\\ \92 - 1 92\ \92V2=\ \92 diff --git a/testsuite/wanted-results/parallel-local8 b/testsuite/wanted-results/parallel-local8 index 20ef0c51..710882bd 100644 --- a/testsuite/wanted-results/parallel-local8 +++ b/testsuite/wanted-results/parallel-local8 @@ -2,7 +2,7 @@ a b c -PARALLEL=--load 100% +PARALLEL=--load 300% 4 4 8 1c0f34fee7176dc367bead8f96cba6bc - 9 9 18 diff --git a/testsuite/wanted-results/sem01 b/testsuite/wanted-results/sem01 index 64a77558..c91ae202 100644 --- a/testsuite/wanted-results/sem01 +++ b/testsuite/wanted-results/sem01 @@ -20,28 +20,6 @@ 19 20 done -### Test default id = --id `tty` and --semaphorename -1 -2 -3 -4 -5 -6 -7 -8 -9 -10 -11 -12 -13 -14 -15 -16 -17 -18 -19 -20 -done ### Test semaphore 2 jobs running simultaneously job1a 1 job2a 2 @@ -61,16 +39,16 @@ done ### Test similar example as from man page - run 2 jobs simultaneously Expect done: 1 2 5 3 4 Scheduling 5 -Scheduling 1 starting 5 -Scheduling 2 +Scheduling 1 starting 1 +Scheduling 2 done 1 -Scheduling 3 starting 2 +Scheduling 3 done 2 -Scheduling 4 starting 3 +Scheduling 4 done 5 starting 4 done 3