diff --git a/NEWS b/NEWS index e52da63a..16a3f578 100644 --- a/NEWS +++ b/NEWS @@ -19,8 +19,7 @@ * --transfer and --basefile support paths relative to the --workdir by inserting /./ into the path. -* GNU Parallel was used (unfortunately with improper citation) in: - 'fastphylo: Fast tools for phylogenetics' +* GNU Parallel was cited in: 'fastphylo: Fast tools for phylogenetics' http://www.biomedcentral.com/1471-2105/14/334/abstract * Using GNU parallel diff --git a/doc/release_new_version b/doc/release_new_version index 24427785..1254de6e 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -244,12 +244,12 @@ New in this release: Analysis Pipeline for Discovery of Human Genetic Variation http://www.google.com/patents/US20130311106 +* Encrypt and decrypt in parallel with GPG + http://blog.tkassembled.com/412/encrypt-and-decrypt-in-parallel-with-gpg/ + * Using GNU Parallel to create a SVM classifier http://ivanyu.me/blog/2013/12/01/parallel/ -* Mon make à moi (French) - http://www.slideshare.net/lindenb/mon-make-moi-tout-sauf-galaxy - * Parallel xz compression http://anthon.home.xs4all.nl/rants/2013/parallel_xz/ @@ -262,9 +262,18 @@ New in this release: * Gnu Parallel for fun and profit https://gist.github.com/celoyd/f7eb55ad69c9b33fd8c3 +* Parallelization of Exploitation + http://rantsideasstuff.com/parallelization-of-exploitation/ + +* Running assemblies in parallel + https://khmer-protocols.readthedocs.org/en/v0.8.3/metagenomics/4-assemble.html + * Procesando la contabilidad del PP http://www.neorazorx.com/2013/07/procesando-la-contabilidad-del-pp.html +* Mon make à moi (French) + http://www.slideshare.net/lindenb/mon-make-moi-tout-sauf-galaxy + * Bug fixes and man page updates. diff --git a/src/parallel.pod b/src/parallel.pod index 0847657a..685e64d3 100644 --- a/src/parallel.pod +++ b/src/parallel.pod @@ -3619,7 +3619,7 @@ B, B, B, and B can reproduce most errors. If your example requires large files, see if you can make them by something like B > B or B > B. If your example requires remote execution, see if you can -use B. +use B - maybe using another login. =item * diff --git a/testsuite/tests-to-run/parallel-local-0.3s.sh b/testsuite/tests-to-run/parallel-local-0.3s.sh new file mode 100644 index 00000000..995c3562 --- /dev/null +++ b/testsuite/tests-to-run/parallel-local-0.3s.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +# Simple jobs that never fails +# Each should be taking 0.3-1s and be possible to run in parallel +# I.e.: No race conditions, no logins +cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -k -j4 -L1 +echo '### Test exit val - true'; + echo true | parallel; + echo $? + +echo '**' + +echo '### Test exit val - false'; + echo false | parallel; + echo $? + +echo '**' + +EOF diff --git a/testsuite/tests-to-run/parallel-local-100s.sh b/testsuite/tests-to-run/parallel-local-100s.sh new file mode 100644 index 00000000..5943dca4 --- /dev/null +++ b/testsuite/tests-to-run/parallel-local-100s.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# Simple jobs that never fails +# Each should be taking >100s and be possible to run in parallel +# I.e.: No race conditions, no logins +cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -k -j4 -L1 +echo '### Test if we can deal with output > 4 GB' +## echo | niceload --io 10 parallel -q perl -e '"\$a=\"x\"x1000000;for(0..4300){print \$a}"' | md5sum + echo | parallel -q perl -e '$a="x"x1000000;for(0..4300){print $a}' | md5sum + +echo '**' + +echo "### Test Force outside the file handle limit, 2009-02-17 Gave fork error" + (echo echo Start; seq 1 20000 | perl -pe 's/^/true /'; echo echo end) | stdout parallel -uj 0 | egrep -v 'processes took|adjusting' + +echo '**' + +echo '### Test of --retries on unreachable host' + seq 2 | stdout parallel -k --retries 2 -v -S 4.3.2.1,: echo + +echo '**' + +EOF diff --git a/testsuite/tests-to-run/parallel-local-10s.sh b/testsuite/tests-to-run/parallel-local-10s.sh new file mode 100644 index 00000000..843738c8 --- /dev/null +++ b/testsuite/tests-to-run/parallel-local-10s.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +# Simple jobs that never fails +# Each should be taking 10-30s and be possible to run in parallel +# I.e.: No race conditions, no logins +cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -k -j4 -L1 +EOF diff --git a/testsuite/tests-to-run/parallel-local-30s.sh b/testsuite/tests-to-run/parallel-local-30s.sh new file mode 100644 index 00000000..0d1e56a4 --- /dev/null +++ b/testsuite/tests-to-run/parallel-local-30s.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +# Simple jobs that never fails +# Each should be taking 30-100s and be possible to run in parallel +# I.e.: No race conditions, no logins +cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -k -j4 -L1 +echo '### Test race condition on 8 CPU (my laptop)'; + seq 1 5000000 > /tmp/parallel_test; + seq 1 10 | parallel -k "cat /tmp/parallel_test | parallel --pipe --recend '' -k gzip >/dev/null; echo {}" + +echo '**' + +echo "### Test --tmpdir running full. bug #40733 was caused by this" + stdout parallel -j1 --tmpdir /run/shm cat /dev/zero ::: dummy + +echo '**' + + +EOF diff --git a/testsuite/tests-to-run/parallel-local11.sh b/testsuite/tests-to-run/parallel-local-3s.sh old mode 100755 new mode 100644 similarity index 57% rename from testsuite/tests-to-run/parallel-local11.sh rename to testsuite/tests-to-run/parallel-local-3s.sh index 59aa9627..0be372a5 --- a/testsuite/tests-to-run/parallel-local11.sh +++ b/testsuite/tests-to-run/parallel-local-3s.sh @@ -1,48 +1,9 @@ #!/bin/bash # Simple jobs that never fails -# Each should be taking 100s and be possible to run in parallel +# Each should be taking 3-10s and be possible to run in parallel # I.e.: No race conditions, no logins cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -k -j4 -L1 -echo '### Test if we can deal with output > 4 GB' -## echo | niceload --io 10 parallel -q perl -e '"\$a=\"x\"x1000000;for(0..4300){print \$a}"' | md5sum - echo | parallel -q perl -e '$a="x"x1000000;for(0..4300){print $a}' | md5sum - -echo '**' - -echo "### Test --tmpdir running full. bug #40733 was caused by this" - stdout parallel -j1 --tmpdir /run/shm cat /dev/zero ::: dummy - -echo '**' - -echo "### Test Force outside the file handle limit, 2009-02-17 Gave fork error" - (echo echo Start; seq 1 20000 | perl -pe 's/^/true /'; echo echo end) | stdout parallel -uj 0 | egrep -v 'processes took|adjusting' - -echo '**' - -echo '### Test of --retries on unreachable host' - seq 2 | stdout parallel -k --retries 2 -v -S 4.3.2.1,: echo - -echo '**' - -echo '### Test race condition on 8 CPU (my laptop)'; - seq 1 5000000 > /tmp/parallel_test; - seq 1 10 | parallel -k "cat /tmp/parallel_test | parallel --pipe --recend '' -k gzip >/dev/null; echo {}" - -echo '**' - -echo '### Test exit val - true'; - echo true | parallel; - echo $? - -echo '**' - -echo '### Test exit val - false'; - echo false | parallel; - echo $? - -echo '**' - echo '### Test --halt-on-error 0'; (echo "sleep 1;true"; echo "sleep 2;false";echo "sleep 3;true") | parallel -j10 --halt-on-error 0; echo $?; @@ -80,6 +41,4 @@ echo '**' echo '### Test slow arguments generation - https://savannah.gnu.org/bugs/?32834'; seq 1 3 | parallel -j1 "sleep 2; echo {}" | parallel -kj2 echo - EOF - diff --git a/testsuite/tests-to-run/parallel-local105.sh b/testsuite/tests-to-run/parallel-local105.sh index 76ac7ada..8c97937b 100755 --- a/testsuite/tests-to-run/parallel-local105.sh +++ b/testsuite/tests-to-run/parallel-local105.sh @@ -1,6 +1,6 @@ #!/bin/bash -TMP=/tmp/parallel_local105 +TMP=/run/shm/parallel_local105 rm -rf $TMP 2>/dev/null mkdir -p $TMP tar -C $TMP -xf input-files/random_dirs_with_newline.tar.bz2 diff --git a/testsuite/tests-to-run/parallel-local15.sh b/testsuite/tests-to-run/parallel-local15.sh index 1760c29e..5af13cf3 100755 --- a/testsuite/tests-to-run/parallel-local15.sh +++ b/testsuite/tests-to-run/parallel-local15.sh @@ -1,7 +1,9 @@ #!/bin/bash -rsync -Ha --delete input-files/testdir/ /tmp/parallel_$$ -cd /tmp/parallel_$$ +TMP=/run/shm/parallel_$$ + +rsync -Ha --delete input-files/testdir/ $TMP/ +cd $TMP/ echo echo test of cat pipe sh | parallel -j 50 2>&1 find . -name '*.jpg' | parallel -j +0 convert -geometry 120 {} {//}/thumb_{/} @@ -25,4 +27,4 @@ find . -type f | parallel -qv --group perl -ne '/^\S+\s+\S+$/ and print $ARGV,"\ EOF cd - -rm -rf /tmp/parallel_$$ +rm -rf $TMP/ diff --git a/testsuite/tests-to-run/parallel-local2.sh b/testsuite/tests-to-run/parallel-local2.sh index 1a60a464..6aeb9983 100644 --- a/testsuite/tests-to-run/parallel-local2.sh +++ b/testsuite/tests-to-run/parallel-local2.sh @@ -27,7 +27,7 @@ echo "bug #38441: CPU usage goes to 100% if load is higher than --load at first echo '### Test slow arguments generation - https://savannah.gnu.org/bugs/?32834' seq 1 3 | parallel -j1 "sleep 2; echo {}" | parallel -kj2 echo -echo '### Test too slow spawning - TODO THIS CURRENTLY DOES NOT OVERLOAD' +echo '### Test too slow spawning' # Let the commands below run during high load seq `parallel --number-of-cores` | parallel -j200% -N0 timeout -k 25 26 burnP6 & sleep 1; diff --git a/testsuite/tests-to-run/parallel-polarhome.sh b/testsuite/tests-to-run/parallel-polarhome.sh index 11ee1f9e..f9d8d99b 100644 --- a/testsuite/tests-to-run/parallel-polarhome.sh +++ b/testsuite/tests-to-run/parallel-polarhome.sh @@ -13,17 +13,14 @@ POLAR=`parallel -k echo {}.polarhome.com ::: $P` echo '### Tests on polarhome machines' echo 'Setup on polarhome machines' stdout parallel -kj0 ssh -oLogLevel=quiet {} mkdir -p bin ::: $POLAR & -# scp to each polarhome machine do not work. Use cat -copy_to_host() { + +copy_and_test() { H=$1 + # scp to each polarhome machine do not work. Use cat # Avoid the stupid /etc/issue.net banner with -oLogLevel=quiet - ssh -oLogLevel=quiet $H rm -f bin/parallel - cat `which parallel` | ssh -oLogLevel=quiet $H 'cat > bin/parallel; chmod 755 bin/parallel' + echo '### Run the test on '$H + cat `which parallel` | ssh -oLogLevel=quiet $H 'cat > bin/p.tmp && chmod 755 bin/p.tmp && mv bin/p.tmp bin/parallel; bin/perl bin/parallel echo Works on ::: '$H } -export -f copy_to_host -stdout parallel -j0 --timeout 20 --tag -kj0 -v copy_to_host {} ::: $POLAR -# Now test -echo '### Run the test on polarhome machines' -stdout parallel -j0 --argsep // -k --tag ssh -oLogLevel=quiet {} bin/perl bin/parallel -k echo Works on ::: {} // $POLAR - +export -f copy_and_test +stdout parallel -j0 -k --timeout 80 --delay 0.1 --tag -v copy_and_test {} ::: $POLAR diff --git a/testsuite/wanted-results/parallel-local-0.3s b/testsuite/wanted-results/parallel-local-0.3s new file mode 100644 index 00000000..52442028 --- /dev/null +++ b/testsuite/wanted-results/parallel-local-0.3s @@ -0,0 +1,6 @@ +### Test exit val - true +0 +** +### Test exit val - false +1 +** diff --git a/testsuite/wanted-results/parallel-local-100s b/testsuite/wanted-results/parallel-local-100s new file mode 100644 index 00000000..487c456d --- /dev/null +++ b/testsuite/wanted-results/parallel-local-100s @@ -0,0 +1,17 @@ +### Test if we can deal with output > 4 GB +46a318993dfc8e2afd71ff2bc6f605f1 - +** +### Test Force outside the file handle limit, 2009-02-17 Gave fork error +parallel: Warning: Only enough file handles to run 506 jobs in parallel. +Raising ulimit -n or /etc/security/limits.conf may help. +Start +end +** +### Test of --retries on unreachable host +ssh: connect to host 4.3.2.1 port 22: Connection timed out +parallel: Warning: Could not figure out number of cpus on 4.3.2.1 (). Using 1. +echo 1 +1 +echo 2 +2 +** diff --git a/testsuite/wanted-results/parallel-local-30s b/testsuite/wanted-results/parallel-local-30s new file mode 100644 index 00000000..d8b65861 --- /dev/null +++ b/testsuite/wanted-results/parallel-local-30s @@ -0,0 +1,16 @@ +### Test race condition on 8 CPU (my laptop) +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +** +### Test --tmpdir running full. bug #40733 was caused by this +parallel: Error: Output is incomplete. Cannot append to buffer file in $TMPDIR. Is the disk full? +parallel: Error: Change $TMPDIR with --tmpdir. +** diff --git a/testsuite/wanted-results/parallel-local11 b/testsuite/wanted-results/parallel-local-3s similarity index 72% rename from testsuite/wanted-results/parallel-local11 rename to testsuite/wanted-results/parallel-local-3s index 35524491..82c03e15 100644 --- a/testsuite/wanted-results/parallel-local11 +++ b/testsuite/wanted-results/parallel-local-3s @@ -1,42 +1,3 @@ -### Test if we can deal with output > 4 GB -46a318993dfc8e2afd71ff2bc6f605f1 - -** -### Test --tmpdir running full. bug #40733 was caused by this -parallel: Error: Output is incomplete. Cannot append to buffer file in $TMPDIR. Is the disk full? -parallel: Error: Change $TMPDIR with --tmpdir. -** -### Test Force outside the file handle limit, 2009-02-17 Gave fork error -parallel: Warning: Only enough file handles to run 506 jobs in parallel. -Raising ulimit -n or /etc/security/limits.conf may help. -Start -end -** -### Test of --retries on unreachable host -ssh: connect to host 4.3.2.1 port 22: Connection timed out -parallel: Warning: Could not figure out number of cpus on 4.3.2.1 (). Using 1. -echo 1 -1 -echo 2 -2 -** -### Test race condition on 8 CPU (my laptop) -1 -2 -3 -4 -5 -6 -7 -8 -9 -10 -** -### Test exit val - true -0 -** -### Test exit val - false -1 -** ### Test --halt-on-error 0 1 2 diff --git a/testsuite/wanted-results/parallel-local2 b/testsuite/wanted-results/parallel-local2 index 440fc80c..07ee806d 100644 --- a/testsuite/wanted-results/parallel-local2 +++ b/testsuite/wanted-results/parallel-local2 @@ -5,6 +5,5 @@ Less than 1 secs user time: OK 1 2 3 -### Test too slow spawning - TODO THIS CURRENTLY DOES NOT OVERLOAD -parallel: Warning: Only enough file handles to run 506 jobs in parallel. -Raising ulimit -n or /etc/security/limits.conf may help. +### Test too slow spawning +OK diff --git a/testsuite/wanted-results/parallel-polarhome b/testsuite/wanted-results/parallel-polarhome index 1ca73aad..5b617651 100644 --- a/testsuite/wanted-results/parallel-polarhome +++ b/testsuite/wanted-results/parallel-polarhome @@ -1,43 +1,61 @@ ### Tests on polarhome machines Setup on polarhome machines -copy_to_host freebsd.polarhome.com -copy_to_host solaris.polarhome.com -copy_to_host openbsd.polarhome.com -copy_to_host netbsd.polarhome.com -copy_to_host debian.polarhome.com -copy_to_host aix.polarhome.com -copy_to_host redhat.polarhome.com -copy_to_host hpux.polarhome.com -copy_to_host qnx.polarhome.com -copy_to_host openindiana.polarhome.com -copy_to_host suse.polarhome.com -copy_to_host mandriva.polarhome.com -copy_to_host ubuntu.polarhome.com -copy_to_host scosysv.polarhome.com -copy_to_host unixware.polarhome.com -copy_to_host dragonfly.polarhome.com -copy_to_host centos.polarhome.com -copy_to_host miros.polarhome.com -copy_to_host hurd.polarhome.com -### Run the test on polarhome machines +copy_and_test freebsd.polarhome.com +freebsd.polarhome.com ### Run the test on freebsd.polarhome.com freebsd.polarhome.com Works on freebsd.polarhome.com +copy_and_test solaris.polarhome.com +solaris.polarhome.com ### Run the test on solaris.polarhome.com solaris.polarhome.com Works on solaris.polarhome.com +copy_and_test openbsd.polarhome.com +openbsd.polarhome.com ### Run the test on openbsd.polarhome.com openbsd.polarhome.com Works on openbsd.polarhome.com +copy_and_test netbsd.polarhome.com +netbsd.polarhome.com ### Run the test on netbsd.polarhome.com netbsd.polarhome.com Works on netbsd.polarhome.com +copy_and_test debian.polarhome.com +debian.polarhome.com ### Run the test on debian.polarhome.com debian.polarhome.com Works on debian.polarhome.com +copy_and_test aix.polarhome.com +aix.polarhome.com ### Run the test on aix.polarhome.com aix.polarhome.com Works on aix.polarhome.com +copy_and_test redhat.polarhome.com +redhat.polarhome.com ### Run the test on redhat.polarhome.com redhat.polarhome.com Works on redhat.polarhome.com +copy_and_test hpux.polarhome.com +hpux.polarhome.com ### Run the test on hpux.polarhome.com hpux.polarhome.com Works on hpux.polarhome.com +copy_and_test qnx.polarhome.com +qnx.polarhome.com ### Run the test on qnx.polarhome.com qnx.polarhome.com Works on qnx.polarhome.com qnx.polarhome.com parallel: Warning: Cannot figure out number of CPU cores. Using 1. +copy_and_test openindiana.polarhome.com +openindiana.polarhome.com ### Run the test on openindiana.polarhome.com openindiana.polarhome.com Works on openindiana.polarhome.com openindiana.polarhome.com parallel: Warning: Cannot figure out number of CPU cores. Using 1. +copy_and_test suse.polarhome.com +suse.polarhome.com ### Run the test on suse.polarhome.com suse.polarhome.com Works on suse.polarhome.com +copy_and_test mandriva.polarhome.com +mandriva.polarhome.com ### Run the test on mandriva.polarhome.com mandriva.polarhome.com Works on mandriva.polarhome.com +copy_and_test ubuntu.polarhome.com +ubuntu.polarhome.com ### Run the test on ubuntu.polarhome.com ubuntu.polarhome.com Works on ubuntu.polarhome.com +copy_and_test scosysv.polarhome.com +scosysv.polarhome.com ### Run the test on scosysv.polarhome.com scosysv.polarhome.com Works on scosysv.polarhome.com +copy_and_test unixware.polarhome.com +unixware.polarhome.com ### Run the test on unixware.polarhome.com unixware.polarhome.com Works on unixware.polarhome.com +copy_and_test dragonfly.polarhome.com +dragonfly.polarhome.com ### Run the test on dragonfly.polarhome.com dragonfly.polarhome.com Works on dragonfly.polarhome.com +copy_and_test centos.polarhome.com +centos.polarhome.com ### Run the test on centos.polarhome.com centos.polarhome.com Works on centos.polarhome.com +copy_and_test miros.polarhome.com +miros.polarhome.com ### Run the test on miros.polarhome.com miros.polarhome.com Works on miros.polarhome.com +copy_and_test hurd.polarhome.com +hurd.polarhome.com ### Run the test on hurd.polarhome.com hurd.polarhome.com Works on hurd.polarhome.com