From 1cd9efc69c72b3a1bcdf98565f484bc96a7289ce Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Sun, 1 Jan 2017 12:42:52 +0100 Subject: [PATCH] parallel: --tmux(pane) --fg implemented. parallel_alternatives.pod: Split from parallel.pod. --- .gitignore | 13 +- 10seconds_install | 2 +- doc/parallel.latest.directive | 5 + doc/release_new_version | 29 +- packager/obs/home:tange/.osc/_apiurl | 1 + packager/obs/home:tange/.osc/_packages | 3 + packager/obs/home:tange/.osc/_project | 1 + packager/obs/home:tange/parallel/.osc/_apiurl | 1 + packager/obs/home:tange/parallel/.osc/_files | 6 + packager/obs/home:tange/parallel/.osc/_meta | 54 + .../home:tange/parallel/.osc/_osclib_version | 1 + .../obs/home:tange/parallel/.osc/_package | 1 + .../obs/home:tange/parallel/.osc/_project | 1 + .../home:tange/parallel/.osc/parallel.spec | 106 ++ .../obs/home:tange/parallel/parallel.spec | 5 +- src/Makefile.am | 52 +- src/niceload | 25 +- src/parallel | 22 +- src/parallel.pod | 887 +------------- src/parallel_alternatives.pod | 1023 +++++++++++++++++ src/sql | 2 +- testsuite/tests-to-run/parallel-local-0.3s.sh | 23 +- testsuite/tests-to-run/parallel-local-10s.sh | 70 ++ testsuite/tests-to-run/test13.sh | 23 +- testsuite/tests-to-run/test15.sh | 46 +- testsuite/wanted-results/parallel-local-0.3s | 11 + testsuite/wanted-results/parallel-local-10s | 81 ++ 27 files changed, 1520 insertions(+), 974 deletions(-) create mode 100644 doc/parallel.latest.directive create mode 100644 packager/obs/home:tange/.osc/_apiurl create mode 100644 packager/obs/home:tange/.osc/_packages create mode 100644 packager/obs/home:tange/.osc/_project create mode 100644 packager/obs/home:tange/parallel/.osc/_apiurl create mode 100644 packager/obs/home:tange/parallel/.osc/_files create mode 100644 packager/obs/home:tange/parallel/.osc/_meta create mode 100644 packager/obs/home:tange/parallel/.osc/_osclib_version create mode 100644 packager/obs/home:tange/parallel/.osc/_package create mode 100644 packager/obs/home:tange/parallel/.osc/_project create mode 100644 packager/obs/home:tange/parallel/.osc/parallel.spec create mode 100644 src/parallel_alternatives.pod diff --git a/.gitignore b/.gitignore index 0fa6ad36..939636e4 100644 --- a/.gitignore +++ b/.gitignore @@ -10,11 +10,8 @@ stamp-h1 actual-results tmp video -src/parallel.1 -src/parallel.html -src/sem.1 -src/sem.html -src/sql.1 -src/sql.html -src/niceload.1 -src/niceload.html +src/*.html +src/*.pdf +src/*.texi +src/*.1 +src/*.7 diff --git a/10seconds_install b/10seconds_install index 904a92a0..ccc4cdc8 100644 --- a/10seconds_install +++ b/10seconds_install @@ -93,7 +93,7 @@ else else mkdir -p $HOME/bin/; chmod 755 src/*; - cp src/parallel src/sem src/sql src/niceload $HOME/bin; + cp src/parallel src/env_parallel* src/sem src/sql src/niceload $HOME/bin; echo echo GNU $LATEST copied to $HOME/bin fi diff --git a/doc/parallel.latest.directive b/doc/parallel.latest.directive new file mode 100644 index 00000000..6799b651 --- /dev/null +++ b/doc/parallel.latest.directive @@ -0,0 +1,5 @@ +version: 1.2 +replace: true +directory: parallel +filename: parallel-latest.tar.bz2 +comment: New release diff --git a/doc/release_new_version b/doc/release_new_version index 3c44911f..7e04e1f7 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -4,7 +4,7 @@ Check that documentation is updated (compare to web): -Fixet for 20161122 +Fixet for 20161223 git diff last-release-commit Unmodified beta since last version => production Unmodified alpha since last version => beta @@ -144,8 +144,8 @@ TAG=MyTag echo "Released as $YYYYMMDD ('$TAG')." git commit -a -m "Released as $YYYYMMDD ('$TAG')" -git tag -s -m "Released as $YYYYMMDD ('$TAG')" $TAG -git tag -s -m "Released as $YYYYMMDD ('$TAG')" $YYYYMMDD +git tag -s -u 88888888 -m "Released as $YYYYMMDD ('$TAG')" $TAG +git tag -s -u 88888888 -m "Released as $YYYYMMDD ('$TAG')" $YYYYMMDD git push git push origin $TAG @@ -219,9 +219,9 @@ cc:Tim Cuthbertson , Ryoichiro Suzuki , Jesse Alama -Subject: GNU Parallel 20161222 ('Castro') released <<[stable]>> +Subject: GNU Parallel 20170122 ('George Michael') released <<[stable]>> -GNU Parallel 20161222 ('Castro') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/ +GNU Parallel 20170122 ('George Michael') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/ <> @@ -230,21 +230,10 @@ Haiku of the month: -- Ole Tange New in this release: -http://www.nature.com/articles/srep39259 -http://paperity.org/p/78557440/learning-string-distance-with-smoothing-for-ocr-spelling-correction -http://blog.genoglobe.com/2016/11/gnu-parallel.html -http://os.51cto.com/art/201612/524182.htm -http://tomkimpson.com/howto/gnuparallel/ -https://gist.github.com/Brainiarc7/24c966c8a001061ee86cc4bc05826bf4 -https://www.researchgate.net/publication/308186413_Determination_of_crystal_structures_of_proteins_of_unknown_identity_using_a_marathon_molecular_replacement_procedure_Structure_of_Stenotrophomonas_maltophilia_phosphate-binding_protein -https://www.msi.umn.edu/support/faq/how-can-i-use-gnu-parallel-run-lot-commands-parallel -https://cs.gmu.edu/~ygingold/singleimage/Decomposing%20Images%20into%20Layers%20via%20RGB-space%20Geometry%20(Tan%20et%20al%202016%20TOG)%20small.pdf -https://arxiv.org/pdf/1511.02895v2.pdf -http://www.tecmint.com/linux-image-conversion-tools/ -http://cohesity.com/blog/alls-fair-love-distributed-storage/ -http://microbiomejournal.biomedcentral.com/articles/10.1186/s40168-016-0208-8 -https://giorgos.sealabs.net/taco-bell-parallel-programming.html -http://www.tecmint.com/linux-image-conversion-tools/ +https://www2.eecs.berkeley.edu/Pubs/TechRpts/2016/EECS-2016-212.pdf +http://journals.plos.org/plosone/article?id=10.1371/journal.pone.0168456#references +http://people.inf.ethz.ch/trayteld/papers/aerial/aerial.pdf +https://blog.razrlele.com/p/1843 * <> diff --git a/packager/obs/home:tange/.osc/_apiurl b/packager/obs/home:tange/.osc/_apiurl new file mode 100644 index 00000000..e10ef33c --- /dev/null +++ b/packager/obs/home:tange/.osc/_apiurl @@ -0,0 +1 @@ +https://api.opensuse.org diff --git a/packager/obs/home:tange/.osc/_packages b/packager/obs/home:tange/.osc/_packages new file mode 100644 index 00000000..786e75e2 --- /dev/null +++ b/packager/obs/home:tange/.osc/_packages @@ -0,0 +1,3 @@ + + + \ No newline at end of file diff --git a/packager/obs/home:tange/.osc/_project b/packager/obs/home:tange/.osc/_project new file mode 100644 index 00000000..e1cd6997 --- /dev/null +++ b/packager/obs/home:tange/.osc/_project @@ -0,0 +1 @@ +home:tange diff --git a/packager/obs/home:tange/parallel/.osc/_apiurl b/packager/obs/home:tange/parallel/.osc/_apiurl new file mode 100644 index 00000000..e10ef33c --- /dev/null +++ b/packager/obs/home:tange/parallel/.osc/_apiurl @@ -0,0 +1 @@ +https://api.opensuse.org diff --git a/packager/obs/home:tange/parallel/.osc/_files b/packager/obs/home:tange/parallel/.osc/_files new file mode 100644 index 00000000..76872b7e --- /dev/null +++ b/packager/obs/home:tange/parallel/.osc/_files @@ -0,0 +1,6 @@ + + + + + + diff --git a/packager/obs/home:tange/parallel/.osc/_meta b/packager/obs/home:tange/parallel/.osc/_meta new file mode 100644 index 00000000..6e08c658 --- /dev/null +++ b/packager/obs/home:tange/parallel/.osc/_meta @@ -0,0 +1,54 @@ + + GNU Parallel + = About GNU Parallel = + +GNU Parallel is a shell tool for executing jobs in parallel using one or more computers. A job is can be a single command or a small script that has to be run for each of the lines in the input. The typical input is a list of files, a list of hosts, a list of users, a list of URLs, or a list of tables. A job can also be a command that reads from a pipe. GNU Parallel can then split the input and pipe it into commands in parallel. + +If you use xargs and tee today you will find GNU Parallel very easy to use as GNU Parallel is written to have the same options as xargs. If you write loops in shell, you will find GNU Parallel may be able to replace most of the loops and make them run faster by running several jobs in parallel. GNU Parallel can even replace nested loops. + +GNU Parallel makes sure output from the commands is the same output as you would get had you run the commands sequentially. This makes it +possible to use output from GNU Parallel as input for other programs. + +You can find more about GNU Parallel at: http://www.gnu.org/s/parallel/ + +Watch the intro video on http://www.youtube.com/playlist?list=PL284C9FF2488BC6D1 or walk through the tutorial http://www.gnu.org/software/parallel/parallel_tutorial.html + +When using GNU Parallel for a publication please cite: + +O. Tange (2011): GNU Parallel - The Command-Line Power Tool, ;login: The USENIX Magazine, February 2011:42-47. + + += About GNU SQL = + +GNU sql aims to give a simple, unified interface for accessing databases through all the different databases' command line clients. So far the focus has been on giving a common way to specify login information (protocol, username, password, hostname, and port number), size (database and table size), and running queries. + +The database is addressed using a DBURL. If commands are left out you will get that database's interactive shell. + +When using GNU SQL for a publication please cite: + +O. Tange (2011): GNU SQL - A Command Line Tool for Accessing Different Databases Using DBURLs, ;login: The USENIX Magazine, April 2011:29-32. + + += About GNU Niceload = + +GNU niceload slows down a program when the computer load average (or other system activity) is above a certain limit. When the limit is reached the program will be suspended for some time. If the limit is a soft limit the program will be allowed to run for short amounts of time before being suspended again. If the limit is a hard limit the program will only be allowed to run when the system is below the limit. + + + + + + + + + + + + + + + + + + http://www.gnu.org/software/parallel/ + + diff --git a/packager/obs/home:tange/parallel/.osc/_osclib_version b/packager/obs/home:tange/parallel/.osc/_osclib_version new file mode 100644 index 00000000..d3827e75 --- /dev/null +++ b/packager/obs/home:tange/parallel/.osc/_osclib_version @@ -0,0 +1 @@ +1.0 diff --git a/packager/obs/home:tange/parallel/.osc/_package b/packager/obs/home:tange/parallel/.osc/_package new file mode 100644 index 00000000..2e456f30 --- /dev/null +++ b/packager/obs/home:tange/parallel/.osc/_package @@ -0,0 +1 @@ +parallel diff --git a/packager/obs/home:tange/parallel/.osc/_project b/packager/obs/home:tange/parallel/.osc/_project new file mode 100644 index 00000000..e1cd6997 --- /dev/null +++ b/packager/obs/home:tange/parallel/.osc/_project @@ -0,0 +1 @@ +home:tange diff --git a/packager/obs/home:tange/parallel/.osc/parallel.spec b/packager/obs/home:tange/parallel/.osc/parallel.spec new file mode 100644 index 00000000..86b680ac --- /dev/null +++ b/packager/obs/home:tange/parallel/.osc/parallel.spec @@ -0,0 +1,106 @@ +Summary: Shell tool for executing jobs in parallel +Name: parallel +Version: 20161222 +Release: 1.1 +License: GPL +Group: Productivity/File utilities +URL: ftp://ftp.gnu.org/gnu/parallel +Source0: %{name}_%{version}.tar.gz +BuildArch: noarch +BuildArchitectures: noarch +BuildRoot: %{_tmppath}/%{name}-%{version}-buildroot + +%description +GNU Parallel is a shell tool for executing jobs in parallel using one +or more computers. A job can be a single command or a small script +that has to be run for each of the lines in the input. The typical +input is a list of files, a list of hosts, a list of users, a list of +URLs, or a list of tables. A job can also be a command that reads from +a pipe. GNU Parallel can then split the input and pipe it into +commands in parallel. + +If you use xargs and tee today you will find GNU Parallel very easy to +use as GNU Parallel is written to have the same options as xargs. If +you write loops in shell, you will find GNU Parallel may be able to +replace most of the loops and make them run faster by running several +jobs in parallel. + +GNU Parallel makes sure output from the commands is the same output as +you would get had you run the commands sequentially. This makes it +possible to use output from GNU Parallel as input for other programs. + +For each line of input GNU Parallel will execute command with the line +as arguments. If no command is given, the line of input is +executed. Several lines will be run in parallel. GNU Parallel can +often be used as a substitute for xargs or cat | bash. + +%prep +if [ "${RPM_BUILD_ROOT}x" == "x" ]; then + echo "RPM_BUILD_ROOT empty, bad idea!" + exit 1 +fi +if [ "${RPM_BUILD_ROOT}" == "/" ]; then + echo "RPM_BUILD_ROOT is set to "/", bad idea!" + exit 1 +fi +%setup -q + +%build +./configure +make + +%install +rm -rf $RPM_BUILD_ROOT +make install prefix=$RPM_BUILD_ROOT%{_prefix} exec_prefix=$RPM_BUILD_ROOT%{_prefix} \ + datarootdir=$RPM_BUILD_ROOT%{_prefix} docdir=$RPM_BUILD_ROOT%{_docdir} \ + mandir=$RPM_BUILD_ROOT%{_mandir} + +rm $RPM_BUILD_ROOT%{_docdir}/parallel.html +rm $RPM_BUILD_ROOT%{_docdir}/env_parallel.html +rm $RPM_BUILD_ROOT%{_docdir}/parallel_tutorial.html +rm $RPM_BUILD_ROOT%{_docdir}/parallel_design.html +rm $RPM_BUILD_ROOT%{_docdir}/niceload.html +rm $RPM_BUILD_ROOT%{_docdir}/sem.html +rm $RPM_BUILD_ROOT%{_docdir}/sql.html +rm $RPM_BUILD_ROOT%{_docdir}/parcat.html +rm $RPM_BUILD_ROOT%{_docdir}/parallel.texi +rm $RPM_BUILD_ROOT%{_docdir}/env_parallel.texi +rm $RPM_BUILD_ROOT%{_docdir}/parallel_tutorial.texi +rm $RPM_BUILD_ROOT%{_docdir}/parallel_design.texi +rm $RPM_BUILD_ROOT%{_docdir}/niceload.texi +rm $RPM_BUILD_ROOT%{_docdir}/sem.texi +rm $RPM_BUILD_ROOT%{_docdir}/sql.texi +rm $RPM_BUILD_ROOT%{_docdir}/parcat.texi +rm $RPM_BUILD_ROOT%{_docdir}/parallel.pdf +rm $RPM_BUILD_ROOT%{_docdir}/env_parallel.pdf +rm $RPM_BUILD_ROOT%{_docdir}/parallel_tutorial.pdf +rm $RPM_BUILD_ROOT%{_docdir}/parallel_design.pdf +rm $RPM_BUILD_ROOT%{_docdir}/niceload.pdf +rm $RPM_BUILD_ROOT%{_docdir}/sem.pdf +rm $RPM_BUILD_ROOT%{_docdir}/sql.pdf +rm $RPM_BUILD_ROOT%{_docdir}/parcat.pdf + +%clean +rm -rf $RPM_BUILD_ROOT + +%files +%defattr(-,root,root,-) +/usr/bin/* +/usr/share/man/man1/* +/usr/share/man/man7/* +%doc README NEWS src/parallel.html src/env_parallel.html src/parallel_tutorial.html src/parallel_design.html src/sem.html src/sql.html src/parcat.html src/niceload.html src/parallel.texi src/env_parallel.texi src/parallel_tutorial.texi src/parallel_design.texi src/niceload.texi src/sem.texi src/sql.texi src/parcat.texi src/parallel.pdf src/env_parallel.pdf src/parallel_tutorial.pdf src/parallel_design.pdf src/niceload.pdf src/sem.pdf src/sql.pdf src/parcat.pdf + +%changelog +* Sat Jan 22 2011 Ole Tange +- Upgrade to 20110122 +* Wed Dec 22 2010 Ole Tange +- Upgrade to 20101222 +* Wed Sep 22 2010 Ole Tange +- Upgrade to 20100922 +* Mon Sep 06 2010 Ole Tange +- Upgrade to current git-version of source. Tested on build.opensuse.org +* Fri Aug 27 2010 Ole Tange +- Untested upgrade to current git-version of source. +* Sat Aug 08 2010 Markus Ammer +- Initial package setup. + diff --git a/packager/obs/home:tange/parallel/parallel.spec b/packager/obs/home:tange/parallel/parallel.spec index 86b680ac..97f8c928 100644 --- a/packager/obs/home:tange/parallel/parallel.spec +++ b/packager/obs/home:tange/parallel/parallel.spec @@ -59,6 +59,7 @@ rm $RPM_BUILD_ROOT%{_docdir}/parallel.html rm $RPM_BUILD_ROOT%{_docdir}/env_parallel.html rm $RPM_BUILD_ROOT%{_docdir}/parallel_tutorial.html rm $RPM_BUILD_ROOT%{_docdir}/parallel_design.html +rm $RPM_BUILD_ROOT%{_docdir}/parallel_alternatives.html rm $RPM_BUILD_ROOT%{_docdir}/niceload.html rm $RPM_BUILD_ROOT%{_docdir}/sem.html rm $RPM_BUILD_ROOT%{_docdir}/sql.html @@ -67,6 +68,7 @@ rm $RPM_BUILD_ROOT%{_docdir}/parallel.texi rm $RPM_BUILD_ROOT%{_docdir}/env_parallel.texi rm $RPM_BUILD_ROOT%{_docdir}/parallel_tutorial.texi rm $RPM_BUILD_ROOT%{_docdir}/parallel_design.texi +rm $RPM_BUILD_ROOT%{_docdir}/parallel_alternatives.texi rm $RPM_BUILD_ROOT%{_docdir}/niceload.texi rm $RPM_BUILD_ROOT%{_docdir}/sem.texi rm $RPM_BUILD_ROOT%{_docdir}/sql.texi @@ -75,6 +77,7 @@ rm $RPM_BUILD_ROOT%{_docdir}/parallel.pdf rm $RPM_BUILD_ROOT%{_docdir}/env_parallel.pdf rm $RPM_BUILD_ROOT%{_docdir}/parallel_tutorial.pdf rm $RPM_BUILD_ROOT%{_docdir}/parallel_design.pdf +rm $RPM_BUILD_ROOT%{_docdir}/parallel_alternatives.pdf rm $RPM_BUILD_ROOT%{_docdir}/niceload.pdf rm $RPM_BUILD_ROOT%{_docdir}/sem.pdf rm $RPM_BUILD_ROOT%{_docdir}/sql.pdf @@ -88,7 +91,7 @@ rm -rf $RPM_BUILD_ROOT /usr/bin/* /usr/share/man/man1/* /usr/share/man/man7/* -%doc README NEWS src/parallel.html src/env_parallel.html src/parallel_tutorial.html src/parallel_design.html src/sem.html src/sql.html src/parcat.html src/niceload.html src/parallel.texi src/env_parallel.texi src/parallel_tutorial.texi src/parallel_design.texi src/niceload.texi src/sem.texi src/sql.texi src/parcat.texi src/parallel.pdf src/env_parallel.pdf src/parallel_tutorial.pdf src/parallel_design.pdf src/niceload.pdf src/sem.pdf src/sql.pdf src/parcat.pdf +%doc README NEWS src/parallel.html src/env_parallel.html src/parallel_tutorial.html src/parallel_design.html src/parallel_alternatives.html src/sem.html src/sql.html src/parcat.html src/niceload.html src/parallel.texi src/env_parallel.texi src/parallel_tutorial.texi src/parallel_design.texi src/parallel_alternatives.texi src/niceload.texi src/sem.texi src/sql.texi src/parcat.texi src/parallel.pdf src/env_parallel.pdf src/parallel_tutorial.pdf src/parallel_design.pdf src/parallel_alternatives.pdf src/niceload.pdf src/sem.pdf src/sql.pdf src/parcat.pdf %changelog * Sat Jan 22 2011 Ole Tange diff --git a/src/Makefile.am b/src/Makefile.am index 6df36999..6f47b5da 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -8,13 +8,17 @@ install-exec-hook: if DOCUMENTATION man_MANS = parallel.1 env_parallel.1 sem.1 sql.1 niceload.1 \ - parallel_tutorial.7 parallel_design.7 parcat.1 + parallel_tutorial.7 parallel_design.7 parallel_alternatives.7 \ + parcat.1 doc_DATA = parallel.html env_parallel.html sem.html sql.html niceload.html \ - parallel_tutorial.html parallel_design.html parcat.html \ + parallel_tutorial.html parallel_design.html parallel_alternatives.html \ + parcat.html \ parallel.texi env_parallel.texi sem.texi sql.texi niceload.texi \ - parallel_tutorial.texi parallel_design.texi parcat.texi \ + parallel_tutorial.texi parallel_design.texi parallel_alternatives.texi \ + parcat.texi \ parallel.pdf env_parallel.pdf sem.pdf sql.pdf niceload.pdf \ - parallel_tutorial.pdf parallel_design.pdf parcat.pdf + parallel_tutorial.pdf parallel_design.pdf parallel_alternatives.pdf \ + parcat.pdf endif # Build documentation file if the tool to build exists. @@ -43,6 +47,13 @@ parallel_design.7: parallel_design.pod && mv $(srcdir)/parallel_design.7n $(srcdir)/parallel_design.7 \ || echo "Warning: pod2man not found. Using old parallel_design.7" + +parallel_alternatives.7: parallel_alternatives.pod + pod2man --release='$(PACKAGE_VERSION)' --center='$(PACKAGE_NAME)' \ + --section=7 $(srcdir)/parallel_alternatives.pod > $(srcdir)/parallel_alternatives.7n \ + && mv $(srcdir)/parallel_alternatives.7n $(srcdir)/parallel_alternatives.7 \ + || echo "Warning: pod2man not found. Using old parallel_alternatives.7" + sem.1: sem.pod pod2man --release='$(PACKAGE_VERSION)' --center='$(PACKAGE_NAME)' \ --section=1 $(srcdir)/sem.pod > $(srcdir)/sem.1n \ @@ -88,14 +99,21 @@ parallel_tutorial.html: parallel_tutorial.pod env_parallel.html rm -f $(srcdir)/pod2htm* # Depending on parallel_tutorial.html to avoid stupid pod2html race condition -parallel_design.html: parallel_design.pod parallel.html +parallel_design.html: parallel_design.pod parallel_tutorial.html pod2html --title "GNU Parallel design" $(srcdir)/parallel_design.pod > $(srcdir)/parallel_design.htmln \ && mv $(srcdir)/parallel_design.htmln $(srcdir)/parallel_design.html \ || echo "Warning: pod2html not found. Using old parallel_design.html" rm -f $(srcdir)/pod2htm* # Depending on parallel_design.html to avoid stupid pod2html race condition -sem.html: sem.pod parallel_design.html +parallel_alternatives.html: parallel_alternatives.pod parallel_design.html + pod2html --title "GNU Parallel alternatives" $(srcdir)/parallel_alternatives.pod > $(srcdir)/parallel_alternatives.htmln \ + && mv $(srcdir)/parallel_alternatives.htmln $(srcdir)/parallel_alternatives.html \ + || echo "Warning: pod2html not found. Using old parallel_alternatives.html" + rm -f $(srcdir)/pod2htm* + +# Depending on parallel_alternatives.html to avoid stupid pod2html race condition +sem.html: sem.pod parallel_alternatives.html pod2html --title "sem (GNU Parallel)" $(srcdir)/sem.pod > $(srcdir)/sem.htmln \ && mv $(srcdir)/sem.htmln $(srcdir)/sem.html \ || echo "Warning: pod2html not found. Using old sem.html" @@ -138,6 +156,10 @@ parallel_design.texi: parallel_design.pod pod2texi --output=$(srcdir)/parallel_design.texi $(srcdir)/parallel_design.pod \ || echo "Warning: pod2texi not found. Using old parallel_design.texi" +parallel_alternatives.texi: parallel_alternatives.pod + pod2texi --output=$(srcdir)/parallel_alternatives.texi $(srcdir)/parallel_alternatives.pod \ + || echo "Warning: pod2texi not found. Using old parallel_alternatives.texi" + sem.texi: sem.pod pod2texi --output=$(srcdir)/sem.texi $(srcdir)/sem.pod \ || echo "Warning: pod2texi not found. Using old sem.texi" @@ -170,6 +192,10 @@ parallel_design.pdf: parallel_design.pod pod2pdf --output-file $(srcdir)/parallel_design.pdf $(srcdir)/parallel_design.pod --title "GNU Parallel Design" \ || echo "Warning: pod2pdf not found. Using old parallel_design.pdf" +parallel_alternatives.pdf: parallel_alternatives.pod + pod2pdf --output-file $(srcdir)/parallel_alternatives.pdf $(srcdir)/parallel_alternatives.pod --title "GNU Parallel alternatives" \ + || echo "Warning: pod2pdf not found. Using old parallel_alternatives.pdf" + sem.pdf: sem.pod pod2pdf --output-file $(srcdir)/sem.pdf $(srcdir)/sem.pod --title "GNU sem" \ || echo "Warning: pod2pdf not found. Using old sem.pdf" @@ -190,17 +216,21 @@ sem: parallel ln -fs parallel sem DISTCLEANFILES = parallel.1 env_parallel.1 sem.1 sql.1 niceload.1 \ - parallel_tutorial.7 parallel_design.7 parcat.1 \ + parallel_tutorial.7 parallel_design.7 parallel_alternatives.7 \ + parcat.1 \ parallel.html env_parallel.html sem.html sql.html niceload.html \ - parallel_tutorial.html parallel_design.html parcat.html \ + parallel_tutorial.html parallel_design.html parallel_alternatives.html \ + parcat.html \ parallel.texi env_parallel.texi sem.texi sql.texi niceload.texi \ - parallel_tutorial.texi parallel_design.texi parcat.texi \ + parallel_tutorial.texi parallel_design.texi parallel_alternatives.texi \ + parcat.texi \ parallel.pdf env_parallel.pdf sem.pdf sql.pdf niceload.pdf \ - parallel_tutorial.pdf parallel_design.pdf parcat.pdf + parallel_tutorial.pdf parallel_design.pdf parallel_alternatives.pdf \ + parcat.pdf EXTRA_DIST = parallel sem sql niceload parcat env_parallel \ env_parallel.bash env_parallel.zsh env_parallel.fish env_parallel.ksh \ env_parallel.pdksh env_parallel.csh env_parallel.tcsh \ sem.pod parallel.pod env_parallel.pod niceload.pod parallel_tutorial.pod \ - parallel_design.pod \ + parallel_design.pod parallel_alternatives.pod \ $(DISTCLEANFILES) diff --git a/src/niceload b/src/niceload index 8726141e..5a70ff0d 100755 --- a/src/niceload +++ b/src/niceload @@ -24,7 +24,7 @@ use strict; use Getopt::Long; $Global::progname="niceload"; -$Global::version = 20161222; +$Global::version = 20161223; Getopt::Long::Configure("bundling","require_order"); get_options_from_array(\@ARGV) || die_usage(); if($opt::version) { @@ -61,6 +61,7 @@ if(not defined $opt::start_noswap) { $opt::start_noswap = $opt::noswap; } if(not defined $opt::run_noswap) { $opt::run_noswap = $opt::noswap; } if(defined $opt::load) { multiply_binary_prefix($opt::load); } +if(defined $opt::baseline) { collect_net_baseline(); } my $limit = Limit->new(); my $process = Process->new($opt::nice,@ARGV); @@ -299,6 +300,7 @@ sub get_options_from_array { "battery|B" => \$opt::battery, "net" => \$opt::net, "nethops=i" => \$opt::nethops, + "baseline" => \$opt::baseline, "nice|n=i" => \$opt::nice, "program|prg=s" => \@opt::prg, @@ -492,6 +494,11 @@ sub min { return $min; } +sub collect_net_baseline { + # Collect what a normal (unloaded) net connection looks line +} + + sub netsensor_script { # Script for --sensor when using --net my $hops = shift; @@ -500,14 +507,17 @@ sub netsensor_script { use Net::Ping; my $medtrc = MedianTraceroute->new(shift); - $medtrc->ping(); - $medtrc->ping(); + $medtrc->set_remedian($medtrc->ping()); + $medtrc->set_remedian($medtrc->ping()); while(1) { my $ms = $medtrc->ping(); my $m = $medtrc->remedian(); - # printf("%.2f %.2f med*1.5 %f\n",$m*1000,$ms*1000,$m*1.5 < $ms); - # 1 = median*1.5 < current latency - # 0 = median*1.5 > current latency + if($m*1.5 < $ms) { + # Bad 1 = median*1.5 < current latency + } else { + # OK 0 = median*1.5 > current latency + $medtrc->set_remedian($ms); + } printf("%d\n",$m*1.5 < $ms); sleep(1); } @@ -532,7 +542,7 @@ sub netsensor_script { die("Cannot traceroute to 8.8.8.8 and 89.233.43.71"); } } - my $p = Net::Ping->new("external"); + my $p = Net::Ping->new(); $p->hires(); return bless { @@ -552,7 +562,6 @@ sub netsensor_script { my ($ret, $duration, $ip) = $self->{'pinger'}->ping($self->{'host'}, 5.5); if($ret) { - $self->set_remedian($duration); return $duration; } } diff --git a/src/parallel b/src/parallel index ac8e39bf..7e255805 100755 --- a/src/parallel +++ b/src/parallel @@ -1259,7 +1259,7 @@ sub check_invalid_option_combinations { sub init_globals { # Defaults: - $Global::version = 20161222; + $Global::version = 20161223; $Global::progname = 'parallel'; $Global::infinity = 2**31; $Global::debug = 0; @@ -1443,7 +1443,9 @@ sub parse_semaphore { if(defined $opt::semaphore) { $Global::semaphore = 1; } if(defined $opt::semaphoretimeout) { $Global::semaphore = 1; } if(defined $opt::semaphorename) { $Global::semaphore = 1; } - if(defined $opt::fg) { $Global::semaphore = 1; } + if(defined $opt::fg and not $opt::tmux and not $opt::tmuxpane) { + $Global::semaphore = 1; + } if(defined $opt::bg) { $Global::semaphore = 1; } if(defined $opt::wait and not $opt::sqlmaster) { $Global::semaphore = 1; @ARGV = "true"; @@ -1507,6 +1509,9 @@ sub open_joblog { ::error("--resume and --resume-failed require --joblog or --results."); ::wait_and_exit(255); } + if(defined $opt::joblog and $opt::joblog =~ s/^\+//) { + $append = 1; + } if($opt::joblog and ($opt::sqlmaster @@ -1645,7 +1650,7 @@ sub find_compression_program { # parallel --shuf -j1 --joblog jl-m --arg-sep , parallel --compress-program \'{3}" "-{2}\' cat ::: 1gb '>'/dev/null , 1 2 3 , {1..3} , $multithread # sort -nk4 jl-? # 1-core: - # 2-cores: lz4 lzop zstd gzip bzip2 lzma xz clzip pigz lbzip2 pbzip2 lzip lrz plzip pxz pzstd + # 2-cores: pzstd zstd lz4 lzop pigz gzip lbzip2 pbzip2 lrz bzip2 lzma pxz plzip xz lzip clzip # 4-cores: # 8-cores: pzstd lz4 zstd pigz lzop lbzip2 pbzip2 gzip lzip lrz plzip pxz bzip2 lzma xz clzip # 16-cores: pzstd lz4 pigz lzop lbzip2 pbzip2 plzip lzip lrz pxz gzip lzma xz bzip2 @@ -8034,7 +8039,16 @@ sub print_dryrun_and_verbose { $ENV{'PARALLEL_TMUX'} ||= "tmux"; if(not $tmuxsocket) { $tmuxsocket = ::tmpname("tms"); - ::status("See output with: $ENV{'PARALLEL_TMUX'} -S $tmuxsocket attach"); + if($opt::fg) { + if(not fork) { + # Run tmux in the foreground + # Give it 1 second to get started + `sleep 1;$ENV{'PARALLEL_TMUX'} -S $tmuxsocket attach`; + exit; + } + } else { + ::status("See output with: $ENV{'PARALLEL_TMUX'} -S $tmuxsocket attach"); + } } $tmux = "sh -c '". $ENV{'PARALLEL_TMUX'}." -S $tmuxsocket new-session -s p$$ -d \"sleep .2\" >/dev/null 2>&1';" . diff --git a/src/parallel.pod b/src/parallel.pod index 365314c3..037e8f79 100644 --- a/src/parallel.pod +++ b/src/parallel.pod @@ -21,6 +21,10 @@ B<#!/usr/bin/parallel> --shebang [options] [I [arguments]] =head1 DESCRIPTION +STOP! + +Read the B below if you are new to GNU B. + GNU B is a shell tool for executing jobs in parallel using one or more computers. A job can be a single command or a small script that has to be run for each of the lines in the input. The @@ -49,8 +53,9 @@ often be used as a substitute for B or B. Start by watching the intro videos for a quick introduction: http://www.youtube.com/playlist?list=PL284C9FF2488BC6D1 -Then look at the Bs after the list of B. That will -give you an idea of what GNU B is capable of. +Then look at the Bs after the list of B (Use +B). That will give you an idea of what +GNU B is capable of. Then spend an hour walking through the tutorial (B). Your command line will love you for it. @@ -486,9 +491,10 @@ EUR, you should feel free to use B<--will-cite> in scripts. =item B<--block-size> I Size of block in bytes to read at a time. The I can be postfixed -with K, M, G, T, P, k, m, g, t, or p which would multiply the size -with 1024, 1048576, 1073741824, 1099511627776, 1125899906842624, 1000, -1000000, 1000000000, 1000000000000, or 1000000000000000, respectively. +with K, M, G, T, P, E, k, m, g, t, p, or e which would multiply the +size with 1024, 1048576, 1073741824, 1099511627776, 1125899906842624, +1152921504606846976, 1000, 1000000, 1000000000, 1000000000000, +1000000000000000, or 1000000000000000000 respectively. GNU B tries to meet the block size but can be off by the length of one record. For performance reasons I should be bigger @@ -1442,9 +1448,9 @@ it to the command. Only used with B<--pipe>. -=item B<--results> I (alpha testing) +=item B<--results> I (beta testing) -=item B<--res> I (alpha testing) +=item B<--res> I (beta testing) Save the output into files. @@ -1670,9 +1676,9 @@ B<--return> is ignored when used with B<--sshlogin :> or when not used with B<--sshlogin>. -=item B<--round-robin> (alpha testing) +=item B<--round-robin> (beta testing) -=item B<--round> (alpha testing) +=item B<--round> (beta testing) Normally B<--pipe> will give a single block to each instance of the command. With B<--round-robin> all blocks will at random be written to @@ -1894,7 +1900,7 @@ when called with B<--shebang>). Use B<--sqlmaster> instead. -=item B<--sqlmaster> I (alpha testing) +=item B<--sqlmaster> I (beta testing) Submit jobs via SQL server. I must point to a table, which will contain the same information as B<--joblog>, the values from the input @@ -1933,12 +1939,12 @@ It can also be an alias from ~/.sql/aliases: :myalias mysql:///mydb/paralleljobs -=item B<--sqlandworker> I (alpha testing) +=item B<--sqlandworker> I (beta testing) Shorthand for: B<--sqlmaster> I B<--sqlworker> I. -=item B<--sqlworker> I (alpha testing) +=item B<--sqlworker> I (beta testing) Execute jobs via SQL server. Read the input sources variables from the table pointed to by I. The I on the command line @@ -2338,7 +2344,7 @@ To see the difference try: I can contain GNU B's replacement strings. -=item B<--wait> (alpha testing) +=item B<--wait> (beta testing) Wait for all commands to complete. @@ -3476,6 +3482,20 @@ can read into R or LibreCalc, then you can use B<--results>: > print(mydf[2,]) > write(as.character(mydf[2,c("Stdout")]),'') + +=head1 EXAMPLE: Use XML as input + +The show Aflyttet on Radio 24syv publishes a RSS feed with their audio +podcasts on: http://arkiv.radio24syv.dk/audiopodcast/channel/4466232 + +Using B you can extract the URLs for 2016 and download them +using GNU B: + + wget -O - http://arkiv.radio24syv.dk/audiopodcast/channel/4466232 | + xpath -e "//ancestor::pubDate[contains(text(),'2016')]/../enclosure/@url" | + parallel -u wget '{= s/ url="//; s/"//; =}' + + =head1 EXAMPLE: Run the same command 10 times If you want to run the same command with the same arguments 10 times @@ -4095,846 +4115,7 @@ failing job. =head1 DIFFERENCES BETWEEN GNU Parallel AND ALTERNATIVES -There are a lot programs with some of the functionality of GNU -B. GNU B strives to include the best of the -functionality without sacrificing ease of use. - - -=head2 SUMMARY TABLE - -The following features are in some of the comparable tools: - -Inputs - I1. Arguments can be read from stdin - I2. Arguments can be read from a file - I3. Arguments can be read from multiple files - I4. Arguments can be read from command line - I5. Arguments can be read from a table - I6. Arguments can be read from the same file using #! (shebang) - I7. Line oriented input as default (Quoting of special chars not needed) - -Manipulation of input - M1. Composed command - M2. Multiple arguments can fill up an execution line - M3. Arguments can be put anywhere in the execution line - M4. Multiple arguments can be put anywhere in the execution line - M5. Arguments can be replaced with context - M6. Input can be treated as the complete command line - -Outputs - O1. Grouping output so output from different jobs do not mix - O2. Send stderr (standard error) to stderr (standard error) - O3. Send stdout (standard output) to stdout (standard output) - O4. Order of output can be same as order of input - O5. Stdout only contains stdout (standard output) from the command - O6. Stderr only contains stderr (standard error) from the command - -Execution - E1. Running jobs in parallel - E2. List running jobs - E3. Finish running jobs, but do not start new jobs - E4. Number of running jobs can depend on number of cpus - E5. Finish running jobs, but do not start new jobs after first failure - E6. Number of running jobs can be adjusted while running - -Remote execution - R1. Jobs can be run on remote computers - R2. Basefiles can be transferred - R3. Argument files can be transferred - R4. Result files can be transferred - R5. Cleanup of transferred files - R6. No config files needed - R7. Do not run more than SSHD's MaxStartups can handle - R8. Configurable SSH command - R9. Retry if connection breaks occasionally - -Semaphore - S1. Possibility to work as a mutex - S2. Possibility to work as a counting semaphore - -Legend - - = no - x = not applicable - ID = yes - -As every new version of the programs are not tested the table may be -outdated. Please file a bug-report if you find errors (See REPORTING -BUGS). - -parallel: -I1 I2 I3 I4 I5 I6 I7 -M1 M2 M3 M4 M5 M6 -O1 O2 O3 O4 O5 O6 -E1 E2 E3 E4 E5 E6 -R1 R2 R3 R4 R5 R6 R7 R8 R9 -S1 S2 - -xargs: -I1 I2 - - - - - -- M2 M3 - - - -- O2 O3 - O5 O6 -E1 - - - - - -- - - - - x - - - -- - - -find -exec: -- - - x - x - -- M2 M3 - - - - -- O2 O3 O4 O5 O6 -- - - - - - - -- - - - - - - - - -x x - -make -j: -- - - - - - - -- - - - - - -O1 O2 O3 - x O6 -E1 - - - E5 - -- - - - - - - - - -- - - -ppss: -I1 I2 - - - - I7 -M1 - M3 - - M6 -O1 - - x - - -E1 E2 ?E3 E4 - - -R1 R2 R3 R4 - - ?R7 ? ? -- - - -pexec: -I1 I2 - I4 I5 - - -M1 - M3 - - M6 -O1 O2 O3 - O5 O6 -E1 - - E4 - E6 -R1 - - - - R6 - - - -S1 - - -xjobs, prll, dxargs, mdm/middelman, xapply, paexec, ladon, jobflow, -ClusterSSH: TODO - Please file a bug-report if you know what features -they support (See REPORTING BUGS). - - -=head2 DIFFERENCES BETWEEN xargs AND GNU Parallel - -B offers some of the same possibilities as GNU B. - -B deals badly with special characters (such as space, \, ' and -"). To see the problem try this: - - touch important_file - touch 'not important_file' - ls not* | xargs rm - mkdir -p "My brother's 12\" records" - ls | xargs rmdir - touch 'c:\windows\system32\clfs.sys' - echo 'c:\windows\system32\clfs.sys' | xargs ls -l - -You can specify B<-0>, but many input generators are not -optimized for using B as separator but are optimized for -B as separator. E.g B, B, B, B, B, -B, B, B (B<-0> and \0 instead of \n), B -(requires using B<-0>), B (requires using B<-print0>), B -(requires user to use B<-z> or B<-Z>), B (requires using B<-z>). - -GNU B's newline separation can be emulated with: - -B> - -B can run a given number of jobs in parallel, but has no -support for running number-of-cpu-cores jobs in parallel. - -B has no support for grouping the output, therefore output may -run together, e.g. the first half of a line is from one process and -the last half of the line is from another process. The example -B cannot be done reliably with B because of -this. To see this in action try: - - parallel perl -e '\$a=\"1{}\"x10000000\;print\ \$a,\"\\n\"' '>' {} \ - ::: a b c d e f - ls -l a b c d e f - parallel -kP4 -n1 grep 1 > out.par ::: a b c d e f - echo a b c d e f | xargs -P4 -n1 grep 1 > out.xargs-unbuf - echo a b c d e f | \ - xargs -P4 -n1 grep --line-buffered 1 > out.xargs-linebuf - echo a b c d e f | xargs -n1 grep 1 > out.xargs-serial - ls -l out* - md5sum out* - -B has no support for keeping the order of the output, therefore -if running jobs in parallel using B the output of the second -job cannot be postponed till the first job is done. - -B has no support for running jobs on remote computers. - -B has no support for context replace, so you will have to create the -arguments. - -If you use a replace string in B (B<-I>) you can not force -B to use more than one argument. - -Quoting in B works like B<-q> in GNU B. This means -composed commands and redirection require using B. - - ls | parallel "wc {} >{}.wc" - ls | parallel "echo {}; ls {}|wc" - -becomes (assuming you have 8 cores) - - ls | xargs -d "\n" -P8 -I {} bash -c "wc {} >{}.wc" - ls | xargs -d "\n" -P8 -I {} bash -c "echo {}; ls {}|wc" - - -=head2 DIFFERENCES BETWEEN find -exec AND GNU Parallel - -B offer some of the same possibilities as GNU B. - -B only works on files. So processing other input (such as -hosts or URLs) will require creating these inputs as files. B has no support for running commands in parallel. - - -=head2 DIFFERENCES BETWEEN make -j AND GNU Parallel - -B can run jobs in parallel, but requires a crafted Makefile -to do this. That results in extra quoting to get filename containing -newline to work correctly. - -B computes a dependency graph before running jobs. Jobs run -by GNU B does not depend on eachother. - -(Very early versions of GNU B were coincidently implemented -using B). - - -=head2 DIFFERENCES BETWEEN ppss AND GNU Parallel - -B is also a tool for running jobs in parallel. - -The output of B is status information and thus not useful for -using as input for another command. The output from the jobs are put -into files. - -The argument replace string ($ITEM) cannot be changed. Arguments must -be quoted - thus arguments containing special characters (space '"&!*) -may cause problems. More than one argument is not supported. File -names containing newlines are not processed correctly. When reading -input from a file null cannot be used as a terminator. B needs -to read the whole input file before starting any jobs. - -Output and status information is stored in ppss_dir and thus requires -cleanup when completed. If the dir is not removed before running -B again it may cause nothing to happen as B thinks the -task is already done. GNU B will normally not need cleaning -up if running locally and will only need cleaning up if stopped -abnormally and running remote (B<--cleanup> may not complete if -stopped abnormally). The example B would require extra -postprocessing if written using B. - -For remote systems PPSS requires 3 steps: config, deploy, and -start. GNU B only requires one step. - -=head3 EXAMPLES FROM ppss MANUAL - -Here are the examples from B's manual page with the equivalent -using GNU B: - -B<1> ./ppss.sh standalone -d /path/to/files -c 'gzip ' - -B<1> find /path/to/files -type f | parallel gzip - -B<2> ./ppss.sh standalone -d /path/to/files -c 'cp "$ITEM" /destination/dir ' - -B<2> find /path/to/files -type f | parallel cp {} /destination/dir - -B<3> ./ppss.sh standalone -f list-of-urls.txt -c 'wget -q ' - -B<3> parallel -a list-of-urls.txt wget -q - -B<4> ./ppss.sh standalone -f list-of-urls.txt -c 'wget -q "$ITEM"' - -B<4> parallel -a list-of-urls.txt wget -q {} - -B<5> ./ppss config -C config.cfg -c 'encode.sh ' -d /source/dir -m -192.168.1.100 -u ppss -k ppss-key.key -S ./encode.sh -n nodes.txt -o -/some/output/dir --upload --download ; ./ppss deploy -C config.cfg ; -./ppss start -C config - -B<5> # parallel does not use configs. If you want a different username put it in nodes.txt: user@hostname - -B<5> find source/dir -type f | parallel --sshloginfile nodes.txt --trc {.}.mp3 lame -a {} -o {.}.mp3 --preset standard --quiet - -B<6> ./ppss stop -C config.cfg - -B<6> killall -TERM parallel - -B<7> ./ppss pause -C config.cfg - -B<7> Press: CTRL-Z or killall -SIGTSTP parallel - -B<8> ./ppss continue -C config.cfg - -B<8> Enter: fg or killall -SIGCONT parallel - -B<9> ./ppss.sh status -C config.cfg - -B<9> killall -SIGUSR2 parallel - - -=head2 DIFFERENCES BETWEEN pexec AND GNU Parallel - -B is also a tool for running jobs in parallel. - -=head3 EXAMPLES FROM pexec MANUAL - -Here are the examples from B's info page with the equivalent -using GNU B: - -B<1> pexec -o sqrt-%s.dat -p "$(seq 10)" -e NUM -n 4 -c -- \ - 'echo "scale=10000;sqrt($NUM)" | bc' - -B<1> seq 10 | parallel -j4 'echo "scale=10000;sqrt({})" | bc > sqrt-{}.dat' - -B<2> pexec -p "$(ls myfiles*.ext)" -i %s -o %s.sort -- sort - -B<2> ls myfiles*.ext | parallel sort {} ">{}.sort" - -B<3> pexec -f image.list -n auto -e B -u star.log -c -- \ - 'fistar $B.fits -f 100 -F id,x,y,flux -o $B.star' - -B<3> parallel -a image.list \ - 'fistar {}.fits -f 100 -F id,x,y,flux -o {}.star' 2>star.log - -B<4> pexec -r *.png -e IMG -c -o - -- \ - 'convert $IMG ${IMG%.png}.jpeg ; "echo $IMG: done"' - -B<4> ls *.png | parallel 'convert {} {.}.jpeg; echo {}: done' - -B<5> pexec -r *.png -i %s -o %s.jpg -c 'pngtopnm | pnmtojpeg' - -B<5> ls *.png | parallel 'pngtopnm < {} | pnmtojpeg > {}.jpg' - -B<6> for p in *.png ; do echo ${p%.png} ; done | \ - pexec -f - -i %s.png -o %s.jpg -c 'pngtopnm | pnmtojpeg' - -B<6> ls *.png | parallel 'pngtopnm < {} | pnmtojpeg > {.}.jpg' - -B<7> LIST=$(for p in *.png ; do echo ${p%.png} ; done) - pexec -r $LIST -i %s.png -o %s.jpg -c 'pngtopnm | pnmtojpeg' - -B<7> ls *.png | parallel 'pngtopnm < {} | pnmtojpeg > {.}.jpg' - -B<8> pexec -n 8 -r *.jpg -y unix -e IMG -c \ - 'pexec -j -m blockread -d $IMG | \ - jpegtopnm | pnmscale 0.5 | pnmtojpeg | \ - pexec -j -m blockwrite -s th_$IMG' - -B<8> Combining GNU B and GNU B. - -B<8> ls *jpg | parallel -j8 'sem --id blockread cat {} | jpegtopnm |' \ - 'pnmscale 0.5 | pnmtojpeg | sem --id blockwrite cat > th_{}' - -B<8> If reading and writing is done to the same disk, this may be -faster as only one process will be either reading or writing: - -B<8> ls *jpg | parallel -j8 'sem --id diskio cat {} | jpegtopnm |' \ - 'pnmscale 0.5 | pnmtojpeg | sem --id diskio cat > th_{}' - - -=head2 DIFFERENCES BETWEEN xjobs AND GNU Parallel - -B is also a tool for running jobs in parallel. It only supports -running jobs on your local computer. - -B deals badly with special characters just like B. See -the section B. - -Here are the examples from B's man page with the equivalent -using GNU B: - -B<1> ls -1 *.zip | xjobs unzip - -B<1> ls *.zip | parallel unzip - -B<2> ls -1 *.zip | xjobs -n unzip - -B<2> ls *.zip | parallel unzip >/dev/null - -B<3> find . -name '*.bak' | xjobs gzip - -B<3> find . -name '*.bak' | parallel gzip - -B<4> ls -1 *.jar | sed 's/\(.*\)/\1 > \1.idx/' | xjobs jar tf - -B<4> ls *.jar | parallel jar tf {} '>' {}.idx - -B<5> xjobs -s script - -B<5> cat script | parallel - -B<6> mkfifo /var/run/my_named_pipe; -xjobs -s /var/run/my_named_pipe & -echo unzip 1.zip >> /var/run/my_named_pipe; -echo tar cf /backup/myhome.tar /home/me >> /var/run/my_named_pipe - -B<6> mkfifo /var/run/my_named_pipe; -cat /var/run/my_named_pipe | parallel & -echo unzip 1.zip >> /var/run/my_named_pipe; -echo tar cf /backup/myhome.tar /home/me >> /var/run/my_named_pipe - - -=head2 DIFFERENCES BETWEEN prll AND GNU Parallel - -B is also a tool for running jobs in parallel. It does not -support running jobs on remote computers. - -B encourages using BASH aliases and BASH functions instead of -scripts. GNU B supports scripts directly, functions if they -are exported using B, and aliases if using B. - -B generates a lot of status information on stderr (standard -error) which makes it harder to use the stderr (standard error) output -of the job directly as input for another program. - -Here is the example from B's man page with the equivalent -using GNU B: - - prll -s 'mogrify -flip $1' *.jpg - parallel mogrify -flip ::: *.jpg - - -=head2 DIFFERENCES BETWEEN dxargs AND GNU Parallel - -B is also a tool for running jobs in parallel. - -B does not deal well with more simultaneous jobs than SSHD's -MaxStartups. B is only built for remote run jobs, but does not -support transferring of files. - - -=head2 DIFFERENCES BETWEEN mdm/middleman AND GNU Parallel - -middleman(mdm) is also a tool for running jobs in parallel. - -Here are the shellscripts of http://mdm.berlios.de/usage.html ported -to GNU B: - - seq 19 | parallel buffon -o - | sort -n > result - cat files | parallel cmd - find dir -execdir sem cmd {} \; - - -=head2 DIFFERENCES BETWEEN xapply AND GNU Parallel - -B can run jobs in parallel on the local computer. - -Here are the examples from B's man page with the equivalent -using GNU B: - -B<1> xapply '(cd %1 && make all)' */ - -B<1> parallel 'cd {} && make all' ::: */ - -B<2> xapply -f 'diff %1 ../version5/%1' manifest | more - -B<2> parallel diff {} ../version5/{} < manifest | more - -B<3> xapply -p/dev/null -f 'diff %1 %2' manifest1 checklist1 - -B<3> parallel --link diff {1} {2} :::: manifest1 checklist1 - -B<4> xapply 'indent' *.c - -B<4> parallel indent ::: *.c - -B<5> find ~ksb/bin -type f ! -perm -111 -print | xapply -f -v 'chmod a+x' - - -B<5> find ~ksb/bin -type f ! -perm -111 -print | parallel -v chmod a+x - -B<6> find */ -... | fmt 960 1024 | xapply -f -i /dev/tty 'vi' - - -B<6> sh <(find */ -... | parallel -s 1024 echo vi) - -B<6> find */ -... | parallel -s 1024 -Xuj1 vi - -B<7> find ... | xapply -f -5 -i /dev/tty 'vi' - - - - - - -B<7> sh <(find ... |parallel -n5 echo vi) - -B<7> find ... |parallel -n5 -uj1 vi - -B<8> xapply -fn "" /etc/passwd - -B<8> parallel -k echo < /etc/passwd - -B<9> tr ':' '\012' < /etc/passwd | xapply -7 -nf 'chown %1 %6' - - - - - - - - -B<9> tr ':' '\012' < /etc/passwd | parallel -N7 chown {1} {6} - -B<10> xapply '[ -d %1/RCS ] || echo %1' */ - -B<10> parallel '[ -d {}/RCS ] || echo {}' ::: */ - -B<11> xapply -f '[ -f %1 ] && echo %1' List | ... - -B<11> parallel '[ -f {} ] && echo {}' < List | ... - - -=head2 DIFFERENCES BETWEEN AIX apply AND GNU Parallel - -B can build command lines based on a template and arguments - -very much like GNU B. B does not run jobs in -parallel. B does not use an argument separator (like B<:::>); -instead the template must be the first argument. - -Here are the examples from -https://www-01.ibm.com/support/knowledgecenter/ssw_aix_71/com.ibm.aix.cmds1/apply.htm - -1. To obtain results similar to those of the B command, enter: - - apply echo * - parallel echo ::: * - -2. To compare the file named B to the file named B, and the -file named B to the file named B, enter: - - apply -2 cmp a1 b1 a2 b2 - parallel -N2 cmp ::: a1 b1 a2 b2 - -3. To run the B command five times, enter: - - apply -0 who 1 2 3 4 5 - parallel -N0 who ::: 1 2 3 4 5 - -4. To link all files in the current directory to the directory -B, enter: - - apply 'ln %1 /usr/joe' * - parallel ln {} /usr/joe ::: * - - -=head2 DIFFERENCES BETWEEN paexec AND GNU Parallel - -B can run jobs in parallel on both the local and remote computers. - -B requires commands to print a blank line as the last -output. This means you will have to write a wrapper for most programs. - -B has a job dependency facility so a job can depend on another -job to be executed successfully. Sort of a poor-man's B. - -Here are the examples from B's example catalog with the equivalent -using GNU B: - -=over 1 - -=item 1_div_X_run: - - ../../paexec -s -l -c "`pwd`/1_div_X_cmd" -n +1 < sees it as a feature to have less features and in doing so it -also handles corner cases incorrectly. A lot of GNU B's code -is to handle corner cases correctly on every platform, so you will not -get a nasty surprise if a user for example saves a file called: I - -B's example showing how to deal with special characters fails on -special characters: - - echo "The Cure" > My\ brother\'s\ 12\"\ records - - ls | \ - map 'echo -n `gzip < "%" | wc -c`; echo -n '*100/'; wc -c < "%"' | bc - -It works with GNU B: - - ls | \ - parallel 'echo -n `gzip < {} | wc -c`; echo -n '*100/'; wc -c < {}' | bc - -And you can even get the file name prepended: - - ls | \ - parallel --tag '(echo -n `gzip < {} | wc -c`'*100/'; wc -c < {}) | bc' - -B has no support for grouping. So this gives the wrong results -without any warnings: - - parallel perl -e '\$a=\"1{}\"x10000000\;print\ \$a,\"\\n\"' '>' {} \ - ::: a b c d e f - ls -l a b c d e f - parallel -kP4 -n1 grep 1 > out.par ::: a b c d e f - map -p 4 'grep 1' a b c d e f > out.map-unbuf - map -p 4 'grep --line-buffered 1' a b c d e f > out.map-linebuf - map -p 1 'grep --line-buffered 1' a b c d e f > out.map-serial - ls -l out* - md5sum out* - -The documentation shows a workaround, but not only does that mix -stdout (standard output) with stderr (standard error) it also fails -completely for certain jobs (and may even be considered less readable): - - parallel echo -n {} ::: 1 2 3 - - map -p 4 'echo -n % 2>&1 | sed -e "s/^/$$:/"' 1 2 3 | sort | cut -f2- -d: - -Bs replacement strings (% %D %B %E) can be simulated in GNU -B by putting this in B<~/.parallel/config>: - - --rpl '%' - --rpl '%D $_=::shell_quote(::dirname($_));' - --rpl '%B s:.*/::;s:\.[^/.]+$::;' - --rpl '%E s:.*\.::' - -B cannot handle bundled options: B - -B does not have an argument separator on the command line, but -uses the first argument as command. This makes quoting harder which again -may affect readability. Compare: - - map -p 2 perl\\\ -ne\\\ \\\'/^\\\\S+\\\\s+\\\\S+\\\$/\\\ and\\\ print\\\ \\\$ARGV,\\\"\\\\n\\\"\\\' * - - parallel -q perl -ne '/^\S+\s+\S+$/ and print $ARGV,"\n"' ::: * - -B can do multiple arguments with context replace, but not without -context replace: - - parallel --xargs echo 'BEGIN{'{}'}END' ::: 1 2 3 - -B does not set exit value according to whether one of the jobs -failed: - - parallel false ::: 1 || echo Job failed - - map false 1 || echo Never run - -B requires Perl v5.10.0 making it harder to use on old systems. - -B has no way of using % in the command (GNU Parallel has -I to -specify another replacement string than B<{}>). - -By design B is option incompatible with B, it does not -have remote job execution, a structured way of saving results, -multiple input sources, progress indicator, configurable record -delimiter (only field delimiter), logging of jobs run with possibility -to resume, keeping the output in the same order as input, --pipe -processing, and dynamically timeouts. - - -=head2 DIFFERENCES BETWEEN ladon AND GNU Parallel - -B can run multiple jobs on files in parallel. - -B only works on files and the only way to specify files is -using a quoted glob string (such as \*.jpg). It is not possible to -list the files manually. - -As replacement strings it uses FULLPATH DIRNAME BASENAME EXT RELDIR RELPATH - -These can be simulated using GNU B by putting this in B<~/.parallel/config>: - - --rpl 'FULLPATH $_=::shell_quote($_);chomp($_=qx{readlink -f $_});' - --rpl 'DIRNAME $_=::shell_quote(::dirname($_));chomp($_=qx{readlink -f $_});' - --rpl 'BASENAME s:.*/::;s:\.[^/.]+$::;' - --rpl 'EXT s:.*\.::' - --rpl 'RELDIR $_=::shell_quote($_);chomp(($_,$c)=qx{readlink -f $_;pwd});s:\Q$c/\E::;$_=::dirname($_);' - --rpl 'RELPATH $_=::shell_quote($_);chomp(($_,$c)=qx{readlink -f $_;pwd});s:\Q$c/\E::;' - -B deals badly with filenames containing " and newline, and it fails for output larger than 200k: - - ladon '*' -- seq 36000 | wc - -=head3 EXAMPLES FROM ladon MANUAL - -It is assumed that the '--rpl's above are put in B<~/.parallel/config> -and that it is run under a shell that supports '**' globbing (such as B): - -B<1> ladon "**/*.txt" -- echo RELPATH - -B<1> parallel echo RELPATH ::: **/*.txt - -B<2> ladon "~/Documents/**/*.pdf" -- shasum FULLPATH >hashes.txt - -B<2> parallel shasum FULLPATH ::: ~/Documents/**/*.pdf >hashes.txt - -B<3> ladon -m thumbs/RELDIR "**/*.jpg" -- convert FULLPATH -thumbnail 100x100^ -gravity center -extent 100x100 thumbs/RELPATH - -B<3> parallel mkdir -p thumbs/RELDIR\; convert FULLPATH -thumbnail 100x100^ -gravity center -extent 100x100 thumbs/RELPATH ::: **/*.jpg - -B<4> ladon "~/Music/*.wav" -- lame -V 2 FULLPATH DIRNAME/BASENAME.mp3 - -B<4> parallel lame -V 2 FULLPATH DIRNAME/BASENAME.mp3 ::: ~/Music/*.wav - - -=head2 DIFFERENCES BETWEEN jobflow AND GNU Parallel - -B can run multiple jobs in parallel. - -Just like B output from B jobs running in parallel mix -together by default. B can buffer into files (placed in -/run/shm), but these are not cleaned up - not even if B dies -unexpectently. If the total output is big (in the order of RAM+swap) -it can cause the system to run out of memory. - -B gives no error if the command is unknown, and like B -redirection requires wrapping with B. - -B makes it possible to set ressource limits on the running -jobs. This can be emulated by GNU B using B's B: - - - jobflow -limits=mem=100M,cpu=3,fsize=20M,nofiles=300 myjob - - parallel 'ulimit -v 102400 -t 3 -f 204800 -n 300 myjob' - - -=head3 EXAMPLES FROM jobflow README - -B<1> cat things.list | jobflow -threads=8 -exec ./mytask {} - -B<1> cat things.list | parallel -j8 ./mytask {} - -B<2> seq 100 | jobflow -threads=100 -exec echo {} - -B<2> seq 100 | parallel -j100 echo {} - -B<3> cat urls.txt | jobflow -threads=32 -exec wget {} - -B<3> cat urls.txt | parallel -j32 wget {} - -B<4> find . -name '*.bmp' | jobflow -threads=8 -exec bmp2jpeg {.}.bmp {.}.jpg - -B<4> find . -name '*.bmp' | parallel -j8 bmp2jpeg {.}.bmp {.}.jpg - - -=head2 DIFFERENCES BETWEEN gargs AND GNU Parallel - -B can run multiple jobs in parallel. - -It caches output in memory. This causes it to be extremely slow when -the output is larger than the physical RAM, and can cause the system -to run out of memory. - -See more details on this in B. - - -Output to stderr (standard error) is changed if the command fails. - -Here are the two examples from B website. - -B<1> seq 12 -1 1 | gargs -p 4 -n 3 "sleep {0}; echo {1} {2}" - -B<1> seq 12 -1 1 | parallel -P 4 -n 3 "sleep {1}; echo {2} {3}" - -B<2> cat t.txt | gargs --sep "\s+" -p 2 "echo '{0}:{1}-{2}' full-line: \'{}\'" - -B<2> cat t.txt | parallel --colsep "\\s+" -P 2 "echo '{1}:{2}-{3}' full-line: \'{}\'" - - -=head2 DIFFERENCES BETWEEN orgalorg AND GNU Parallel - -B can run the same job on multiple machines. This is related -to B<--onall> and B<--nonall>. - -B supports entering the SSH password - provided it is the -same for all servers. GNU B advocates using B -instead, but it is possible to emulate B's behavior by -setting SSHPASS and by using B<--ssh "sshpass ssh">. - -To make the emulation easier, make a simple alias: - - alias par_emul="parallel -j0 --ssh 'sshpass ssh' --nonall --tag --linebuffer" - -If you want to supply a password run: - - SSHPASS=`ssh-askpass` - -or set the password directly: - - SSHPASS=P4$$w0rd! - -If the above is set up you can then do: - - orgalorg -o frontend1 -o frontend2 -p -C uptime - par_emul -S frontend1 -S frontend2 uptime - - orgalorg -o frontend1 -o frontend2 -p -C top -bid 1 - par_emul -S frontend1 -S frontend2 top -bid 1 - - orgalorg -o frontend1 -o frontend2 -p -er /tmp -n 'md5sum /tmp/bigfile' -S bigfile - par_emul -S frontend1 -S frontend2 --basefile bigfile --workdir /tmp md5sum /tmp/bigfile - -B has a progress indicator for the transferring of a -file. GNU B does not. - - -=head2 DIFFERENCES BETWEEN Rust parallel AND GNU Parallel - -Rust parallel implements a few features from GNU B, but -lacks many functions. All of these fail: - - # -q to protect quoted $ and space - parallel -q perl -e '$a=shift; print "$a"x10000000' ::: a b c - # Generation of combination of inputs - parallel echo {1} {2} ::: red green blue ::: S M L XL XXL - # Show what would be executed - parallel --dry-run echo ::: a - # Run different shell dialects - zsh -c 'parallel echo \={} ::: zsh && true' - csh -c 'parallel echo \$\{\} ::: shell && true' - bash -c 'parallel echo \$\({}\) ::: pwd && true' - -Rust parallel lacks B<::::>, B<--pipe>, and has no remote facilities. - -Rust parallel buffers in RAM like gargs. This can cause -death-by-swapping. See B. - - -=head2 DIFFERENCES BETWEEN ClusterSSH AND GNU Parallel - -ClusterSSH solves a different problem than GNU B. - -ClusterSSH opens a terminal window for each computer and using a -master window you can run the same command on all the computers. This -is typically used for administrating several computers that are almost -identical. - -GNU B runs the same (or different) commands with different -arguments in parallel possibly using remote computers to help -computing. If more than one computer is listed in B<-S> GNU B may -only use one of these (e.g. if there are 8 jobs to be run and one -computer has 8 cores). - -GNU B can be used as a poor-man's version of ClusterSSH: - -B +See: B =head1 BUGS diff --git a/src/parallel_alternatives.pod b/src/parallel_alternatives.pod new file mode 100644 index 00000000..1eab4738 --- /dev/null +++ b/src/parallel_alternatives.pod @@ -0,0 +1,1023 @@ +#!/usr/bin/perl -w + +=encoding utf8 + +=head1 NAME + +parallel_alternatives - Alternatives to GNU B + + +=head1 DIFFERENCES BETWEEN GNU Parallel AND ALTERNATIVES + +There are a lot programs with some of the functionality of GNU +B. GNU B strives to include the best of the +functionality without sacrificing ease of use. + + +=head2 SUMMARY TABLE + +The following features are in some of the comparable tools: + +Inputs + I1. Arguments can be read from stdin + I2. Arguments can be read from a file + I3. Arguments can be read from multiple files + I4. Arguments can be read from command line + I5. Arguments can be read from a table + I6. Arguments can be read from the same file using #! (shebang) + I7. Line oriented input as default (Quoting of special chars not needed) + +Manipulation of input + M1. Composed command + M2. Multiple arguments can fill up an execution line + M3. Arguments can be put anywhere in the execution line + M4. Multiple arguments can be put anywhere in the execution line + M5. Arguments can be replaced with context + M6. Input can be treated as the complete command line + +Outputs + O1. Grouping output so output from different jobs do not mix + O2. Send stderr (standard error) to stderr (standard error) + O3. Send stdout (standard output) to stdout (standard output) + O4. Order of output can be same as order of input + O5. Stdout only contains stdout (standard output) from the command + O6. Stderr only contains stderr (standard error) from the command + +Execution + E1. Running jobs in parallel + E2. List running jobs + E3. Finish running jobs, but do not start new jobs + E4. Number of running jobs can depend on number of cpus + E5. Finish running jobs, but do not start new jobs after first failure + E6. Number of running jobs can be adjusted while running + +Remote execution + R1. Jobs can be run on remote computers + R2. Basefiles can be transferred + R3. Argument files can be transferred + R4. Result files can be transferred + R5. Cleanup of transferred files + R6. No config files needed + R7. Do not run more than SSHD's MaxStartups can handle + R8. Configurable SSH command + R9. Retry if connection breaks occasionally + +Semaphore + S1. Possibility to work as a mutex + S2. Possibility to work as a counting semaphore + +Legend + - = no + x = not applicable + ID = yes + +As every new version of the programs are not tested the table may be +outdated. Please file a bug-report if you find errors (See REPORTING +BUGS). + +parallel: +I1 I2 I3 I4 I5 I6 I7 +M1 M2 M3 M4 M5 M6 +O1 O2 O3 O4 O5 O6 +E1 E2 E3 E4 E5 E6 +R1 R2 R3 R4 R5 R6 R7 R8 R9 +S1 S2 + +xargs: +I1 I2 - - - - - +- M2 M3 - - - +- O2 O3 - O5 O6 +E1 - - - - - +- - - - - x - - - +- - + +find -exec: +- - - x - x - +- M2 M3 - - - - +- O2 O3 O4 O5 O6 +- - - - - - - +- - - - - - - - - +x x + +make -j: +- - - - - - - +- - - - - - +O1 O2 O3 - x O6 +E1 - - - E5 - +- - - - - - - - - +- - + +ppss: +I1 I2 - - - - I7 +M1 - M3 - - M6 +O1 - - x - - +E1 E2 ?E3 E4 - - +R1 R2 R3 R4 - - ?R7 ? ? +- - + +pexec: +I1 I2 - I4 I5 - - +M1 - M3 - - M6 +O1 O2 O3 - O5 O6 +E1 - - E4 - E6 +R1 - - - - R6 - - - +S1 - + +xjobs, prll, dxargs, mdm/middelman, xapply, paexec, ladon, jobflow, +ClusterSSH: TODO - Please file a bug-report if you know what features +they support (See REPORTING BUGS). + + +=head2 DIFFERENCES BETWEEN xargs AND GNU Parallel + +B offers some of the same possibilities as GNU B. + +B deals badly with special characters (such as space, \, ' and +"). To see the problem try this: + + touch important_file + touch 'not important_file' + ls not* | xargs rm + mkdir -p "My brother's 12\" records" + ls | xargs rmdir + touch 'c:\windows\system32\clfs.sys' + echo 'c:\windows\system32\clfs.sys' | xargs ls -l + +You can specify B<-0>, but many input generators are not +optimized for using B as separator but are optimized for +B as separator. E.g B, B, B, B, B, +B, B, B (B<-0> and \0 instead of \n), B +(requires using B<-0>), B (requires using B<-print0>), B +(requires user to use B<-z> or B<-Z>), B (requires using B<-z>). + +GNU B's newline separation can be emulated with: + +B> + +B can run a given number of jobs in parallel, but has no +support for running number-of-cpu-cores jobs in parallel. + +B has no support for grouping the output, therefore output may +run together, e.g. the first half of a line is from one process and +the last half of the line is from another process. The example +B cannot be done reliably with B because of +this. To see this in action try: + + parallel perl -e '\$a=\"1{}\"x10000000\;print\ \$a,\"\\n\"' '>' {} \ + ::: a b c d e f + ls -l a b c d e f + parallel -kP4 -n1 grep 1 > out.par ::: a b c d e f + echo a b c d e f | xargs -P4 -n1 grep 1 > out.xargs-unbuf + echo a b c d e f | \ + xargs -P4 -n1 grep --line-buffered 1 > out.xargs-linebuf + echo a b c d e f | xargs -n1 grep 1 > out.xargs-serial + ls -l out* + md5sum out* + +B has no support for keeping the order of the output, therefore +if running jobs in parallel using B the output of the second +job cannot be postponed till the first job is done. + +B has no support for running jobs on remote computers. + +B has no support for context replace, so you will have to create the +arguments. + +If you use a replace string in B (B<-I>) you can not force +B to use more than one argument. + +Quoting in B works like B<-q> in GNU B. This means +composed commands and redirection require using B. + + ls | parallel "wc {} >{}.wc" + ls | parallel "echo {}; ls {}|wc" + +becomes (assuming you have 8 cores) + + ls | xargs -d "\n" -P8 -I {} bash -c "wc {} >{}.wc" + ls | xargs -d "\n" -P8 -I {} bash -c "echo {}; ls {}|wc" + + +=head2 DIFFERENCES BETWEEN find -exec AND GNU Parallel + +B offer some of the same possibilities as GNU B. + +B only works on files. So processing other input (such as +hosts or URLs) will require creating these inputs as files. B has no support for running commands in parallel. + + +=head2 DIFFERENCES BETWEEN make -j AND GNU Parallel + +B can run jobs in parallel, but requires a crafted Makefile +to do this. That results in extra quoting to get filename containing +newline to work correctly. + +B computes a dependency graph before running jobs. Jobs run +by GNU B does not depend on eachother. + +(Very early versions of GNU B were coincidently implemented +using B). + + +=head2 DIFFERENCES BETWEEN ppss AND GNU Parallel + +B is also a tool for running jobs in parallel. + +The output of B is status information and thus not useful for +using as input for another command. The output from the jobs are put +into files. + +The argument replace string ($ITEM) cannot be changed. Arguments must +be quoted - thus arguments containing special characters (space '"&!*) +may cause problems. More than one argument is not supported. File +names containing newlines are not processed correctly. When reading +input from a file null cannot be used as a terminator. B needs +to read the whole input file before starting any jobs. + +Output and status information is stored in ppss_dir and thus requires +cleanup when completed. If the dir is not removed before running +B again it may cause nothing to happen as B thinks the +task is already done. GNU B will normally not need cleaning +up if running locally and will only need cleaning up if stopped +abnormally and running remote (B<--cleanup> may not complete if +stopped abnormally). The example B would require extra +postprocessing if written using B. + +For remote systems PPSS requires 3 steps: config, deploy, and +start. GNU B only requires one step. + +=head3 EXAMPLES FROM ppss MANUAL + +Here are the examples from B's manual page with the equivalent +using GNU B: + +B<1> ./ppss.sh standalone -d /path/to/files -c 'gzip ' + +B<1> find /path/to/files -type f | parallel gzip + +B<2> ./ppss.sh standalone -d /path/to/files -c 'cp "$ITEM" /destination/dir ' + +B<2> find /path/to/files -type f | parallel cp {} /destination/dir + +B<3> ./ppss.sh standalone -f list-of-urls.txt -c 'wget -q ' + +B<3> parallel -a list-of-urls.txt wget -q + +B<4> ./ppss.sh standalone -f list-of-urls.txt -c 'wget -q "$ITEM"' + +B<4> parallel -a list-of-urls.txt wget -q {} + +B<5> ./ppss config -C config.cfg -c 'encode.sh ' -d /source/dir -m +192.168.1.100 -u ppss -k ppss-key.key -S ./encode.sh -n nodes.txt -o +/some/output/dir --upload --download ; ./ppss deploy -C config.cfg ; +./ppss start -C config + +B<5> # parallel does not use configs. If you want a different username put it in nodes.txt: user@hostname + +B<5> find source/dir -type f | parallel --sshloginfile nodes.txt --trc {.}.mp3 lame -a {} -o {.}.mp3 --preset standard --quiet + +B<6> ./ppss stop -C config.cfg + +B<6> killall -TERM parallel + +B<7> ./ppss pause -C config.cfg + +B<7> Press: CTRL-Z or killall -SIGTSTP parallel + +B<8> ./ppss continue -C config.cfg + +B<8> Enter: fg or killall -SIGCONT parallel + +B<9> ./ppss.sh status -C config.cfg + +B<9> killall -SIGUSR2 parallel + + +=head2 DIFFERENCES BETWEEN pexec AND GNU Parallel + +B is also a tool for running jobs in parallel. + +=head3 EXAMPLES FROM pexec MANUAL + +Here are the examples from B's info page with the equivalent +using GNU B: + +B<1> pexec -o sqrt-%s.dat -p "$(seq 10)" -e NUM -n 4 -c -- \ + 'echo "scale=10000;sqrt($NUM)" | bc' + +B<1> seq 10 | parallel -j4 'echo "scale=10000;sqrt({})" | bc > sqrt-{}.dat' + +B<2> pexec -p "$(ls myfiles*.ext)" -i %s -o %s.sort -- sort + +B<2> ls myfiles*.ext | parallel sort {} ">{}.sort" + +B<3> pexec -f image.list -n auto -e B -u star.log -c -- \ + 'fistar $B.fits -f 100 -F id,x,y,flux -o $B.star' + +B<3> parallel -a image.list \ + 'fistar {}.fits -f 100 -F id,x,y,flux -o {}.star' 2>star.log + +B<4> pexec -r *.png -e IMG -c -o - -- \ + 'convert $IMG ${IMG%.png}.jpeg ; "echo $IMG: done"' + +B<4> ls *.png | parallel 'convert {} {.}.jpeg; echo {}: done' + +B<5> pexec -r *.png -i %s -o %s.jpg -c 'pngtopnm | pnmtojpeg' + +B<5> ls *.png | parallel 'pngtopnm < {} | pnmtojpeg > {}.jpg' + +B<6> for p in *.png ; do echo ${p%.png} ; done | \ + pexec -f - -i %s.png -o %s.jpg -c 'pngtopnm | pnmtojpeg' + +B<6> ls *.png | parallel 'pngtopnm < {} | pnmtojpeg > {.}.jpg' + +B<7> LIST=$(for p in *.png ; do echo ${p%.png} ; done) + pexec -r $LIST -i %s.png -o %s.jpg -c 'pngtopnm | pnmtojpeg' + +B<7> ls *.png | parallel 'pngtopnm < {} | pnmtojpeg > {.}.jpg' + +B<8> pexec -n 8 -r *.jpg -y unix -e IMG -c \ + 'pexec -j -m blockread -d $IMG | \ + jpegtopnm | pnmscale 0.5 | pnmtojpeg | \ + pexec -j -m blockwrite -s th_$IMG' + +B<8> Combining GNU B and GNU B. + +B<8> ls *jpg | parallel -j8 'sem --id blockread cat {} | jpegtopnm |' \ + 'pnmscale 0.5 | pnmtojpeg | sem --id blockwrite cat > th_{}' + +B<8> If reading and writing is done to the same disk, this may be +faster as only one process will be either reading or writing: + +B<8> ls *jpg | parallel -j8 'sem --id diskio cat {} | jpegtopnm |' \ + 'pnmscale 0.5 | pnmtojpeg | sem --id diskio cat > th_{}' + + +=head2 DIFFERENCES BETWEEN xjobs AND GNU Parallel + +B is also a tool for running jobs in parallel. It only supports +running jobs on your local computer. + +B deals badly with special characters just like B. See +the section B. + +Here are the examples from B's man page with the equivalent +using GNU B: + +B<1> ls -1 *.zip | xjobs unzip + +B<1> ls *.zip | parallel unzip + +B<2> ls -1 *.zip | xjobs -n unzip + +B<2> ls *.zip | parallel unzip >/dev/null + +B<3> find . -name '*.bak' | xjobs gzip + +B<3> find . -name '*.bak' | parallel gzip + +B<4> ls -1 *.jar | sed 's/\(.*\)/\1 > \1.idx/' | xjobs jar tf + +B<4> ls *.jar | parallel jar tf {} '>' {}.idx + +B<5> xjobs -s script + +B<5> cat script | parallel + +B<6> mkfifo /var/run/my_named_pipe; +xjobs -s /var/run/my_named_pipe & +echo unzip 1.zip >> /var/run/my_named_pipe; +echo tar cf /backup/myhome.tar /home/me >> /var/run/my_named_pipe + +B<6> mkfifo /var/run/my_named_pipe; +cat /var/run/my_named_pipe | parallel & +echo unzip 1.zip >> /var/run/my_named_pipe; +echo tar cf /backup/myhome.tar /home/me >> /var/run/my_named_pipe + + +=head2 DIFFERENCES BETWEEN prll AND GNU Parallel + +B is also a tool for running jobs in parallel. It does not +support running jobs on remote computers. + +B encourages using BASH aliases and BASH functions instead of +scripts. GNU B supports scripts directly, functions if they +are exported using B, and aliases if using B. + +B generates a lot of status information on stderr (standard +error) which makes it harder to use the stderr (standard error) output +of the job directly as input for another program. + +Here is the example from B's man page with the equivalent +using GNU B: + + prll -s 'mogrify -flip $1' *.jpg + parallel mogrify -flip ::: *.jpg + + +=head2 DIFFERENCES BETWEEN dxargs AND GNU Parallel + +B is also a tool for running jobs in parallel. + +B does not deal well with more simultaneous jobs than SSHD's +MaxStartups. B is only built for remote run jobs, but does not +support transferring of files. + + +=head2 DIFFERENCES BETWEEN mdm/middleman AND GNU Parallel + +middleman(mdm) is also a tool for running jobs in parallel. + +Here are the shellscripts of http://mdm.berlios.de/usage.html ported +to GNU B: + + seq 19 | parallel buffon -o - | sort -n > result + cat files | parallel cmd + find dir -execdir sem cmd {} \; + + +=head2 DIFFERENCES BETWEEN xapply AND GNU Parallel + +B can run jobs in parallel on the local computer. + +Here are the examples from B's man page with the equivalent +using GNU B: + +B<1> xapply '(cd %1 && make all)' */ + +B<1> parallel 'cd {} && make all' ::: */ + +B<2> xapply -f 'diff %1 ../version5/%1' manifest | more + +B<2> parallel diff {} ../version5/{} < manifest | more + +B<3> xapply -p/dev/null -f 'diff %1 %2' manifest1 checklist1 + +B<3> parallel --link diff {1} {2} :::: manifest1 checklist1 + +B<4> xapply 'indent' *.c + +B<4> parallel indent ::: *.c + +B<5> find ~ksb/bin -type f ! -perm -111 -print | xapply -f -v 'chmod a+x' - + +B<5> find ~ksb/bin -type f ! -perm -111 -print | parallel -v chmod a+x + +B<6> find */ -... | fmt 960 1024 | xapply -f -i /dev/tty 'vi' - + +B<6> sh <(find */ -... | parallel -s 1024 echo vi) + +B<6> find */ -... | parallel -s 1024 -Xuj1 vi + +B<7> find ... | xapply -f -5 -i /dev/tty 'vi' - - - - - + +B<7> sh <(find ... |parallel -n5 echo vi) + +B<7> find ... |parallel -n5 -uj1 vi + +B<8> xapply -fn "" /etc/passwd + +B<8> parallel -k echo < /etc/passwd + +B<9> tr ':' '\012' < /etc/passwd | xapply -7 -nf 'chown %1 %6' - - - - - - - + +B<9> tr ':' '\012' < /etc/passwd | parallel -N7 chown {1} {6} + +B<10> xapply '[ -d %1/RCS ] || echo %1' */ + +B<10> parallel '[ -d {}/RCS ] || echo {}' ::: */ + +B<11> xapply -f '[ -f %1 ] && echo %1' List | ... + +B<11> parallel '[ -f {} ] && echo {}' < List | ... + + +=head2 DIFFERENCES BETWEEN AIX apply AND GNU Parallel + +B can build command lines based on a template and arguments - +very much like GNU B. B does not run jobs in +parallel. B does not use an argument separator (like B<:::>); +instead the template must be the first argument. + +Here are the examples from +https://www-01.ibm.com/support/knowledgecenter/ssw_aix_71/com.ibm.aix.cmds1/apply.htm + +1. To obtain results similar to those of the B command, enter: + + apply echo * + parallel echo ::: * + +2. To compare the file named B to the file named B, and the +file named B to the file named B, enter: + + apply -2 cmp a1 b1 a2 b2 + parallel -N2 cmp ::: a1 b1 a2 b2 + +3. To run the B command five times, enter: + + apply -0 who 1 2 3 4 5 + parallel -N0 who ::: 1 2 3 4 5 + +4. To link all files in the current directory to the directory +B, enter: + + apply 'ln %1 /usr/joe' * + parallel ln {} /usr/joe ::: * + + +=head2 DIFFERENCES BETWEEN paexec AND GNU Parallel + +B can run jobs in parallel on both the local and remote computers. + +B requires commands to print a blank line as the last +output. This means you will have to write a wrapper for most programs. + +B has a job dependency facility so a job can depend on another +job to be executed successfully. Sort of a poor-man's B. + +Here are the examples from B's example catalog with the equivalent +using GNU B: + +=over 1 + +=item 1_div_X_run: + + ../../paexec -s -l -c "`pwd`/1_div_X_cmd" -n +1 < sees it as a feature to have less features and in doing so it +also handles corner cases incorrectly. A lot of GNU B's code +is to handle corner cases correctly on every platform, so you will not +get a nasty surprise if a user for example saves a file called: I + +B's example showing how to deal with special characters fails on +special characters: + + echo "The Cure" > My\ brother\'s\ 12\"\ records + + ls | \ + map 'echo -n `gzip < "%" | wc -c`; echo -n '*100/'; wc -c < "%"' | bc + +It works with GNU B: + + ls | \ + parallel 'echo -n `gzip < {} | wc -c`; echo -n '*100/'; wc -c < {}' | bc + +And you can even get the file name prepended: + + ls | \ + parallel --tag '(echo -n `gzip < {} | wc -c`'*100/'; wc -c < {}) | bc' + +B has no support for grouping. So this gives the wrong results +without any warnings: + + parallel perl -e '\$a=\"1{}\"x10000000\;print\ \$a,\"\\n\"' '>' {} \ + ::: a b c d e f + ls -l a b c d e f + parallel -kP4 -n1 grep 1 > out.par ::: a b c d e f + map -p 4 'grep 1' a b c d e f > out.map-unbuf + map -p 4 'grep --line-buffered 1' a b c d e f > out.map-linebuf + map -p 1 'grep --line-buffered 1' a b c d e f > out.map-serial + ls -l out* + md5sum out* + +The documentation shows a workaround, but not only does that mix +stdout (standard output) with stderr (standard error) it also fails +completely for certain jobs (and may even be considered less readable): + + parallel echo -n {} ::: 1 2 3 + + map -p 4 'echo -n % 2>&1 | sed -e "s/^/$$:/"' 1 2 3 | sort | cut -f2- -d: + +Bs replacement strings (% %D %B %E) can be simulated in GNU +B by putting this in B<~/.parallel/config>: + + --rpl '%' + --rpl '%D $_=::shell_quote(::dirname($_));' + --rpl '%B s:.*/::;s:\.[^/.]+$::;' + --rpl '%E s:.*\.::' + +B cannot handle bundled options: B + +B does not have an argument separator on the command line, but +uses the first argument as command. This makes quoting harder which again +may affect readability. Compare: + + map -p 2 perl\\\ -ne\\\ \\\'/^\\\\S+\\\\s+\\\\S+\\\$/\\\ and\\\ print\\\ \\\$ARGV,\\\"\\\\n\\\"\\\' * + + parallel -q perl -ne '/^\S+\s+\S+$/ and print $ARGV,"\n"' ::: * + +B can do multiple arguments with context replace, but not without +context replace: + + parallel --xargs echo 'BEGIN{'{}'}END' ::: 1 2 3 + +B does not set exit value according to whether one of the jobs +failed: + + parallel false ::: 1 || echo Job failed + + map false 1 || echo Never run + +B requires Perl v5.10.0 making it harder to use on old systems. + +B has no way of using % in the command (GNU Parallel has -I to +specify another replacement string than B<{}>). + +By design B is option incompatible with B, it does not +have remote job execution, a structured way of saving results, +multiple input sources, progress indicator, configurable record +delimiter (only field delimiter), logging of jobs run with possibility +to resume, keeping the output in the same order as input, --pipe +processing, and dynamically timeouts. + + +=head2 DIFFERENCES BETWEEN ladon AND GNU Parallel + +B can run multiple jobs on files in parallel. + +B only works on files and the only way to specify files is +using a quoted glob string (such as \*.jpg). It is not possible to +list the files manually. + +As replacement strings it uses FULLPATH DIRNAME BASENAME EXT RELDIR RELPATH + +These can be simulated using GNU B by putting this in B<~/.parallel/config>: + + --rpl 'FULLPATH $_=::shell_quote($_);chomp($_=qx{readlink -f $_});' + --rpl 'DIRNAME $_=::shell_quote(::dirname($_));chomp($_=qx{readlink -f $_});' + --rpl 'BASENAME s:.*/::;s:\.[^/.]+$::;' + --rpl 'EXT s:.*\.::' + --rpl 'RELDIR $_=::shell_quote($_);chomp(($_,$c)=qx{readlink -f $_;pwd});s:\Q$c/\E::;$_=::dirname($_);' + --rpl 'RELPATH $_=::shell_quote($_);chomp(($_,$c)=qx{readlink -f $_;pwd});s:\Q$c/\E::;' + +B deals badly with filenames containing " and newline, and it fails for output larger than 200k: + + ladon '*' -- seq 36000 | wc + +=head3 EXAMPLES FROM ladon MANUAL + +It is assumed that the '--rpl's above are put in B<~/.parallel/config> +and that it is run under a shell that supports '**' globbing (such as B): + +B<1> ladon "**/*.txt" -- echo RELPATH + +B<1> parallel echo RELPATH ::: **/*.txt + +B<2> ladon "~/Documents/**/*.pdf" -- shasum FULLPATH >hashes.txt + +B<2> parallel shasum FULLPATH ::: ~/Documents/**/*.pdf >hashes.txt + +B<3> ladon -m thumbs/RELDIR "**/*.jpg" -- convert FULLPATH -thumbnail 100x100^ -gravity center -extent 100x100 thumbs/RELPATH + +B<3> parallel mkdir -p thumbs/RELDIR\; convert FULLPATH -thumbnail 100x100^ -gravity center -extent 100x100 thumbs/RELPATH ::: **/*.jpg + +B<4> ladon "~/Music/*.wav" -- lame -V 2 FULLPATH DIRNAME/BASENAME.mp3 + +B<4> parallel lame -V 2 FULLPATH DIRNAME/BASENAME.mp3 ::: ~/Music/*.wav + + +=head2 DIFFERENCES BETWEEN jobflow AND GNU Parallel + +B can run multiple jobs in parallel. + +Just like B output from B jobs running in parallel mix +together by default. B can buffer into files (placed in +/run/shm), but these are not cleaned up - not even if B dies +unexpectently. If the total output is big (in the order of RAM+swap) +it can cause the system to run out of memory. + +B gives no error if the command is unknown, and like B +redirection requires wrapping with B. + +B makes it possible to set ressource limits on the running +jobs. This can be emulated by GNU B using B's B: + + + jobflow -limits=mem=100M,cpu=3,fsize=20M,nofiles=300 myjob + + parallel 'ulimit -v 102400 -t 3 -f 204800 -n 300 myjob' + + +=head3 EXAMPLES FROM jobflow README + +B<1> cat things.list | jobflow -threads=8 -exec ./mytask {} + +B<1> cat things.list | parallel -j8 ./mytask {} + +B<2> seq 100 | jobflow -threads=100 -exec echo {} + +B<2> seq 100 | parallel -j100 echo {} + +B<3> cat urls.txt | jobflow -threads=32 -exec wget {} + +B<3> cat urls.txt | parallel -j32 wget {} + +B<4> find . -name '*.bmp' | jobflow -threads=8 -exec bmp2jpeg {.}.bmp {.}.jpg + +B<4> find . -name '*.bmp' | parallel -j8 bmp2jpeg {.}.bmp {.}.jpg + + +=head2 DIFFERENCES BETWEEN gargs AND GNU Parallel + +B can run multiple jobs in parallel. + +It caches output in memory. This causes it to be extremely slow when +the output is larger than the physical RAM, and can cause the system +to run out of memory. + +See more details on this in B. + + +Output to stderr (standard error) is changed if the command fails. + +Here are the two examples from B website. + +B<1> seq 12 -1 1 | gargs -p 4 -n 3 "sleep {0}; echo {1} {2}" + +B<1> seq 12 -1 1 | parallel -P 4 -n 3 "sleep {1}; echo {2} {3}" + +B<2> cat t.txt | gargs --sep "\s+" -p 2 "echo '{0}:{1}-{2}' full-line: \'{}\'" + +B<2> cat t.txt | parallel --colsep "\\s+" -P 2 "echo '{1}:{2}-{3}' full-line: \'{}\'" + + +=head2 DIFFERENCES BETWEEN orgalorg AND GNU Parallel + +B can run the same job on multiple machines. This is related +to B<--onall> and B<--nonall>. + +B supports entering the SSH password - provided it is the +same for all servers. GNU B advocates using B +instead, but it is possible to emulate B's behavior by +setting SSHPASS and by using B<--ssh "sshpass ssh">. + +To make the emulation easier, make a simple alias: + + alias par_emul="parallel -j0 --ssh 'sshpass ssh' --nonall --tag --linebuffer" + +If you want to supply a password run: + + SSHPASS=`ssh-askpass` + +or set the password directly: + + SSHPASS=P4$$w0rd! + +If the above is set up you can then do: + + orgalorg -o frontend1 -o frontend2 -p -C uptime + par_emul -S frontend1 -S frontend2 uptime + + orgalorg -o frontend1 -o frontend2 -p -C top -bid 1 + par_emul -S frontend1 -S frontend2 top -bid 1 + + orgalorg -o frontend1 -o frontend2 -p -er /tmp -n 'md5sum /tmp/bigfile' -S bigfile + par_emul -S frontend1 -S frontend2 --basefile bigfile --workdir /tmp md5sum /tmp/bigfile + +B has a progress indicator for the transferring of a +file. GNU B does not. + + +=head2 DIFFERENCES BETWEEN Rust parallel AND GNU Parallel + +Rust parallel focuses on speed. It is almost as fast as B. It +implements a few features from GNU B, but lacks many +functions. All these fail: + + # Show what would be executed + parallel --dry-run echo ::: a + # Read arguments from file + parallel -a file echo + # Changing the delimiter + parallel -d _ echo ::: a_b_c_ + +These do something different from GNU B + + # Read more arguments at a time -n + parallel -n 2 echo ::: 1 a 2 b + # -q to protect quoted $ and space + parallel -q perl -e '$a=shift; print "$a"x10000000' ::: a b c + # Generation of combination of inputs + parallel echo {1} {2} ::: red green blue ::: S M L XL XXL + # {= perl expression =} replacement string + parallel echo '{= s/new/old/ =}' ::: my.new your.new + # --pipe + seq 100000 | parallel --pipe wc + # linked arguments + parallel echo ::: S M L :::+ small medium large ::: R G B :::+ red green blue + # Run different shell dialects + zsh -c 'parallel echo \={} ::: zsh && true' + csh -c 'parallel echo \$\{\} ::: shell && true' + bash -c 'parallel echo \$\({}\) ::: pwd && true' + # Rust parallel does not start before the last argument is read + (seq 10; sleep 5; echo 2) | time parallel -j2 'sleep 2; echo' + tail -f /var/log/syslog | parallel echo + +Rust parallel has no remote facilities. + +It uses /tmp/parallel for tmp files and does not clean up if +terminated abrubtly. If another user on the system uses Rust parallel, +then /tmp/parallel will have the wrong permissions and Rust parallel +will fail. A malicious user can setup the right permissions and +symlink the output file to one of the user's files and next time the +user uses Rust parallel it will overwrite this file. + +If /tmp/parallel runs full during the run, Rust parallel does not +report this, but finishes with success - thereby risking data loss. + + +=head2 DIFFERENCES BETWEEN ClusterSSH AND GNU Parallel + +ClusterSSH solves a different problem than GNU B. + +ClusterSSH opens a terminal window for each computer and using a +master window you can run the same command on all the computers. This +is typically used for administrating several computers that are almost +identical. + +GNU B runs the same (or different) commands with different +arguments in parallel possibly using remote computers to help +computing. If more than one computer is listed in B<-S> GNU B may +only use one of these (e.g. if there are 8 jobs to be run and one +computer has 8 cores). + +GNU B can be used as a poor-man's version of ClusterSSH: + +B + + +=head1 AUTHOR + +When using GNU B for a publication please cite: + +O. Tange (2011): GNU Parallel - The Command-Line Power Tool, ;login: +The USENIX Magazine, February 2011:42-47. + +This helps funding further development; and it won't cost you a cent. +If you pay 10000 EUR you should feel free to use GNU Parallel without citing. + +Copyright (C) 2007-10-18 Ole Tange, http://ole.tange.dk + +Copyright (C) 2008,2009,2010 Ole Tange, http://ole.tange.dk + +Copyright (C) 2010,2011,2012,2013,2014,2015,2016 Ole Tange, +http://ole.tange.dk and Free Software Foundation, Inc. + +Parts of the manual concerning B compatibility is inspired by +the manual of B from GNU findutils 4.4.2. + + +=head1 LICENSE + +Copyright (C) 2007,2008,2009,2010,2011,2012,2013,2014,2015,2016 Free +Software Foundation, Inc. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3 of the License, or +at your option any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . + +=head2 Documentation license I + +Permission is granted to copy, distribute and/or modify this documentation +under the terms of the GNU Free Documentation License, Version 1.3 or +any later version published by the Free Software Foundation; with no +Invariant Sections, with no Front-Cover Texts, and with no Back-Cover +Texts. A copy of the license is included in the file fdl.txt. + +=head2 Documentation license II + +You are free: + +=over 9 + +=item B + +to copy, distribute and transmit the work + +=item B + +to adapt the work + +=back + +Under the following conditions: + +=over 9 + +=item B + +You must attribute the work in the manner specified by the author or +licensor (but not in any way that suggests that they endorse you or +your use of the work). + +=item B + +If you alter, transform, or build upon this work, you may distribute +the resulting work only under the same, similar or a compatible +license. + +=back + +With the understanding that: + +=over 9 + +=item B + +Any of the above conditions can be waived if you get permission from +the copyright holder. + +=item B + +Where the work or any of its elements is in the public domain under +applicable law, that status is in no way affected by the license. + +=item B + +In no way are any of the following rights affected by the license: + +=over 2 + +=item * + +Your fair dealing or fair use rights, or other applicable +copyright exceptions and limitations; + +=item * + +The author's moral rights; + +=item * + +Rights other persons may have either in the work itself or in +how the work is used, such as publicity or privacy rights. + +=back + +=back + +=over 9 + +=item B + +For any reuse or distribution, you must make clear to others the +license terms of this work. + +=back + +A copy of the full license is included in the file as cc-by-sa.txt. + + +=head1 DEPENDENCIES + +GNU B uses Perl, and the Perl modules Getopt::Long, +IPC::Open3, Symbol, IO::File, POSIX, and File::Temp. For remote usage +it also uses rsync with ssh. + + +=head1 SEE ALSO + +B(1), B(1), B(1), B(1), B(1), +B(1), B(1), B(1), B(1) + +=cut diff --git a/src/sql b/src/sql index 5b6f8552..55eb761a 100755 --- a/src/sql +++ b/src/sql @@ -566,7 +566,7 @@ $Global::Initfile && unlink $Global::Initfile; exit ($err); sub parse_options { - $Global::version = 20161222; + $Global::version = 20161223; $Global::progname = 'sql'; # This must be done first as this may exec myself diff --git a/testsuite/tests-to-run/parallel-local-0.3s.sh b/testsuite/tests-to-run/parallel-local-0.3s.sh index 2fcf818c..4b3ba28c 100644 --- a/testsuite/tests-to-run/parallel-local-0.3s.sh +++ b/testsuite/tests-to-run/parallel-local-0.3s.sh @@ -671,6 +671,27 @@ par_empty() { parallel echo ::: true } +par_empty_line() { + echo '### Test bug: empty line for | sh with -k' + (echo echo a ; echo ; echo echo b) | parallel -k +} + +par_append_joblog() { + echo '### can you append to a joblog using +' + parallel --joblog /tmp/parallel_append_joblog echo ::: 1 + parallel --joblog +/tmp/parallel_append_joblog echo ::: 1 + wc -l /tmp/parallel_append_joblog +} + +par_file_ending_in_newline() { + echo '### Hans found a bug giving unitialized variable' + echo >/tmp/parallel_f1 + echo >/tmp/parallel_f2' +' + echo /tmp/parallel_f1 /tmp/parallel_f2 | + stdout parallel -kv --delimiter ' ' gzip + rm /tmp/parallel_f* +} export -f $(compgen -A function | grep par_) -compgen -A function | grep par_ | sort | parallel -j6 --tag -k '{} 2>&1' +compgen -A function | grep par_ | sort | parallel -j6 --tag -k --joblog +/tmp/jl-`basename $0` '{} 2>&1' diff --git a/testsuite/tests-to-run/parallel-local-10s.sh b/testsuite/tests-to-run/parallel-local-10s.sh index 48692b4a..78d3d95e 100644 --- a/testsuite/tests-to-run/parallel-local-10s.sh +++ b/testsuite/tests-to-run/parallel-local-10s.sh @@ -3,6 +3,57 @@ # Simple jobs that never fails # Each should be taking 10-30s and be possible to run in parallel # I.e.: No race conditions, no logins + +par_interactive() { + echo '### Test -p --interactive' + cat >/tmp/parallel-script-for-expect <<_EOF +#!/bin/bash + +seq 1 3 | parallel -k -p "sleep 0.1; echo opt-p" +seq 1 3 | parallel -k --interactive "sleep 0.1; echo opt--interactive" +_EOF + chmod 755 /tmp/parallel-script-for-expect + + expect -b - <<_EOF +spawn /tmp/parallel-script-for-expect +expect "echo opt-p 1" +send "y\n" +expect "echo opt-p 2" +send "n\n" +expect "echo opt-p 3" +send "y\n" +expect "opt-p 1" +expect "opt-p 3" +expect "echo opt--interactive 1" +send "y\n" +expect "echo opt--interactive 2" +send "n\n" +expect "opt--interactive 1" +expect "echo opt--interactive 3" +send "y\n" +expect "opt--interactive 3" +_EOF + echo +} + +par_k() { + echo '### Test -k' + ulimit -n 50 + (echo "sleep 3; echo begin"; seq 1 30 | + parallel -kq echo "sleep 1; echo {}"; + echo "echo end") | stdout parallel -k -j0 +} + +par_sigterm() { + echo '### Test SIGTERM' + parallel -k -j20 sleep 3';' echo ::: {1..99} >/tmp/parallel$$ 2>&1 & + A=$! + sleep 5; kill -TERM $A + wait + sort /tmp/parallel$$ + rm /tmp/parallel$$ +} + par_pipepart_spawn() { echo '### bug #46214: Using --pipepart doesnt spawn multiple jobs in version 20150922' seq 1000000 > /tmp/num1000000; @@ -165,5 +216,24 @@ par_results_compress() { parallel --results /tmp/ged echo ::: 1 | wc -l } +par_kill_children_timeout() { + echo '### Test killing children with --timeout and exit value (failed if timed out)' + pstree $$ | grep sleep | grep -v anacron | grep -v screensave | wc; + doit() { + for i in `seq 100 120`; do + bash -c "(sleep $i)" & + sleep $i & + done; + wait; + echo No good; + } + export -f doit + parallel --timeout 3 doit ::: 1000000000 1000000001; + echo $?; + sleep 2; + pstree $$ | grep sleep | grep -v anacron | grep -v screensave | wc +} + + export -f $(compgen -A function | grep par_) compgen -A function | grep par_ | sort | parallel -j6 --tag -k '{} 2>&1' diff --git a/testsuite/tests-to-run/test13.sh b/testsuite/tests-to-run/test13.sh index 400d544b..28c1d9c7 100755 --- a/testsuite/tests-to-run/test13.sh +++ b/testsuite/tests-to-run/test13.sh @@ -1,23 +1,4 @@ #!/bin/bash -echo '### Test -k' -ulimit -n 50 -(echo "sleep 3; echo begin"; seq 1 30 | parallel -kq echo "sleep 1; echo {}"; echo "echo end") \ -| stdout parallel -k -j0 - -echo '### Test --keep-order' -(seq 0 2) | parallel --keep-order -j100% -S 1/:,2/parallel@parallel-server2 -q perl -e 'sleep 1;print "job{}\n";exit({})' - -echo '### Test --keeporder' -(seq 0 2) | parallel --keeporder -j100% -S 1/:,2/parallel@parallel-server2 -q perl -e 'sleep 1;print "job{}\n";exit({})' - -echo '### Test SIGTERM' -parallel -k -j20 sleep 3';' echo ::: {1..99} >/tmp/parallel$$ 2>&1 & -A=$! -sleep 5; kill -TERM $A -wait -sort /tmp/parallel$$ -rm /tmp/parallel$$ - -echo '### Test bug: empty line for | sh with -k' -(echo echo a ; echo ; echo echo b) | parallel -k +# moved to parallel-local-ssh8.sh +# parallel-local-0.3s.sh diff --git a/testsuite/tests-to-run/test15.sh b/testsuite/tests-to-run/test15.sh index cf527c7d..988d0696 100755 --- a/testsuite/tests-to-run/test15.sh +++ b/testsuite/tests-to-run/test15.sh @@ -4,43 +4,7 @@ #rm -f ~/.parallel/will-cite -echo '### Test -p --interactive' -cat >/tmp/parallel-script-for-expect <<_EOF -#!/bin/bash - -seq 1 3 | parallel -k -p "sleep 0.1; echo opt-p" -seq 1 3 | parallel -k --interactive "sleep 0.1; echo opt--interactive" -_EOF -chmod 755 /tmp/parallel-script-for-expect - -expect -b - <<_EOF -spawn /tmp/parallel-script-for-expect -expect "echo opt-p 1" -send "y\n" -expect "echo opt-p 2" -send "n\n" -expect "echo opt-p 3" -send "y\n" -expect "opt-p 1" -expect "opt-p 3" -expect "echo opt--interactive 1" -send "y\n" -expect "echo opt--interactive 2" -send "n\n" -expect "opt--interactive 1" -expect "echo opt--interactive 3" -send "y\n" -expect "opt--interactive 3" -_EOF -echo -cat <<'EOF' | parallel -vj0 -k -L1 -echo '### Test killing children with --timeout and exit value (failed if timed out)' - pstree $$ | grep sleep | grep -v anacron | grep -v screensave | wc; - parallel --timeout 3 'true {} ; for i in `seq 100 120`; do bash -c "(sleep $i)" & sleep $i & done; wait; echo No good' ::: 1000000000 1000000001 ; - echo $?; sleep 2; pstree $$ | grep sleep | grep -v anacron | grep -v screensave | wc -EOF - -cd input-files/test15 +cd input-files/test15 || cd ../input-files/test15 echo 'xargs Expect: 3 1' echo 3 | xargs -P 1 -n 1 -a files cat - @@ -64,12 +28,4 @@ chmod 755 /tmp/parallel-script-for-script2 echo via pseudotty | script -q -f -c /tmp/parallel-script-for-script2 /dev/null sleep 1 -echo '### Hans found a bug giving unitialized variable' -echo >/tmp/parallel_f1 -echo >/tmp/parallel_f2' -' -echo /tmp/parallel_f1 /tmp/parallel_f2 | stdout parallel -kv --delimiter ' ' gzip -rm /tmp/parallel_f* - - touch ~/.parallel/will-cite diff --git a/testsuite/wanted-results/parallel-local-0.3s b/testsuite/wanted-results/parallel-local-0.3s index 6d88b53b..a1d4b9be 100644 --- a/testsuite/wanted-results/parallel-local-0.3s +++ b/testsuite/wanted-results/parallel-local-0.3s @@ -1759,5 +1759,16 @@ echo '**' ** ### 1 .par file from --files expected 10 +par_append_joblog ### can you append to a joblog using + +par_append_joblog 1 +par_append_joblog 1 +par_append_joblog 3 /tmp/parallel_append_joblog par_empty bug #: par_empty true +par_empty_line ### Test bug: empty line for | sh with -k +par_empty_line a +par_empty_line b +par_file_ending_in_newline ### Hans found a bug giving unitialized variable +par_file_ending_in_newline gzip /tmp/parallel_f1 +par_file_ending_in_newline gzip /tmp/parallel_f2' +par_file_ending_in_newline ' diff --git a/testsuite/wanted-results/parallel-local-10s b/testsuite/wanted-results/parallel-local-10s index db1a25e2..b00fddff 100644 --- a/testsuite/wanted-results/parallel-local-10s +++ b/testsuite/wanted-results/parallel-local-10s @@ -257,6 +257,44 @@ par_halt_on_error 2 false false parallel: This job failed: par_halt_on_error 2 false false sleep 1;false par_halt_on_error 2 false false parallel: This job failed: par_halt_on_error 2 false false sleep 1;false +par_k ### Test -k +par_k parallel: Warning: Only enough file handles to run 9 jobs in parallel. +par_k parallel: Warning: Running 'parallel -j0 -N 9 --pipe parallel -j0' or +par_k parallel: Warning: raising ulimit -n or /etc/security/limits.conf may help. +par_k parallel: Warning: No more file handles. +par_k parallel: Warning: Raising ulimit -n or /etc/security/limits.conf may help. +par_k begin +par_k 1 +par_k 2 +par_k 3 +par_k 4 +par_k 5 +par_k 6 +par_k 7 +par_k 8 +par_k 9 +par_k 10 +par_k 11 +par_k 12 +par_k 13 +par_k 14 +par_k 15 +par_k 16 +par_k 17 +par_k 18 +par_k 19 +par_k 20 +par_k 21 +par_k 22 +par_k 23 +par_k 24 +par_k 25 +par_k 26 +par_k 27 +par_k 28 +par_k 29 +par_k 30 +par_k end par_k_linebuffer ### bug #47750: -k --line-buffer should give current job up to now par_k_linebuffer 10 1 par_k_linebuffer 10 2 @@ -888,6 +926,49 @@ par_results_csv 9,:,999.999,999.999,0,6,0,0,"echo 23 12",23,12,"23 12 par_results_csv ", par_round_robin_blocks bug #49664: --round-robin does not complete par_round_robin_blocks 2 +par_sigterm ### Test SIGTERM +par_sigterm 1 +par_sigterm 10 +par_sigterm 11 +par_sigterm 12 +par_sigterm 13 +par_sigterm 14 +par_sigterm 15 +par_sigterm 16 +par_sigterm 17 +par_sigterm 18 +par_sigterm 19 +par_sigterm 2 +par_sigterm 20 +par_sigterm 3 +par_sigterm 4 +par_sigterm 5 +par_sigterm 6 +par_sigterm 7 +par_sigterm 8 +par_sigterm 9 +par_sigterm parallel: SIGTERM received. No new jobs will be started. +par_sigterm parallel: Waiting for these 20 jobs to finish. Send SIGTERM again to stop now. +par_sigterm parallel: sleep 3; echo 1 +par_sigterm parallel: sleep 3; echo 10 +par_sigterm parallel: sleep 3; echo 11 +par_sigterm parallel: sleep 3; echo 12 +par_sigterm parallel: sleep 3; echo 13 +par_sigterm parallel: sleep 3; echo 14 +par_sigterm parallel: sleep 3; echo 15 +par_sigterm parallel: sleep 3; echo 16 +par_sigterm parallel: sleep 3; echo 17 +par_sigterm parallel: sleep 3; echo 18 +par_sigterm parallel: sleep 3; echo 19 +par_sigterm parallel: sleep 3; echo 2 +par_sigterm parallel: sleep 3; echo 20 +par_sigterm parallel: sleep 3; echo 3 +par_sigterm parallel: sleep 3; echo 4 +par_sigterm parallel: sleep 3; echo 5 +par_sigterm parallel: sleep 3; echo 6 +par_sigterm parallel: sleep 3; echo 7 +par_sigterm parallel: sleep 3; echo 8 +par_sigterm parallel: sleep 3; echo 9 par_testhalt ### testhalt --halt now,fail=0 par_testhalt parallel: This job failed: par_testhalt sleep 1.3; exit 1