From 259820f48cbd59aad0c932b44affe81b63139b59 Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Wed, 21 Jan 2015 22:24:23 +0100 Subject: [PATCH] Released as 20150122 (' (((:~{> Je Suis Charlie'). --- Makefile.am | 56 ++++++++ Makefile.in | 56 ++++++++ NEWS | 129 ++++++++++++++++++ README | 12 +- configure | 20 +-- configure.ac | 2 +- doc/release_new_version | 54 +++----- packager/obs/Makefile | 2 +- .../obs/home:tange/parallel/parallel.spec | 2 +- src/niceload | 2 +- src/parallel | 13 +- src/parallel.pod | 71 ++++++---- src/parallel_design.pod | 11 ++ src/sql | 2 +- testsuite/wanted-results/parallel-local12 | 10 +- testsuite/wanted-results/test15 | 20 +-- 16 files changed, 362 insertions(+), 100 deletions(-) diff --git a/Makefile.am b/Makefile.am index fbaa0729..d7dd8c2e 100644 --- a/Makefile.am +++ b/Makefile.am @@ -3,4 +3,60 @@ SUBDIRS = src test: cd testsuite; make 1 +upload: + eval `gpg-agent --daemon` +# make sure $YYYYMMDD is set + echo $(YYYYMMDD) | grep 20 + cp parallel-$(YYYYMMDD).tar.bz2 parallel-latest.tar.bz2 + cp doc/parallel.directive parallel-$(YYYYMMDD).tar.bz2.directive + perl -i -pe "s/20\d\d\d\d\d\d/$(YYYYMMDD)/" parallel-*.tar.*directive + gpg --clearsign --yes parallel-$(YYYYMMDD).tar.bz2.directive + + cp doc/parallel.latest.directive parallel-latest.tar.bz2.directive + perl -i -pe "s/20\d\d\d\d\d\d/$(YYYYMMDD)/" parallel-latest.tar.*directive + gpg --clearsign --yes parallel-latest.tar.bz2.directive + + (echo '#!/bin/bash'; \ + echo; \ + echo "# To check the signature run:"; \ + echo "# echo | gpg"; \ + echo "# gpg --auto-key-locate keyserver --keyserver-options auto-key-retrieve parallel-$(YYYYMMDD).tar.bz2.sig"; \ + echo; \ + echo "echo | gpg 2>/dev/null"; \ + echo 'gpg --auto-key-locate keyserver --keyserver-options auto-key-retrieve $$0'; \ + echo 'exit $$?' ; \ + echo; \ + gpg -ab -o - parallel-$(YYYYMMDD).tar.bz2; \ + ) > parallel-$(YYYYMMDD).tar.bz2.sig + + (echo '#!/bin/bash'; \ + echo; \ + echo "# To check the signature run:"; \ + echo "# echo | gpg"; \ + echo "# gpg --auto-key-locate keyserver --keyserver-options auto-key-retrieve parallel-latest.tar.bz2.sig"; \ + echo; \ + echo "echo | gpg 2>/dev/null"; \ + echo 'gpg --auto-key-locate keyserver --keyserver-options auto-key-retrieve $$0'; \ + echo 'exit $$?' ; \ + echo; \ + gpg -ab -o - parallel-$(YYYYMMDD).tar.bz2; \ + ) > parallel-latest.tar.bz2.sig + + gpg --auto-key-locate keyserver --keyserver-options auto-key-retrieve parallel-latest.tar.bz2.sig + gpg --auto-key-locate keyserver --keyserver-options auto-key-retrieve parallel-$(YYYYMMDD).tar.bz2.sig + + ../ftpsync/src/ftpsync parallel-$(YYYYMMDD).tar.bz2{,.sig,*asc} ftp://ftp-upload.gnu.org/incoming/ftp/ + ../ftpsync/src/ftpsync parallel-latest.tar.bz2{,.sig,*asc} ftp://ftp-upload.gnu.org/incoming/ftp/ + +# This can take 7 minutes + pushd /tmp; \ + rm -rf /tmp/parallel-$(YYYYMMDD)*; \ + while ! wget http://ftp.gnu.org/gnu/parallel/parallel-$(YYYYMMDD).tar.bz2 ; do sleep 2; done; \ + tar xjvf parallel-$(YYYYMMDD).tar.bz2; \ + cd parallel-$(YYYYMMDD); \ + ./configure; \ + make -j && sudo make -j install; \ + pushd; \ + sudo cp /usr/local/bin/parallel /usr/local/bin/parallel-$(YYYYMMDD) + EXTRA_DIST = CITATION diff --git a/Makefile.in b/Makefile.in index b6643e67..a9cc198f 100644 --- a/Makefile.in +++ b/Makefile.in @@ -738,6 +738,62 @@ uninstall-am: test: cd testsuite; make 1 +upload: + eval `gpg-agent --daemon` +# make sure $YYYYMMDD is set + echo $(YYYYMMDD) | grep 20 + cp parallel-$(YYYYMMDD).tar.bz2 parallel-latest.tar.bz2 + cp doc/parallel.directive parallel-$(YYYYMMDD).tar.bz2.directive + perl -i -pe "s/20\d\d\d\d\d\d/$(YYYYMMDD)/" parallel-*.tar.*directive + gpg --clearsign --yes parallel-$(YYYYMMDD).tar.bz2.directive + + cp doc/parallel.latest.directive parallel-latest.tar.bz2.directive + perl -i -pe "s/20\d\d\d\d\d\d/$(YYYYMMDD)/" parallel-latest.tar.*directive + gpg --clearsign --yes parallel-latest.tar.bz2.directive + + (echo '#!/bin/bash'; \ + echo; \ + echo "# To check the signature run:"; \ + echo "# echo | gpg"; \ + echo "# gpg --auto-key-locate keyserver --keyserver-options auto-key-retrieve parallel-$(YYYYMMDD).tar.bz2.sig"; \ + echo; \ + echo "echo | gpg 2>/dev/null"; \ + echo 'gpg --auto-key-locate keyserver --keyserver-options auto-key-retrieve $$0'; \ + echo 'exit $$?' ; \ + echo; \ + gpg -ab -o - parallel-$(YYYYMMDD).tar.bz2; \ + ) > parallel-$(YYYYMMDD).tar.bz2.sig + + (echo '#!/bin/bash'; \ + echo; \ + echo "# To check the signature run:"; \ + echo "# echo | gpg"; \ + echo "# gpg --auto-key-locate keyserver --keyserver-options auto-key-retrieve parallel-latest.tar.bz2.sig"; \ + echo; \ + echo "echo | gpg 2>/dev/null"; \ + echo 'gpg --auto-key-locate keyserver --keyserver-options auto-key-retrieve $$0'; \ + echo 'exit $$?' ; \ + echo; \ + gpg -ab -o - parallel-$(YYYYMMDD).tar.bz2; \ + ) > parallel-latest.tar.bz2.sig + + gpg --auto-key-locate keyserver --keyserver-options auto-key-retrieve parallel-latest.tar.bz2.sig + gpg --auto-key-locate keyserver --keyserver-options auto-key-retrieve parallel-$(YYYYMMDD).tar.bz2.sig + + ../ftpsync/src/ftpsync parallel-$(YYYYMMDD).tar.bz2{,.sig,*asc} ftp://ftp-upload.gnu.org/incoming/ftp/ + ../ftpsync/src/ftpsync parallel-latest.tar.bz2{,.sig,*asc} ftp://ftp-upload.gnu.org/incoming/ftp/ + +# This can take 7 minutes + pushd /tmp; \ + rm -rf /tmp/parallel-$(YYYYMMDD)*; \ + while ! wget http://ftp.gnu.org/gnu/parallel/parallel-$(YYYYMMDD).tar.bz2 ; do sleep 2; done; \ + tar xjvf parallel-$(YYYYMMDD).tar.bz2; \ + cd parallel-$(YYYYMMDD); \ + ./configure; \ + make -j && sudo make -j install; \ + pushd; \ + sudo cp /usr/local/bin/parallel /usr/local/bin/parallel-$(YYYYMMDD) + # Tell versions [3.59,3.63) of GNU make to not export all variables. # Otherwise a system limit (for SysV at least) may be exceeded. .NOEXPORT: diff --git a/NEWS b/NEWS index c0dd27a2..58fd464e 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,132 @@ +20150122 + +* Remote jobs now send stderr (standard error) to stderr (standard + error) instead of stdout (standard output). + +* Remote execution command is now packed using base64 encoded + bzip2. This means that big environments (app. 100 KB) can be + transferred. This change makes remote execution alpha quality. + +* --semaphoretimeout now takes a negative value. This means give up if + the semaphore is not gotten within the timeout. + +* --halt -1 and -2 now means halt if a job succeeds (so the opposite + of 1 and 2). + +* Bash's second fix of shellshock caused --env to break again when + exporting functions. This has been fixed again. + +* A semibig refactoring of big functions. All non-trivial functions + are now less than 100 lines. The refactoring makes this release beta + quality. + +* A description of the design decisions for GNU Parallel can be found + in 'man parallel_design'. + +* A bug fix in replacement strings cause rewrite of the replacement + function. This makes use of replacement strings alpha quality. + +* GNU Parallel was cited in: SlideToolkit: An Assistive Toolset for + the Histological Quantification of Whole Slide Images + http://www.plosone.org/article/info%3Adoi%2F10.1371%2Fjournal.pone.0110289 + +* GNU Parallel was cited in: Improving Energy and Performance with + Spintronics Caches in Multicore Systems + http://link.springer.com/chapter/10.1007/978-3-319-14313-2_24 + +* GNU Parallel was cited in: Parallel post-processing with MPI-Bash + http://dl.acm.org/citation.cfm?id=2691137 + +* GNU Parallel was cited in: Distinguishing cause from effect using + observational data: methods and benchmarks + http://arxiv-web3.library.cornell.edu/pdf/1412.3773.pdf + +* GNU Parallel was cited in: Data Science at the Command Line + http://cdn.oreillystatic.com/oreilly/booksamplers/9781491947852_sampler.pdf + +* GNU Parallel was cited in: Resolution of Genetic Map Expansion + Caused by Excess Heterozygosity in Plant Recombinant Inbred + Populations http://www.g3journal.org/content/4/10/1963.full.pdf + +* GNU Parallel was cited in: Bayesian Inference of Protein Structure + from Chemical Shift Data https://peerj.com/preprints/692v1.pdf + +* GNU Parallel was cited in: Clonal interference and Mullerʼs ratchet + in spatial habitats http://iopscience.iop.org/1478-3975/11/5/056003 + +* GNU Parallel was cited in: Web-scale content reuse detection + ftp://info.isi.edu/isi-pubs/tr-692.pdf + +* GNU Parallel was cited in: Computação Verde na Camada de Aplicação + https://linux.ime.usp.br/~tpaiva/mac0499/files/monografia.pdf + +* GNU Parallel was cited in: Frameshift alignment: statistics and + post-genomic applications + http://bioinformatics.oxfordjournals.org/content/30/24/3575.full.pdf+html + +* GNU Parallel was cited in: Towards Recommender Engineering Tools and + Experiments for Identifying Recommender Differences + http://elehack.net/research/thesis/mde-thesis.pdf + +* GNU Parallel was cited in: Speaker Environment Classification Using + Rhythm Metrics in Levantine Arabic Dialect + http://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumber=6923918 + +* GNU Parallel was cited in: A Formal Proof Of The Kepler Conjecture + http://arxiv.org/pdf/1501.02155.pdf + +* GNU Parallel was cited in: Allocation Policy Analysis for Cache + Coherence Protocols for STT-MRAM-based caches + http://conservancy.umn.edu/bitstream/handle/11299/169362/Nandkar_umn_0130M_15551.pdf + +* GNU Parallel was cited in: Using Trusted Platform Module for + securing virtual environment access in Cloud + http://dar.aucegypt.edu:8080/bitstream/handle/10526/4237/Using%20Trusted%20Platform%20Module%20for%20Securing%20Virtual%20Environment%20Access%20in%20Cloud.pdf + +* GNU Parallel was cited in: Weakly Supervised Learning for + Compositional Sentiment Recognition + http://computerlinguist.org/static/downloads/ma-thesis/Master_Thesis-Michael_Haas-Weakly_Supervised_Learning_for_Compositional_Sentiment_Recognition.pdf + +* GNU Parallel was cited in: Agility and Performance in Elastic + Distributed Storage http://dl.acm.org/citation.cfm?id=2668129 + +* GNU Parallel was used (unfortunately without citation) in: Large + Scale Author Name Disambiguation in Digital Libraries + http://semanticommunity.info/@api/deki/files/31523/P227_3546.pdf + +* GNU Parallel is used in BEDOPS: + https://media.readthedocs.org/pdf/bedops/v2p5p0/bedops.pdf + +* Job ad asking for GNU Parallel skills: + http://www.aplitrak.com/?adid=dXMzMTYxMjQuMzg0MTkuMTU1MEBtYW5wb3dlcmZveG5hLmFwbGl0cmFrLmNvbQ#sthash.kYCIhhty.KcoT6HnJ.dpbs + +* GNU Parallel: Open Source For You (OSFY) magazine, October 2013 + edition + http://www.shakthimaan.com/posts/2014/11/27/gnu-parallel/news.html + +* Multithreaded Encryption and Compression + http://www.krazyworks.com/multithreaded-encryption-and-compression/ + +* GNU Parallel on OSX and Linux + https://darknightelf.wordpress.com/2015/01/01/gnu-parallel-on-osx/ + +* How to build gnu parallel from source on Ubuntu PC? + http://blog.ovidiuparvu.com/build-gnu-parallel-source-ubuntu-pc/ + +* GNU parallel for openaddr-process-one + https://nelsonslog.wordpress.com/2015/01/09/gnu-parallel-for-openaddr-process-one/ + +* From SRA Project to FastQ http://danielecook.com/sra-project-fastq/ + +* Software Development Tooling + http://www.computer.org/csdl/mags/so/2014/06/mso2014060021.pdf + +* コマンドを並列に実行するGNU parallelがとても便利 + http://bicycle1885.hatenablog.com/entry/2014/08/10/143612 + +* Bug fixes and man page updates. + + 20141122 * Remote systems can be divided into hostgroups (e.g. web and db) by diff --git a/README b/README index da6e3509..e0ec12aa 100644 --- a/README +++ b/README @@ -40,9 +40,9 @@ document. Full installation of GNU Parallel is as simple as: - wget http://ftpmirror.gnu.org/parallel/parallel-20150109.tar.bz2 - bzip2 -dc parallel-20150109.tar.bz2 | tar xvf - - cd parallel-20150109 + wget http://ftpmirror.gnu.org/parallel/parallel-20150122.tar.bz2 + bzip2 -dc parallel-20150122.tar.bz2 | tar xvf - + cd parallel-20150122 ./configure && make && make install @@ -51,9 +51,9 @@ Full installation of GNU Parallel is as simple as: If you are not root you can add ~/bin to your path and install in ~/bin and ~/share: - wget http://ftpmirror.gnu.org/parallel/parallel-20150109.tar.bz2 - bzip2 -dc parallel-20150109.tar.bz2 | tar xvf - - cd parallel-20150109 + wget http://ftpmirror.gnu.org/parallel/parallel-20150122.tar.bz2 + bzip2 -dc parallel-20150122.tar.bz2 | tar xvf - + cd parallel-20150122 ./configure --prefix=$HOME && make && make install Or if your system lacks 'make' you can simply copy src/parallel diff --git a/configure b/configure index 545b5442..ee691c3b 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.69 for parallel 20150109. +# Generated by GNU Autoconf 2.69 for parallel 20150122. # # Report bugs to . # @@ -579,8 +579,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='parallel' PACKAGE_TARNAME='parallel' -PACKAGE_VERSION='20150109' -PACKAGE_STRING='parallel 20150109' +PACKAGE_VERSION='20150122' +PACKAGE_STRING='parallel 20150122' PACKAGE_BUGREPORT='bug-parallel@gnu.org' PACKAGE_URL='' @@ -1203,7 +1203,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures parallel 20150109 to adapt to many kinds of systems. +\`configure' configures parallel 20150122 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1269,7 +1269,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of parallel 20150109:";; + short | recursive ) echo "Configuration of parallel 20150122:";; esac cat <<\_ACEOF @@ -1345,7 +1345,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -parallel configure 20150109 +parallel configure 20150122 generated by GNU Autoconf 2.69 Copyright (C) 2012 Free Software Foundation, Inc. @@ -1362,7 +1362,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by parallel $as_me 20150109, which was +It was created by parallel $as_me 20150122, which was generated by GNU Autoconf 2.69. Invocation command line was $ $0 $@ @@ -2225,7 +2225,7 @@ fi # Define the identity of the package. PACKAGE='parallel' - VERSION='20150109' + VERSION='20150122' cat >>confdefs.h <<_ACEOF @@ -2867,7 +2867,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by parallel $as_me 20150109, which was +This file was extended by parallel $as_me 20150122, which was generated by GNU Autoconf 2.69. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -2929,7 +2929,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -parallel config.status 20150109 +parallel config.status 20150122 configured by $0, generated by GNU Autoconf 2.69, with options \\"\$ac_cs_config\\" diff --git a/configure.ac b/configure.ac index 5aff40c3..a61c1d6f 100644 --- a/configure.ac +++ b/configure.ac @@ -1,4 +1,4 @@ -AC_INIT([parallel], [20150109], [bug-parallel@gnu.org]) +AC_INIT([parallel], [20150122], [bug-parallel@gnu.org]) AM_INIT_AUTOMAKE([-Wall -Werror foreign]) AC_CONFIG_HEADERS([config.h]) AC_CONFIG_FILES([ diff --git a/doc/release_new_version b/doc/release_new_version index 4bb31a36..6ae08aaa 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -57,18 +57,15 @@ pushd == Upload == -eval `gpg-agent --daemon` +export YYYYMMDD=`yyyymmdd` +make upload -YYYYMMDD=`yyyymmdd` -cp parallel-$YYYYMMDD.tar.bz2 parallel-latest.tar.bz2 +# Only needed for alpha: +eval `gpg-agent --daemon` cp doc/parallel.directive parallel-$YYYYMMDD.tar.bz2.directive perl -i -pe "s/20\d\d\d\d\d\d/$YYYYMMDD/" parallel-*.tar.*directive gpg --clearsign --yes parallel-$YYYYMMDD.tar.bz2.directive -cp doc/parallel.latest.directive parallel-latest.tar.bz2.directive -perl -i -pe "s/20\d\d\d\d\d\d/$YYYYMMDD/" parallel-latest.tar.*directive -gpg --clearsign --yes parallel-latest.tar.bz2.directive - (echo '#!/bin/bash'; echo; echo "# To check the signature run:" @@ -82,30 +79,9 @@ gpg --clearsign --yes parallel-latest.tar.bz2.directive gpg -ab -o - parallel-$YYYYMMDD.tar.bz2 ) > parallel-$YYYYMMDD.tar.bz2.sig -(echo '#!/bin/bash'; - echo; - echo "# To check the signature run:" - echo "# echo | gpg" - echo "# gpg --auto-key-locate keyserver --keyserver-options auto-key-retrieve parallel-latest.tar.bz2.sig" - echo - echo "echo | gpg 2>/dev/null" - echo 'gpg --auto-key-locate keyserver --keyserver-options auto-key-retrieve $0' - echo 'exit $?' - echo - gpg -ab -o - parallel-$YYYYMMDD.tar.bz2 -) > parallel-latest.tar.bz2.sig - -gpg --auto-key-locate keyserver --keyserver-options auto-key-retrieve parallel-latest.tar.bz2.sig gpg --auto-key-locate keyserver --keyserver-options auto-key-retrieve parallel-$YYYYMMDD.tar.bz2.sig -#../ftpsync/src/ftpsync parallel-$YYYYMMDD.tar.bz2{,.sig,*asc} ftp://ftp-upload.gnu.org/incoming/alpha/ - -../ftpsync/src/ftpsync parallel-$YYYYMMDD.tar.bz2{,.sig,*asc} ftp://ftp-upload.gnu.org/incoming/ftp/ -../ftpsync/src/ftpsync parallel-latest.tar.bz2{,.sig,*asc} ftp://ftp-upload.gnu.org/incoming/ftp/ - -#echo put parallel-$YYYYMMDD.tar.bz2{,.sig,*asc} | ncftp ftp://ftp-upload.gnu.org/incoming/ftp/ -#echo put parallel-latest.tar.bz2{,.sig,*asc} | ncftp ftp://ftp-upload.gnu.org/incoming/ftp/ -#echo put parallel-$YYYYMMDD.tar.bz2{,.sig,*asc} | ncftp ftp://ftp-upload.gnu.org/incoming/alpha/ +../ftpsync/src/ftpsync parallel-$YYYYMMDD.tar.bz2{,.sig,*asc} ftp://ftp-upload.gnu.org/incoming/alpha/ == Update OpenSUSE build system == @@ -118,19 +94,20 @@ find home:tange/parallel/* -type f | grep -v parallel.spec | parallel -Xj1 osc r # This should not create new files osc up home:tange/parallel/ make -# Update versionsnummeret from 20110422-2 to YYYYMMDD-1 # Reload build-status: https://build.opensuse.org/package/show?package=parallel&project=home%3Atange # Check that one .deb (Debian 5.0) and one .rpm (CentOS) end with 'succeeded' == Download and test == +# Only needed for alpha + YYYYMMDD=`yyyymmdd` pushd /tmp rm -rf parallel-${YYYYMMDD}* # This can take 7 minutes -while ! wget http://ftp.gnu.org/gnu/parallel/parallel-$YYYYMMDD.tar.bz2 ; do sleep 2; done -#while ! wget http://alpha.gnu.org/gnu/parallel/parallel-$YYYYMMDD.tar.bz2 ; do sleep 2; done +#while ! wget http://ftp.gnu.org/gnu/parallel/parallel-$YYYYMMDD.tar.bz2 ; do sleep 2; done +while ! wget http://alpha.gnu.org/gnu/parallel/parallel-$YYYYMMDD.tar.bz2 ; do sleep 2; done tar xjvf parallel-$YYYYMMDD.tar.bz2 cd parallel-$YYYYMMDD ./configure @@ -228,19 +205,22 @@ cc:Tim Cuthbertson , Ryoichiro Suzuki , Jesse Alama -Subject: GNU Parallel 20150122 (' (((:~{> Hebdo Manila QZ8501') released +Subject: GNU Parallel 20150122 (' (((:~{> Je Suis Charlie') released -GNU Parallel 20150122 ('Hebdo Manila QZ8501') has been released. It is available for download at: http://ftp.gnu.org/gnu/parallel/ +GNU Parallel 20150122 (' (((:~{> Je Suis Charlie') has been released. It is available for download at: http://ftp.gnu.org/gnu/parallel/ Haiku of the month: -<<>> +If not enough time: +Run jobs using Parallel. +Keeps CPUs warm. + -- Ole Tange New in this release: * Remote jobs now send stderr (standard error) to stderr (standard error) instead of stdout (standard output). -* Remote execution command is now packed using base64 encoded bzip2. This means that big environments (app. 250 KB) can be transferred. This change makes remote execution alpha quality. +* Remote execution command is now packed using base64 encoded bzip2. This means that big environments (app. 100 KB) can be transferred. This change makes remote execution alpha quality. * --semaphoretimeout now takes a negative value. This means give up if the semaphore is not gotten within the timeout. @@ -252,6 +232,8 @@ New in this release: * A description of the design decisions for GNU Parallel can be found in 'man parallel_design'. +* A bug fix in replacement strings cause rewrite of the replacement function. This makes use of replacement strings alpha quality. + * GNU Parallel was cited in: SlideToolkit: An Assistive Toolset for the Histological Quantification of Whole Slide Images http://www.plosone.org/article/info%3Adoi%2F10.1371%2Fjournal.pone.0110289 * GNU Parallel was cited in: Improving Energy and Performance with Spintronics Caches in Multicore Systems http://link.springer.com/chapter/10.1007/978-3-319-14313-2_24 diff --git a/packager/obs/Makefile b/packager/obs/Makefile index 7dcb39bf..089268ff 100644 --- a/packager/obs/Makefile +++ b/packager/obs/Makefile @@ -2,7 +2,7 @@ all: cd ../debian/ && make osc rm home\:tange/parallel/parallel_*.dsc || true cp ../debian/parallel_*.dsc ../debian/parallel_*.tar.gz home\:tange/parallel/ - cp `ls ../../parallel-2*.tar.bz2|tail -n1` home\:tange/parallel/ + cp `ls ../../parallel-2*.tar.bz2|grep -v latest|tail -n1` home\:tange/parallel/ cd home\:tange/parallel/ && osc up cd home\:tange/parallel/ && parallel osc add ::: *.spec *.dsc *.tar.gz *.tar.bz2 && echo Src added OK || true cd home\:tange/parallel/ && osc ci -m "New release" diff --git a/packager/obs/home:tange/parallel/parallel.spec b/packager/obs/home:tange/parallel/parallel.spec index b6541780..d2b392c6 100644 --- a/packager/obs/home:tange/parallel/parallel.spec +++ b/packager/obs/home:tange/parallel/parallel.spec @@ -1,6 +1,6 @@ Summary: Shell tool for executing jobs in parallel Name: parallel -Version: 20141122 +Version: 20150122 Release: 1 License: GPL Group: Productivity/File utilities diff --git a/src/niceload b/src/niceload index d75fb81e..305bc29f 100755 --- a/src/niceload +++ b/src/niceload @@ -24,7 +24,7 @@ use strict; use Getopt::Long; $Global::progname="niceload"; -$Global::version = 20150109; +$Global::version = 20150122; Getopt::Long::Configure("bundling","require_order"); get_options_from_array(\@ARGV) || die_usage(); if($opt::version) { diff --git a/src/parallel b/src/parallel index ba9a49f2..bc28105d 100755 --- a/src/parallel +++ b/src/parallel @@ -953,7 +953,7 @@ sub parse_options { sub init_globals { # Defaults: - $Global::version = 20150115; + $Global::version = 20150122; $Global::progname = 'parallel'; $Global::infinity = 2**31; $Global::debug = 0; @@ -2933,7 +2933,8 @@ sub citation_notice { } else { ::status ("Academic tradition requires you to cite works you base your article on.\n", - "When using programs that use GNU Parallel to process data for publication please cite:\n", + "When using programs that use GNU Parallel to process data for publication\n", + "please cite:\n", "\n", " O. Tange (2011): GNU Parallel - The Command-Line Power Tool,\n", " ;login: The USENIX Magazine, February 2011:42-47.\n", @@ -2941,7 +2942,7 @@ sub citation_notice { "This helps funding further development; and it won't cost you a cent.\n", "If you pay 10000 EUR you should feel free to use GNU Parallel without citing.\n", "\n", - "To silence this citation notice run 'parallel --bibtex' once or use '--will-cite'.\n\n", + "Silence the citation notice: run 'parallel --bibtex' once or use '--will-cite'.\n\n", ); } } @@ -3996,7 +3997,7 @@ sub loadavg { $self->{'loadavg'} = $load - 1; ::debug("load", "New loadavg: ", $self->{'loadavg'},"\n"); } elsif ($load_out=~/average: (\d+.\d+)/) { - # Aix does not support instant load average + # AIX does not support instant load average # 04:11AM up 21 days, 12:55, 1 user, load average: 1.85, 1.57, 1.55 $self->{'loadavg'} = $1; } else { @@ -7926,8 +7927,8 @@ sub new { my $fhs = shift; for my $fh (@$fhs) { if(-t $fh) { - ::warning("Input is read from the terminal. ". - "Only experts do this on purpose. ". + ::warning("Input is read from the terminal.\n"); + ::warning("Only experts do this on purpose. ". "Press CTRL-D to exit.\n"); } } diff --git a/src/parallel.pod b/src/parallel.pod index 15e3bcf8..d436cffb 100644 --- a/src/parallel.pod +++ b/src/parallel.pod @@ -100,7 +100,7 @@ B<--env> and use B instead of B. The command cannot contain the character \257 (macron: ¯). -=item B<{}> +=item B<{}> (alpha testing) Input line. This replacement string will be replaced by a full line read from the input source. The input source is normally stdin @@ -113,7 +113,7 @@ If the command line contains no replacement strings then B<{}> will be appended to the command line. -=item B<{.}> +=item B<{.}> (alpha testing) Input line without extension. This replacement string will be replaced by the input with the extension removed. If the input line contains @@ -129,7 +129,7 @@ The replacement string B<{.}> can be changed with B<--er>. To understand replacement strings see B<{}>. -=item B<{/}> +=item B<{/}> (alpha testing) Basename of input line. This replacement string will be replaced by the input with the directory part removed. @@ -140,7 +140,7 @@ B<--basenamereplace>. To understand replacement strings see B<{}>. -=item B<{//}> +=item B<{//}> (alpha testing) Dirname of input line. This replacement string will be replaced by the dir of the input line. See B(1). @@ -151,7 +151,7 @@ B<--dirnamereplace>. To understand replacement strings see B<{}>. -=item B<{/.}> +=item B<{/.}> (alpha testing) Basename of input line without extension. This replacement string will be replaced by the input with the directory and extension part @@ -163,7 +163,7 @@ B<--basenameextensionreplace>. To understand replacement strings see B<{}>. -=item B<{#}> +=item B<{#}> (alpha testing) Sequence number of the job to run. This replacement string will be replaced by the sequence number of the job being run. It contains the @@ -174,7 +174,7 @@ The replacement string B<{#}> can be changed with B<--seqreplace>. To understand replacement strings see B<{}>. -=item B<{%}> +=item B<{%}> (alpha testing) Job slot number. This replacement string will be replaced by the job's slot number between 1 and number of jobs to run in parallel. There @@ -186,7 +186,7 @@ The replacement string B<{%}> can be changed with B<--slotreplace>. To understand replacement strings see B<{}>. -=item B<{>IB<}> +=item B<{>IB<}> (alpha testing) Argument from input source I or the I'th argument. This positional replacement string will be replaced by the input from input @@ -197,7 +197,7 @@ I'th last argument. To understand replacement strings see B<{}>. -=item B<{>I.B<}> +=item B<{>I.B<}> (alpha testing) Argument from input source I or the I'th argument without extension. It is a combination of B<{>IB<}> and B<{.}>. @@ -210,7 +210,7 @@ extension removed. To understand positional replacement strings see B<{>IB<}>. -=item B<{>I/B<}> +=item B<{>I/B<}> (alpha testing) Basename of argument from input source I or the I'th argument. It is a combination of B<{>IB<}> and B<{/}>. @@ -223,7 +223,7 @@ directory (if any) removed. To understand positional replacement strings see B<{>IB<}>. -=item B<{>I//B<}> +=item B<{>I//B<}> (alpha testing) Dirname of argument from input source I or the I'th argument. It is a combination of B<{>IB<}> and B<{//}>. @@ -235,7 +235,7 @@ the I'th argument (when used with B<-N>). See B(1). To understand positional replacement strings see B<{>IB<}>. -=item B<{>I/.B<}> +=item B<{>I/.B<}> (alpha testing) Basename of argument from input source I or the I'th argument without extension. It is a combination of B<{>IB<}>, B<{/}>, and @@ -249,7 +249,7 @@ directory (if any) and extension removed. To understand positional replacement strings see B<{>IB<}>. -=item B<{=>IB<=}> +=item B<{=>IB<=}> (alpha testing) Replace with calculated I. B<$_> will contain the same as B<{}>. After evaluating I B<$_> will be used @@ -262,7 +262,7 @@ The B<{=>IB<=}> must be given as a single string. See also: B<--rpl> B<--parens> -=item B<{=>I IB<=}> +=item B<{=>I IB<=}> (alpha testing) Positional equivalent to B<{= perl expression =}>. To understand positional replacement strings see B<{>IB<}>. @@ -426,7 +426,7 @@ notice. =item B<--block-size> I -Size of block in bytes. The size can be postfixed with K, M, G, T, P, +Size of block in bytes. The I can be postfixed with K, M, G, T, P, k, m, g, t, or p which would multiply the size with 1024, 1048576, 1073741824, 1099511627776, 1125899906842624, 1000, 1000000, 1000000000, 1000000000000, or 1000000000000000 respectively. @@ -440,7 +440,7 @@ I defaults to 1M. See B<--pipe> and B<--pipepart> for use of this. -=item B<--cat> +=item B<--cat> (alpha testing) Create a temporary file with content. Normally B<--pipe>/B<--pipepart> will give data to the program on stdin (standard input). With B<--cat> @@ -450,7 +450,7 @@ you can do: B. See also B<--fifo>. -=item B<--cleanup> +=item B<--cleanup> (alpha testing) Remove transferred files. B<--cleanup> will remove the transferred files on the remote computer after processing is done. @@ -564,7 +564,7 @@ If I is omitted, there is no end of file string. If neither B<-E> nor B<-e> is used, no end of file string is used. -=item B<--env> I +=item B<--env> I (alpha testing) Copy environment variable I. This will copy I to the environment that the command is run in. This is especially useful for @@ -631,7 +631,7 @@ Implies B<--semaphore>. See also B<--bg>, B. -=item B<--fifo> +=item B<--fifo> (alpha testing) Create a temporary fifo with content. Normally B<--pipe> and B<--pipepart> will give data to the program on stdin (standard @@ -921,7 +921,7 @@ limiting factor. See also: B<--group> B<--ungroup> -=item B<--load> I +=item B<--load> I (alpha testing) Do not start new jobs on a given computer unless the number of running processes on the computer is less than I. I uses @@ -969,6 +969,21 @@ See also B<-X> for context replace. If in doubt use B<-X> as that will most likely do what is needed. +=item B<--memfree> I (alpha testing) + +Minimum memory free when starting another job. The I can be +postfixed with K, M, G, T, P, k, m, g, t, or p which would multiply +the size with 1024, 1048576, 1073741824, 1099511627776, +1125899906842624, 1000, 1000000, 1000000000, 1000000000000, or +1000000000000000 respectively. + +If the jobs take up very different amount of RAM, GNU B will +only start as many as there is memory for. If less than I bytes +are free, no more jobs will be started. If less than 50% I bytes +are free, the youngest job will be killed, and put back on the queue +to be run later. + + =item B<--minversion> I Print the version GNU B and exit. If the current version of @@ -1150,6 +1165,12 @@ Print the number of CPU cores and exit (used by GNU B itself to determine the number of CPU cores on remote computers). +=item B<--no-keep-order> (alpha testing) + +Overrides an earlier B<--keep-order> (e.g. if set in +B<~/.parallel/config>). + + =item B<--no-notice> Do not display citation notice. A citation notice is printed on stderr @@ -1367,7 +1388,7 @@ useful if some jobs fail for no apparent reason (such as network failure). -=item B<--return> I +=item B<--return> I (alpha testing) Transfer files from remote computers. B<--return> is used with B<--sshlogin> when the arguments are files on the remote computers. When @@ -1603,9 +1624,9 @@ I seconds after starting each ssh. I can be less than 1 seconds. -=item B<-S> I<[@hostgroups/][ncpu/]sshlogin[,[@hostgroups/][ncpu/]sshlogin[,...]]> +=item B<-S> I<[@hostgroups/][ncpu/]sshlogin[,[@hostgroups/][ncpu/]sshlogin[,...]]> (alpha testing) -=item B<--sshlogin> I<[@hostgroups/][ncpu/]sshlogin[,[@hostgroups/][ncpu/]sshlogin[,...]]> +=item B<--sshlogin> I<[@hostgroups/][ncpu/]sshlogin[,[@hostgroups/][ncpu/]sshlogin[,...]]> (alpha testing) Distribute jobs to remote computers. The jobs will be run on a list of remote computers. @@ -1786,7 +1807,7 @@ Print the job to be run on stderr (standard error). See also B<-v>, B<-p>. -=item B<--transfer> +=item B<--transfer> (alpha testing) Transfer files to remote computers. B<--transfer> is used with B<--sshlogin> when the arguments are files and should be transferred @@ -1813,7 +1834,7 @@ B<--transfer> is often used with B<--return> and B<--cleanup>. B<--transfer> is ignored when used with B<--sshlogin :> or when not used with B<--sshlogin>. -=item B<--trc> I +=item B<--trc> I (alpha testing) Transfer, Return, Cleanup. Short hand for: diff --git a/src/parallel_design.pod b/src/parallel_design.pod index 7c4550a5..4bfad3b4 100644 --- a/src/parallel_design.pod +++ b/src/parallel_design.pod @@ -521,6 +521,17 @@ What can be done about it? Unfortunately it is not always possible to predict the root cause of the error. +=head2 Computation of load + +Contrary to the obvious --load does not use load average. This is due +to load average rising too slowly. Instead it uses B to list the +number of jobs in running or blocked state (state D, O or R). This +gives an instat load. + +As remote calculation of load can be slow, a process is spawned to run +B and put the result in a file, which is then used next time. + + =head1 Ideas for new design =head2 Multiple processes working together diff --git a/src/sql b/src/sql index 01291480..648b41d8 100755 --- a/src/sql +++ b/src/sql @@ -566,7 +566,7 @@ $Global::Initfile && unlink $Global::Initfile; exit ($err); sub parse_options { - $Global::version = 20150109; + $Global::version = 20150122; $Global::progname = 'sql'; # This must be done first as this may exec myself diff --git a/testsuite/wanted-results/parallel-local12 b/testsuite/wanted-results/parallel-local12 index 8f50d890..1465c159 100644 --- a/testsuite/wanted-results/parallel-local12 +++ b/testsuite/wanted-results/parallel-local12 @@ -1,6 +1,7 @@ ### Test stdin goes to first command only ("-" as argument) Academic tradition requires you to cite works you base your article on. -When using programs that use GNU Parallel to process data for publication please cite: +When using programs that use GNU Parallel to process data for publication +please cite: O. Tange (2011): GNU Parallel - The Command-Line Power Tool, ;login: The USENIX Magazine, February 2011:42-47. @@ -8,7 +9,7 @@ When using programs that use GNU Parallel to process data for publication please This helps funding further development; and it won't cost you a cent. If you pay 10000 EUR you should feel free to use GNU Parallel without citing. -To silence this citation notice run 'parallel --bibtex' once or use '--will-cite'. +Silence the citation notice: run 'parallel --bibtex' once or use '--will-cite'. cat - via first cat @@ -16,7 +17,8 @@ cat - via pseudotty ### Test stdin goes to first command only ("cat" as argument) Academic tradition requires you to cite works you base your article on. -When using programs that use GNU Parallel to process data for publication please cite: +When using programs that use GNU Parallel to process data for publication +please cite: O. Tange (2011): GNU Parallel - The Command-Line Power Tool, ;login: The USENIX Magazine, February 2011:42-47. @@ -24,7 +26,7 @@ When using programs that use GNU Parallel to process data for publication please This helps funding further development; and it won't cost you a cent. If you pay 10000 EUR you should feel free to use GNU Parallel without citing. -To silence this citation notice run 'parallel --bibtex' once or use '--will-cite'. +Silence the citation notice: run 'parallel --bibtex' once or use '--will-cite'. echo a a diff --git a/testsuite/wanted-results/test15 b/testsuite/wanted-results/test15 index d63325ac..21c63fc2 100644 --- a/testsuite/wanted-results/test15 +++ b/testsuite/wanted-results/test15 @@ -1,7 +1,8 @@ ### Test -p --interactive spawn /tmp/parallel-script-for-expect Academic tradition requires you to cite works you base your article on. -When using programs that use GNU Parallel to process data for publication please cite: +When using programs that use GNU Parallel to process data for publication +please cite: O. Tange (2011): GNU Parallel - The Command-Line Power Tool, ;login: The USENIX Magazine, February 2011:42-47. @@ -9,7 +10,7 @@ When using programs that use GNU Parallel to process data for publication please This helps funding further development; and it won't cost you a cent. If you pay 10000 EUR you should feel free to use GNU Parallel without citing. -To silence this citation notice run 'parallel --bibtex' once or use '--will-cite'. +Silence the citation notice: run 'parallel --bibtex' once or use '--will-cite'. sleep 0.1; echo opt-p 1 ?...y sleep 0.1; echo opt-p 2 ?...n @@ -17,7 +18,8 @@ sleep 0.1; echo opt-p 3 ?...y opt-p 1 opt-p 3 Academic tradition requires you to cite works you base your article on. -When using programs that use GNU Parallel to process data for publication please cite: +When using programs that use GNU Parallel to process data for publication +please cite: O. Tange (2011): GNU Parallel - The Command-Line Power Tool, ;login: The USENIX Magazine, February 2011:42-47. @@ -25,7 +27,7 @@ When using programs that use GNU Parallel to process data for publication please This helps funding further development; and it won't cost you a cent. If you pay 10000 EUR you should feel free to use GNU Parallel without citing. -To silence this citation notice run 'parallel --bibtex' once or use '--will-cite'. +Silence the citation notice: run 'parallel --bibtex' once or use '--will-cite'. sleep 0.1; echo opt--interactive 1 ?...y sleep 0.1; echo opt--interactive 2 ?...n @@ -259,7 +261,8 @@ xargs Expect: 3 1 2 2 parallel Expect: 3 1 via psedotty 2 Academic tradition requires you to cite works you base your article on. -When using programs that use GNU Parallel to process data for publication please cite: +When using programs that use GNU Parallel to process data for publication +please cite: O. Tange (2011): GNU Parallel - The Command-Line Power Tool, ;login: The USENIX Magazine, February 2011:42-47. @@ -267,7 +270,7 @@ When using programs that use GNU Parallel to process data for publication please This helps funding further development; and it won't cost you a cent. If you pay 10000 EUR you should feel free to use GNU Parallel without citing. -To silence this citation notice run 'parallel --bibtex' once or use '--will-cite'. +Silence the citation notice: run 'parallel --bibtex' once or use '--will-cite'. 3 1 @@ -279,7 +282,8 @@ xargs Expect: 1 3 2 2 parallel Expect: 1 3 2 via pseudotty Academic tradition requires you to cite works you base your article on. -When using programs that use GNU Parallel to process data for publication please cite: +When using programs that use GNU Parallel to process data for publication +please cite: O. Tange (2011): GNU Parallel - The Command-Line Power Tool, ;login: The USENIX Magazine, February 2011:42-47. @@ -287,7 +291,7 @@ When using programs that use GNU Parallel to process data for publication please This helps funding further development; and it won't cost you a cent. If you pay 10000 EUR you should feel free to use GNU Parallel without citing. -To silence this citation notice run 'parallel --bibtex' once or use '--will-cite'. +Silence the citation notice: run 'parallel --bibtex' once or use '--will-cite'. 1 3