From 5be0293a96df4a8b1c624a69d5e4d4dcd40335b7 Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Wed, 21 Oct 2015 23:55:31 +0200 Subject: [PATCH] Released as 20151022 ('LiquidWater') --- NEWS | 37 ++++++++++++ README | 12 ++-- configure | 20 +++---- configure.ac | 2 +- doc/release_new_version | 14 ++++- .../obs/home:tange/parallel/parallel.spec | 2 +- src/niceload | 2 +- src/parallel | 2 +- src/parallel.pod | 34 +++++------ src/parallel_design.pod | 56 ++++++++++++++----- src/sql | 2 +- 11 files changed, 129 insertions(+), 54 deletions(-) diff --git a/NEWS b/NEWS index b3218906..2f210446 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,40 @@ +20151022 + +* --plus makes it possible to use {##} as a short had for {= + $_=$Global::JobQueue->total_jobs() =} which gives the the number of + jobs to run. + +* {= $_=$Global::JobQueue->total_jobs() =} is incompatible with -X, + -m, and --xargs. + +* GNU Parallel is now mostly compatible with lsh + (http://www.lysator.liu.se/~nisse/lsh/) and somewhat compatible with + autossh (http://www.harding.motd.ca/autossh/). + +* --workdir ... now also works when run locally. + +* GNU Parallel was cited in: There is no (75, 32, 10, 16) strongly + regular graph http://arxiv.org/pdf/1509.05933.pdf + +* GNU Parallel was cited in: Roary: rapid large-scale prokaryote pan + genome analysis + http://bioinformatics.oxfordjournals.org/content/early/2015/08/05/bioinformatics.btv421.full.pdf+html + +* GNU Parallel is used in TraitAR: + https://testpypi.python.org/pypi/traitar/0.1.4 + +* GNU Parallel is used in youtube-dl-parallel: + https://github.com/dlh/youtube-dl-parallel + +* A parallel and fast way to download multiple files + http://onetipperday.blogspot.com/2015/10/a-parallel-and-fast-way-to-download.html + +* Usar GNU Parallel para aumentar el rendimiento de tus scripts + http://adrianarroyocalle.github.io/blog/2015/10/20/usar-gnu-parallel/ + +* Bug fixes and man page updates. + + 20150922 * GNU Parallel was cited in: Flexible Modeling of Epidemics with an Empirical Bayes Framework http://journals.plos.org/ploscompbiol/article?id=10.1371%2Fjournal.pcbi.1004382 diff --git a/README b/README index 8e45e12d..2d4f0c9f 100644 --- a/README +++ b/README @@ -40,9 +40,9 @@ document. Full installation of GNU Parallel is as simple as: - wget http://ftpmirror.gnu.org/parallel/parallel-20150922.tar.bz2 - bzip2 -dc parallel-20150922.tar.bz2 | tar xvf - - cd parallel-20150922 + wget http://ftpmirror.gnu.org/parallel/parallel-20151022.tar.bz2 + bzip2 -dc parallel-20151022.tar.bz2 | tar xvf - + cd parallel-20151022 ./configure && make && sudo make install @@ -51,9 +51,9 @@ Full installation of GNU Parallel is as simple as: If you are not root you can add ~/bin to your path and install in ~/bin and ~/share: - wget http://ftpmirror.gnu.org/parallel/parallel-20150922.tar.bz2 - bzip2 -dc parallel-20150922.tar.bz2 | tar xvf - - cd parallel-20150922 + wget http://ftpmirror.gnu.org/parallel/parallel-20151022.tar.bz2 + bzip2 -dc parallel-20151022.tar.bz2 | tar xvf - + cd parallel-20151022 ./configure --prefix=$HOME && make && make install Or if your system lacks 'make' you can simply copy src/parallel diff --git a/configure b/configure index ce103e00..c50931d9 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.69 for parallel 20150922. +# Generated by GNU Autoconf 2.69 for parallel 20151022. # # Report bugs to . # @@ -579,8 +579,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='parallel' PACKAGE_TARNAME='parallel' -PACKAGE_VERSION='20150922' -PACKAGE_STRING='parallel 20150922' +PACKAGE_VERSION='20151022' +PACKAGE_STRING='parallel 20151022' PACKAGE_BUGREPORT='bug-parallel@gnu.org' PACKAGE_URL='' @@ -1203,7 +1203,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures parallel 20150922 to adapt to many kinds of systems. +\`configure' configures parallel 20151022 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1269,7 +1269,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of parallel 20150922:";; + short | recursive ) echo "Configuration of parallel 20151022:";; esac cat <<\_ACEOF @@ -1345,7 +1345,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -parallel configure 20150922 +parallel configure 20151022 generated by GNU Autoconf 2.69 Copyright (C) 2012 Free Software Foundation, Inc. @@ -1362,7 +1362,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by parallel $as_me 20150922, which was +It was created by parallel $as_me 20151022, which was generated by GNU Autoconf 2.69. Invocation command line was $ $0 $@ @@ -2225,7 +2225,7 @@ fi # Define the identity of the package. PACKAGE='parallel' - VERSION='20150922' + VERSION='20151022' cat >>confdefs.h <<_ACEOF @@ -2867,7 +2867,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by parallel $as_me 20150922, which was +This file was extended by parallel $as_me 20151022, which was generated by GNU Autoconf 2.69. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -2929,7 +2929,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -parallel config.status 20150922 +parallel config.status 20151022 configured by $0, generated by GNU Autoconf 2.69, with options \\"\$ac_cs_config\\" diff --git a/configure.ac b/configure.ac index 193f4764..88de8fb2 100644 --- a/configure.ac +++ b/configure.ac @@ -1,4 +1,4 @@ -AC_INIT([parallel], [20150922], [bug-parallel@gnu.org]) +AC_INIT([parallel], [20151022], [bug-parallel@gnu.org]) AM_INIT_AUTOMAKE([-Wall -Werror foreign]) AC_CONFIG_HEADERS([config.h]) AC_CONFIG_FILES([ diff --git a/doc/release_new_version b/doc/release_new_version index e62f4b05..faf3aaff 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -220,7 +220,9 @@ GNU Parallel 20151022 ('Liquid Water') <<[stable]>> has been released. It is ava Haiku of the month: - <<>> + In parallel land + everything is quick and fast + Use GNU Parallel. -- Ole Tange New in this release: @@ -234,6 +236,14 @@ New in this release: * <> GNU Parallel was used (unfortunately with wrong citation) in: TADSim: Discrete Event-Based Performance Prediction for Temperature-Accelerated Dynamics http://vruehle.de/publications/2015c.pdf http://www.researchgate.net/profile/Christoph_Junghans/publication/276178326_TADSim_Discrete_Event-Based_Performance_Prediction_for_Temperature-Accelerated_Dynamics/links/55562b6708ae980ca60c8369.pdf +* --plus makes it possible to use {##} as a short had for {= $_=$Global::JobQueue->total_jobs() =} which gives the the number of jobs to run. + +* {= $_=$Global::JobQueue->total_jobs() =} is incompatible with -X, -m, and --xargs. + +* GNU Parallel is now mostly compatible with lsh (http://www.lysator.liu.se/~nisse/lsh/) and somewhat compatible with autossh (http://www.harding.motd.ca/autossh/). + +* --workdir ... now also works when run locally. + * GNU Parallel was cited in: There is no (75, 32, 10, 16) strongly regular graph http://arxiv.org/pdf/1509.05933.pdf * GNU Parallel was cited in: Roary: rapid large-scale prokaryote pan genome analysis http://bioinformatics.oxfordjournals.org/content/early/2015/08/05/bioinformatics.btv421.full.pdf+html @@ -244,6 +254,8 @@ http://www.researchgate.net/profile/Christoph_Junghans/publication/276178326_TAD * A parallel and fast way to download multiple files http://onetipperday.blogspot.com/2015/10/a-parallel-and-fast-way-to-download.html +* Usar GNU Parallel para aumentar el rendimiento de tus scripts http://adrianarroyocalle.github.io/blog/2015/10/20/usar-gnu-parallel/ + * Bug fixes and man page updates. GNU Parallel - For people who live life in the parallel lane. diff --git a/packager/obs/home:tange/parallel/parallel.spec b/packager/obs/home:tange/parallel/parallel.spec index b49eb1dc..c8006aa7 100644 --- a/packager/obs/home:tange/parallel/parallel.spec +++ b/packager/obs/home:tange/parallel/parallel.spec @@ -1,6 +1,6 @@ Summary: Shell tool for executing jobs in parallel Name: parallel -Version: 20150922 +Version: 20151022 Release: 1 License: GPL Group: Productivity/File utilities diff --git a/src/niceload b/src/niceload index fafac702..7892f22d 100755 --- a/src/niceload +++ b/src/niceload @@ -24,7 +24,7 @@ use strict; use Getopt::Long; $Global::progname="niceload"; -$Global::version = 20150922; +$Global::version = 20151022; Getopt::Long::Configure("bundling","require_order"); get_options_from_array(\@ARGV) || die_usage(); if($opt::version) { diff --git a/src/parallel b/src/parallel index 323a8e91..6768830c 100755 --- a/src/parallel +++ b/src/parallel @@ -1100,7 +1100,7 @@ sub parse_options { sub init_globals { # Defaults: - $Global::version = 20151001; + $Global::version = 20151022; $Global::progname = 'parallel'; $Global::infinity = 2**31; $Global::debug = 0; diff --git a/src/parallel.pod b/src/parallel.pod index 65dd54c0..914c9576 100644 --- a/src/parallel.pod +++ b/src/parallel.pod @@ -426,7 +426,7 @@ string that is not in the command line. See also: B<:::>. -=item B<--bar> +=item B<--bar> (alpha testing) Show progress as a progress bar. In the bar is shown: % of jobs completed, estimated seconds left, and number of jobs started. @@ -513,7 +513,7 @@ I defaults to 1M. See B<--pipe> and B<--pipepart> for use of this. -=item B<--cat> +=item B<--cat> (alpha testing) Create a temporary file with content. Normally B<--pipe>/B<--pipepart> will give data to the program on stdin (standard input). With B<--cat> @@ -613,7 +613,7 @@ I seconds after starting each job. I can be less than 1 second. -=item B<--dry-run> +=item B<--dry-run> (alpha testing) Print the job to run on stdout (standard output), but do not run the job. Use B<-v -v> to include the wrapping that GNU Parallel generates @@ -672,7 +672,7 @@ and functions) use env_parallel as described under the option I. See also: B<--record-env>. -=item B<--eta> +=item B<--eta> (alpha testing) Show the estimated number of seconds before finishing. This forces GNU B to read all jobs before starting to find the number of @@ -693,7 +693,7 @@ Implies B<--semaphore>. See also B<--bg>, B. -=item B<--fifo> +=item B<--fifo> (alpha testing) Create a temporary fifo with content. Normally B<--pipe> and B<--pipepart> will give data to the program on stdin (standard @@ -725,7 +725,7 @@ over B<--tollef>. The B<--tollef> option is now retired, and therefore may not be used. B<--gnu> is kept for compatibility. -=item B<--group> +=item B<--group> (alpha testing) Group output. Output from each jobs is grouped together and is only printed when the command is finished. stderr (standard error) first @@ -747,9 +747,9 @@ See also: B<--line-buffer> B<--ungroup> Print a summary of the options to GNU B and exit. -=item B<--halt-on-error> I +=item B<--halt-on-error> I (alpha testing) -=item B<--halt> I +=item B<--halt> I (alpha testing) When should GNU B terminate? In some situations it makes no sense to run all jobs. GNU B should simply give up as soon @@ -1172,7 +1172,7 @@ performance is important use B<--pipepart>. See also: B<--recstart>, B<--recend>, B<--fifo>, B<--cat>, B<--pipepart>. -=item B<--pipepart> +=item B<--pipepart> (alpha testing) Pipe parts of a physical file. B<--pipepart> works similar to B<--pipe>, but is much faster. It has a few limitations: @@ -1288,7 +1288,7 @@ Overrides an earlier B<--keep-order> (e.g. if set in B<~/.parallel/config>). -=item B<--nice> I +=item B<--nice> I (alpha testing) Run the command at this niceness. For simple commands you can just add B in front of the command. But if the command consists of more @@ -1737,7 +1737,7 @@ Does not run the command but quotes it. Useful for making quoted composed commands for GNU B. -=item B<--shuf> +=item B<--shuf> (alpha testing) Shuffle jobs. When having multiple input sources it is hard to randomize jobs. --shuf will generate all jobs, and shuffle them before @@ -1765,13 +1765,13 @@ I seconds after starting each ssh. I can be less than 1 seconds. -=item B<-S> I<[@hostgroups/][ncpu/]sshlogin[,[@hostgroups/][ncpu/]sshlogin[,...]]> +=item B<-S> I<[@hostgroups/][ncpu/]sshlogin[,[@hostgroups/][ncpu/]sshlogin[,...]]> (alpha testing) -=item B<-S> I<@hostgroup> +=item B<-S> I<@hostgroup> (alpha testing) -=item B<--sshlogin> I<[@hostgroups/][ncpu/]sshlogin[,[@hostgroups/][ncpu/]sshlogin[,...]]> +=item B<--sshlogin> I<[@hostgroups/][ncpu/]sshlogin[,[@hostgroups/][ncpu/]sshlogin[,...]]> (alpha testing) -=item B<--sshlogin> I<@hostgroup> +=item B<--sshlogin> I<@hostgroup> (alpha testing) Distribute jobs to remote computers. The jobs will be run on a list of remote computers. @@ -2088,9 +2088,9 @@ Use B<-v> B<-v> to print the wrapping ssh command when running remotely. Print the version GNU B and exit. -=item B<--workdir> I +=item B<--workdir> I (alpha testing) -=item B<--wd> I +=item B<--wd> I (alpha testing) Files transferred using B<--transfer> and B<--return> will be relative to I on remote computers, and the command will be executed in diff --git a/src/parallel_design.pod b/src/parallel_design.pod index 809b2c06..00aef871 100644 --- a/src/parallel_design.pod +++ b/src/parallel_design.pod @@ -129,11 +129,9 @@ echo I =item --nice I -\nice -n I I -c I +Remote: See B. -The \ is needed to avoid using the builtin nice command, which does not -support -n in B. B -c> is needed to nice composed commands -command. +Local: B =item --cat @@ -374,7 +372,12 @@ the %%-version, and the function definition works on both pre- and post-shellshock versions of B. -=head2 Remote Ctrl-C and standard error (stderr) +=head2 The remote system wrapper + +The remote system wrapper does some initialization before starting the +command on the remote system. + +=head3 Ctrl-C and standard error (stderr) If the user presses Ctrl-C the user expects jobs to stop. This works out of the box if the jobs are run locally. Unfortunately it is not so @@ -394,28 +397,51 @@ propagate the hang-up signal from B. But when B dies, the parent of the login shell becomes B (process id 1). So by exec'ing a Perl wrapper to monitor the parent pid and kill the child if the parent pid becomes 1, then Ctrl-C works and stderr is kept on -stderr. The wrapper looks like this: +stderr. +=head3 --nice + +Bing the remote process is done by B. A +few old systems do not implement this and is thus unsupported. + +=head3 Setting $PARALLEL_TMP + +B<$PARALLEL_TMP> is used by B<--fifo> and B<--cat> and must point to a +non-exitent file in B<$TMPDIR>. This file name is computed on the +remote system. + +=head3 The wrapper + +The wrapper looks like this: + + $shell = $PARALLEL_SHELL || $SHELL; + $tmpdir = $TMPDIR; + $nice = $opt::nice; + # Set $PARALLEL_TMP to a non-existent file name in $TMPDIR + do { + $ENV{PARALLEL_TMP} = $tmpdir."/par". + join"", map { (0..9,"a".."z","A".."Z")[rand(62)] } (1..5); + } while(-e $ENV{PARALLEL_TMP}); $SIG{CHLD} = sub { $done = 1; }; $pid = fork; unless($pid) { - # Make own process group to be able to kill HUP it later - setpgrp; - exec $ENV{SHELL}, "-c", ($bashfunc."@ARGV"); - die "exec: $!\n"; + # Make own process group to be able to kill HUP it later + setpgrp; + eval { setpriority(0,0,$nice) }; + exec $shell, "-c", ($bashfunc."@ARGV"); + die "exec: $!\n"; } do { - # Parent is not init (ppid=1), so sshd is alive - # Exponential sleep up to 1 sec - $s = $s < 1 ? 0.001 + $s * 1.03 : $s; - select(undef, undef, undef, $s); + # Parent is not init (ppid=1), so sshd is alive + # Exponential sleep up to 1 sec + $s = $s < 1 ? 0.001 + $s * 1.03 : $s; + select(undef, undef, undef, $s); } until ($done || getppid == 1); # Kill HUP the process group if job not done kill(SIGHUP, -${pid}) unless $done; wait; exit ($?&127 ? 128+($?&127) : 1+$?>>8) - =head2 Transferring of variables and functions Transferring of variables and functions given by B<--env> is done by diff --git a/src/sql b/src/sql index c34bcebd..00370686 100755 --- a/src/sql +++ b/src/sql @@ -566,7 +566,7 @@ $Global::Initfile && unlink $Global::Initfile; exit ($err); sub parse_options { - $Global::version = 20150922; + $Global::version = 20151022; $Global::progname = 'sql'; # This must be done first as this may exec myself