diff --git a/NEWS b/NEWS index dbf51b6c..3bbfc0b3 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,38 @@ +20141122 + +* Remote systems can be divided into hostgroups (e.g. web and db) by + prepending '@groupname/' to the sshlogin. Multiple groups can be + given by separating groups with '+'. E.g. @web/www1 @web+db/www2 + @db/mariadb + +* Remote execution can be restricted to servers that are part of one + or more groups by '@groupname' as an sshlogin. Multiple groups can + be given by separating groups with '+'. E.g. -S @web or -S @db+web + +* With --hostgroup you can restrict arguments to certain hostgroups by + appending '@groupname' to the argument. Multiple groups can be + given by separating groups with '+'. E.g. my_web_arg@web + db-or-web-arg@db+web db-only-arg@db Thanks to Michel Courtine for + developing a prototype for this. + +* HTSeq-Hadoop: Extending HTSeq for Massively Parallel Sequencing Data + Analysis using Hadoop + http://essenceofescience.se/wp-content/uploads/2014/11/Siretskiy.pdf + +* GNU Parallel was cited in: SlideToolkit: An Assistive Toolset for + the Histological Quantification of Whole Slide Images + http://www.plosone.org/article/info%3Adoi%2F10.1371%2Fjournal.pone.0110289#close + +* GNU Parallel was cited in: Exploring a multiprocessor design space + to analyze the impact of using STT-RAM in the memory hierarchy + http://conservancy.umn.edu/bitstream/handle/11299/167286/Borse_umn_0130M_15431.pdf + +* Command-Line OCR with Tesseract on Mac OS X + https://ryanfb.github.io/etc/2014/11/13/command_line_ocr_on_mac_os_x.html + +* Bug fixes and man page updates. + + 20141022 * --env was changed to deal with Bash's new way of exporting a function. diff --git a/README b/README index e5cd4c62..4f633e42 100644 --- a/README +++ b/README @@ -40,9 +40,9 @@ document. Full installation of GNU Parallel is as simple as: - wget http://ftpmirror.gnu.org/parallel/parallel-20141022.tar.bz2 - bzip2 -dc parallel-20141022.tar.bz2 | tar xvf - - cd parallel-20141022 + wget http://ftpmirror.gnu.org/parallel/parallel-20141122.tar.bz2 + bzip2 -dc parallel-20141122.tar.bz2 | tar xvf - + cd parallel-20141122 ./configure && make && make install @@ -51,9 +51,9 @@ Full installation of GNU Parallel is as simple as: If you are not root you can add ~/bin to your path and install in ~/bin and ~/share: - wget http://ftpmirror.gnu.org/parallel/parallel-20141022.tar.bz2 - bzip2 -dc parallel-20141022.tar.bz2 | tar xvf - - cd parallel-20141022 + wget http://ftpmirror.gnu.org/parallel/parallel-20141122.tar.bz2 + bzip2 -dc parallel-20141122.tar.bz2 | tar xvf - + cd parallel-20141122 ./configure --prefix=$HOME && make && make install Or if your system lacks 'make' you can simply copy src/parallel diff --git a/configure b/configure index 4068316c..c907d532 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.69 for parallel 20141022. +# Generated by GNU Autoconf 2.69 for parallel 20141122. # # Report bugs to . # @@ -579,8 +579,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='parallel' PACKAGE_TARNAME='parallel' -PACKAGE_VERSION='20141022' -PACKAGE_STRING='parallel 20141022' +PACKAGE_VERSION='20141122' +PACKAGE_STRING='parallel 20141122' PACKAGE_BUGREPORT='bug-parallel@gnu.org' PACKAGE_URL='' @@ -1203,7 +1203,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures parallel 20141022 to adapt to many kinds of systems. +\`configure' configures parallel 20141122 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1269,7 +1269,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of parallel 20141022:";; + short | recursive ) echo "Configuration of parallel 20141122:";; esac cat <<\_ACEOF @@ -1345,7 +1345,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -parallel configure 20141022 +parallel configure 20141122 generated by GNU Autoconf 2.69 Copyright (C) 2012 Free Software Foundation, Inc. @@ -1362,7 +1362,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by parallel $as_me 20141022, which was +It was created by parallel $as_me 20141122, which was generated by GNU Autoconf 2.69. Invocation command line was $ $0 $@ @@ -2225,7 +2225,7 @@ fi # Define the identity of the package. PACKAGE='parallel' - VERSION='20141022' + VERSION='20141122' cat >>confdefs.h <<_ACEOF @@ -2867,7 +2867,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by parallel $as_me 20141022, which was +This file was extended by parallel $as_me 20141122, which was generated by GNU Autoconf 2.69. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -2929,7 +2929,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -parallel config.status 20141022 +parallel config.status 20141122 configured by $0, generated by GNU Autoconf 2.69, with options \\"\$ac_cs_config\\" diff --git a/configure.ac b/configure.ac index 119abe10..7db1d9bb 100644 --- a/configure.ac +++ b/configure.ac @@ -1,4 +1,4 @@ -AC_INIT([parallel], [20141022], [bug-parallel@gnu.org]) +AC_INIT([parallel], [20141122], [bug-parallel@gnu.org]) AM_INIT_AUTOMAKE([-Wall -Werror foreign]) AC_CONFIG_HEADERS([config.h]) AC_CONFIG_FILES([ diff --git a/doc/release_new_version b/doc/release_new_version index 88df3ead..7fcc4e58 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -232,7 +232,10 @@ GNU Parallel 20141122 ('Rosetta') has been released. It is available for downloa Haiku of the month: -<<>> +Hadoop bit too much? +Want a simpler syntax now? +Use GNU Parallel. + -- Ole Tange A central piece of command generation was rewritten making this release beta quality. As always it passes the testsuite, so most functionality clearly works. @@ -244,6 +247,8 @@ New in this release: * With --hostgroup you can restrict arguments to certain hostgroups by appending '@groupname' to the argument. Multiple groups can be given by separating groups with '+'. E.g. my_web_arg@web db-or-web-arg@db+web db-only-arg@db Thanks to Michel Courtine for developing a prototype for this. +* HTSeq-Hadoop: Extending HTSeq for Massively Parallel Sequencing Data Analysis using Hadoop http://essenceofescience.se/wp-content/uploads/2014/11/Siretskiy.pdf + * GNU Parallel was cited in: SlideToolkit: An Assistive Toolset for the Histological Quantification of Whole Slide Images http://www.plosone.org/article/info%3Adoi%2F10.1371%2Fjournal.pone.0110289#close * GNU Parallel was cited in: Exploring a multiprocessor design space to analyze the impact of using STT-RAM in the memory hierarchy http://conservancy.umn.edu/bitstream/handle/11299/167286/Borse_umn_0130M_15431.pdf diff --git a/packager/obs/home:tange/parallel/parallel.spec b/packager/obs/home:tange/parallel/parallel.spec index 61c4511f..b6541780 100644 --- a/packager/obs/home:tange/parallel/parallel.spec +++ b/packager/obs/home:tange/parallel/parallel.spec @@ -1,6 +1,6 @@ Summary: Shell tool for executing jobs in parallel Name: parallel -Version: 20141022 +Version: 20141122 Release: 1 License: GPL Group: Productivity/File utilities diff --git a/src/niceload b/src/niceload index c72a281b..534479c6 100755 --- a/src/niceload +++ b/src/niceload @@ -24,7 +24,7 @@ use strict; use Getopt::Long; $Global::progname="niceload"; -$Global::version = 20141023; +$Global::version = 20141122; Getopt::Long::Configure("bundling","require_order"); get_options_from_array(\@ARGV) || die_usage(); if($opt::version) { diff --git a/src/parallel b/src/parallel index 537b8d9f..abbf8f10 100755 --- a/src/parallel +++ b/src/parallel @@ -507,8 +507,6 @@ sub nindex { } $sleep = ::reap_usleep($sleep); } -# TODO Why is needed? -# start_more_jobs(); return $something_written; } } @@ -783,7 +781,7 @@ sub get_options_from_array { sub parse_options { # Returns: N/A # Defaults: - $Global::version = 20141111; + $Global::version = 20141122; $Global::progname = 'parallel'; $Global::infinity = 2**31; $Global::debug = 0; @@ -5159,6 +5157,17 @@ sub write { } sub set_stdin_buffer { + # Copy stdin buffer from $block_ref up to $endpos + # Prepend with $header_ref + # Remove $recstart and $recend if needed + # Input: + # $header_ref = ref to $header to prepend + # $block_ref = ref to $block to pass on + # $endpos = length of $block to pass on + # $recstart = --recstart regexp + # $recend = --recend regexp + # Returns: + # N/A my $self = shift; my ($header_ref,$block_ref,$endpos,$recstart,$recend) = @_; $self->{'stdin_buffer'} = ($self->virgin() ? $$header_ref : "").substr($$block_ref,0,$endpos); @@ -5583,14 +5592,19 @@ sub sshlogin_wrap { # * not mixing STDERR and STDOUT # * terminating with ctrl-c # It works on Linux but not Solaris + # Finishes on Solaris, but wrong exit code: + # $SIG{CHLD} = sub {exit ($?&127 ? 128+($?&127) : 1+$?>>8)}; + # Hangs on Solaris, but correct exit code on Linux: + # $SIG{CHLD} = sub { $done = 1 }; + # $p->poll; my $signal_script = "perl -e '". q{ use IO::Poll; - $SIG{CHLD} = sub {exit ($?&127 ? 128+($?&127) : 1+$?>>8)}; + $SIG{CHLD} = sub { $done = 1 }; $p = IO::Poll->new; $p->mask(STDOUT, POLLHUP); $pid=fork; unless($pid) {setpgrp; exec $ENV{SHELL}, "-c", @ARGV; die "exec: $!\n"} - $p->poll; + $p->poll; kill SIGHUP, -${pid} unless $done; wait; exit ($?&127 ? 128+($?&127) : 1+$?>>8) } . "' "; @@ -5600,7 +5614,7 @@ sub sshlogin_wrap { ($pre . "$sshcmd $ssh_options $serverlogin $parallel_env " . $remote_pre -# . ::shell_quote_scalar($signal_script . ::shell_quote_scalar($next_command_line)) +# . ::shell_quote_scalar($signal_script . ::shell_quote_scalar($command)) . ::shell_quote_scalar($command) . ";" . $post); diff --git a/src/parallel.pod b/src/parallel.pod index 7d20b1c8..a6fa5711 100644 --- a/src/parallel.pod +++ b/src/parallel.pod @@ -2373,6 +2373,27 @@ can be written like this: cat list | parallel "do_something {} scale {.}.jpg ; do_step2 <{} {.}" |\ process_output +If the body of the loop is bigger, it improves readability to use a function: + + (for x in `cat list` ; do + do_something $x + [... 100 lines that do something with $x ...] + done) | process_output + + cat list | (while read x ; do + do_something $x + [... 100 lines that do something with $x ...] + done) | process_output + +can both be rewritten as: + + doit() { + x=$1 + do_something $x + [... 100 lines that do something with $x ...] + } + export -f doit + cat list | parallel doit =head1 EXAMPLE: Rewriting nested for-loops diff --git a/src/sql b/src/sql index 870b9ef8..25327353 100755 --- a/src/sql +++ b/src/sql @@ -566,7 +566,7 @@ $Global::Initfile && unlink $Global::Initfile; exit ($err); sub parse_options { - $Global::version = 20141023; + $Global::version = 20141122; $Global::progname = 'sql'; # This must be done first as this may exec myself diff --git a/testsuite/tests-to-run/niceload03.sh b/testsuite/tests-to-run/niceload03.sh index 21a13d05..979875c7 100755 --- a/testsuite/tests-to-run/niceload03.sh +++ b/testsuite/tests-to-run/niceload03.sh @@ -1,7 +1,7 @@ #!/bin/bash # force load > 10 -while uptime | grep -v age:.[1-9][0-9].[0-9][0-9] >/dev/null ; do (timeout 5 nice burnP6 2>/dev/null &) done +while uptime | grep -v age:.[1-9]\\+[0-9].[0-9][0-9] >/dev/null ; do (timeout 5 nice burnP6 2>/dev/null &) done int() { perl -pe 's/(\d+\.\d*)/int($1)/e'