Released as 20141122 ('Rosetta').

NEWS

@@ -1,3 +1,38 @@
+20141122
+
+* Remote systems can be divided into hostgroups (e.g. web and db) by
+  prepending '@groupname/' to the sshlogin. Multiple groups can be
+  given by separating groups with '+'. E.g. @web/www1 @web+db/www2
+  @db/mariadb
+
+* Remote execution can be restricted to servers that are part of one
+  or more groups by '@groupname' as an sshlogin. Multiple groups can
+  be given by separating groups with '+'. E.g. -S @web or -S @db+web
+
+* With --hostgroup you can restrict arguments to certain hostgroups by
+  appending '@groupname' to the argument.  Multiple groups can be
+  given by separating groups with '+'. E.g. my_web_arg@web
+  db-or-web-arg@db+web db-only-arg@db Thanks to Michel Courtine for
+  developing a prototype for this.
+
+* HTSeq-Hadoop: Extending HTSeq for Massively Parallel Sequencing Data
+  Analysis using Hadoop
+  http://essenceofescience.se/wp-content/uploads/2014/11/Siretskiy.pdf
+
+* GNU Parallel was cited in: SlideToolkit: An Assistive Toolset for
+  the Histological Quantification of Whole Slide Images
+  http://www.plosone.org/article/info%3Adoi%2F10.1371%2Fjournal.pone.0110289#close
+
+* GNU Parallel was cited in: Exploring a multiprocessor design space
+  to analyze the impact of using STT-RAM in the memory hierarchy
+  http://conservancy.umn.edu/bitstream/handle/11299/167286/Borse_umn_0130M_15431.pdf
+
+* Command-Line OCR with Tesseract on Mac OS X
+  https://ryanfb.github.io/etc/2014/11/13/command_line_ocr_on_mac_os_x.html
+
+* Bug fixes and man page updates.
+
+
 20141022
 
 * --env was changed to deal with Bash's new way of exporting a function.

README

@@ -40,9 +40,9 @@ document.
 
 Full installation of GNU Parallel is as simple as:
 
-    wget http://ftpmirror.gnu.org/parallel/parallel-20141022.tar.bz2
+    wget http://ftpmirror.gnu.org/parallel/parallel-20141122.tar.bz2
-    bzip2 -dc parallel-20141022.tar.bz2 | tar xvf -
+    bzip2 -dc parallel-20141122.tar.bz2 | tar xvf -
-    cd parallel-20141022
+    cd parallel-20141122
     ./configure && make && make install
 
 
@@ -51,9 +51,9 @@ Full installation of GNU Parallel is as simple as:
 If you are not root you can add ~/bin to your path and install in
 ~/bin and ~/share:
 
-    wget http://ftpmirror.gnu.org/parallel/parallel-20141022.tar.bz2
+    wget http://ftpmirror.gnu.org/parallel/parallel-20141122.tar.bz2
-    bzip2 -dc parallel-20141022.tar.bz2 | tar xvf -
+    bzip2 -dc parallel-20141122.tar.bz2 | tar xvf -
-    cd parallel-20141022
+    cd parallel-20141122
     ./configure --prefix=$HOME && make && make install
 
 Or if your system lacks 'make' you can simply copy src/parallel

configure

@@ -1,6 +1,6 @@
 #! /bin/sh
 # Guess values for system-dependent variables and create Makefiles.
-# Generated by GNU Autoconf 2.69 for parallel 20141022.
+# Generated by GNU Autoconf 2.69 for parallel 20141122.
 #
 # Report bugs to <bug-parallel@gnu.org>.
 #
@@ -579,8 +579,8 @@ MAKEFLAGS=
 # Identity of this package.
 PACKAGE_NAME='parallel'
 PACKAGE_TARNAME='parallel'
-PACKAGE_VERSION='20141022'
+PACKAGE_VERSION='20141122'
-PACKAGE_STRING='parallel 20141022'
+PACKAGE_STRING='parallel 20141122'
 PACKAGE_BUGREPORT='bug-parallel@gnu.org'
 PACKAGE_URL=''
 
@@ -1203,7 +1203,7 @@ if test "$ac_init_help" = "long"; then
   # Omit some internal or obsolete options to make the list less imposing.
   # This message is too long to be a string in the A/UX 3.1 sh.
   cat <<_ACEOF
-\`configure' configures parallel 20141022 to adapt to many kinds of systems.
+\`configure' configures parallel 20141122 to adapt to many kinds of systems.
 
 Usage: $0 [OPTION]... [VAR=VALUE]...
 
@@ -1269,7 +1269,7 @@ fi
 
 if test -n "$ac_init_help"; then
   case $ac_init_help in
-     short | recursive ) echo "Configuration of parallel 20141022:";;
+     short | recursive ) echo "Configuration of parallel 20141122:";;
    esac
   cat <<\_ACEOF
 
@@ -1345,7 +1345,7 @@ fi
 test -n "$ac_init_help" && exit $ac_status
 if $ac_init_version; then
   cat <<\_ACEOF
-parallel configure 20141022
+parallel configure 20141122
 generated by GNU Autoconf 2.69
 
 Copyright (C) 2012 Free Software Foundation, Inc.
@@ -1362,7 +1362,7 @@ cat >config.log <<_ACEOF
 This file contains any messages produced by compilers while
 running configure, to aid debugging if configure makes a mistake.
 
-It was created by parallel $as_me 20141022, which was
+It was created by parallel $as_me 20141122, which was
 generated by GNU Autoconf 2.69.  Invocation command line was
 
   $ $0 $@
@@ -2225,7 +2225,7 @@ fi
 
 # Define the identity of the package.
  PACKAGE='parallel'
- VERSION='20141022'
+ VERSION='20141122'
 
 
 cat >>confdefs.h <<_ACEOF
@@ -2867,7 +2867,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
 # report actual input values of CONFIG_FILES etc. instead of their
 # values after options handling.
 ac_log="
-This file was extended by parallel $as_me 20141022, which was
+This file was extended by parallel $as_me 20141122, which was
 generated by GNU Autoconf 2.69.  Invocation command line was
 
   CONFIG_FILES    = $CONFIG_FILES
@@ -2929,7 +2929,7 @@ _ACEOF
 cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
 ac_cs_version="\\
-parallel config.status 20141022
+parallel config.status 20141122
 configured by $0, generated by GNU Autoconf 2.69,
   with options \\"\$ac_cs_config\\"
 

configure.ac

@@ -1,4 +1,4 @@
-AC_INIT([parallel], [20141022], [bug-parallel@gnu.org])
+AC_INIT([parallel], [20141122], [bug-parallel@gnu.org])
 AM_INIT_AUTOMAKE([-Wall -Werror foreign])
 AC_CONFIG_HEADERS([config.h])
 AC_CONFIG_FILES([

doc/release_new_version

@@ -232,7 +232,10 @@ GNU Parallel 20141122 ('Rosetta') has been released. It is available for downloa
 
 Haiku of the month:
 
-<<>>
+Hadoop bit too much?
+Want a simpler syntax now?
+Use GNU Parallel.
+  -- Ole Tange
 
 A central piece of command generation was rewritten making this release beta quality. As always it passes the testsuite, so most functionality clearly works.
 
@@ -244,6 +247,8 @@ New in this release:
 
 * With --hostgroup you can restrict arguments to certain hostgroups by appending '@groupname' to the argument.  Multiple groups can be given by separating groups with '+'. E.g. my_web_arg@web db-or-web-arg@db+web db-only-arg@db Thanks to Michel Courtine for developing a prototype for this.
 
+* HTSeq-Hadoop: Extending HTSeq for Massively Parallel Sequencing Data Analysis using Hadoop http://essenceofescience.se/wp-content/uploads/2014/11/Siretskiy.pdf
+
 * GNU Parallel was cited in: SlideToolkit: An Assistive Toolset for the Histological Quantification of Whole Slide Images http://www.plosone.org/article/info%3Adoi%2F10.1371%2Fjournal.pone.0110289#close
 
 * GNU Parallel was cited in: Exploring a multiprocessor design space to analyze the impact of using STT-RAM in the memory hierarchy http://conservancy.umn.edu/bitstream/handle/11299/167286/Borse_umn_0130M_15431.pdf

packager/obs/home:tange/parallel/parallel.spec

@@ -1,6 +1,6 @@
 Summary:	Shell tool for executing jobs in parallel
 Name: 		parallel
-Version: 	20141022
+Version: 	20141122
 Release: 	1
 License: 	GPL
 Group: 		Productivity/File utilities

src/niceload

@@ -24,7 +24,7 @@
 use strict;
 use Getopt::Long;
 $Global::progname="niceload";
-$Global::version = 20141023;
+$Global::version = 20141122;
 Getopt::Long::Configure("bundling","require_order");
 get_options_from_array(\@ARGV) || die_usage();
 if($opt::version) {

src/parallel

@@ -507,8 +507,6 @@ sub nindex {
 	    }
 	    $sleep = ::reap_usleep($sleep);
 	}
-# TODO Why is needed?
-#	start_more_jobs();
 	return $something_written;
     }
 }
@@ -783,7 +781,7 @@ sub get_options_from_array {
 sub parse_options {
     # Returns: N/A
     # Defaults:
-    $Global::version = 20141111;
+    $Global::version = 20141122;
     $Global::progname = 'parallel';
     $Global::infinity = 2**31;
     $Global::debug = 0;
@@ -5159,6 +5157,17 @@ sub write {
 }
 
 sub set_stdin_buffer {
+    # Copy stdin buffer from $block_ref up to $endpos
+    # Prepend with $header_ref
+    # Remove $recstart and $recend if needed
+    # Input:
+    #   $header_ref = ref to $header to prepend
+    #   $block_ref = ref to $block to pass on
+    #   $endpos = length of $block to pass on
+    #   $recstart = --recstart regexp
+    #   $recend = --recend regexp
+    # Returns:
+    #   N/A
     my $self = shift;
     my ($header_ref,$block_ref,$endpos,$recstart,$recend) = @_;
     $self->{'stdin_buffer'} = ($self->virgin() ? $$header_ref : "").substr($$block_ref,0,$endpos);
@@ -5583,14 +5592,19 @@ sub sshlogin_wrap {
 	    # * not mixing STDERR and STDOUT
 	    # * terminating with ctrl-c
 	    # It works on Linux but not Solaris
+	    # Finishes on Solaris, but wrong exit code:
+	    # $SIG{CHLD} = sub {exit ($?&127 ? 128+($?&127) : 1+$?>>8)};
+	    # Hangs on Solaris, but correct exit code on Linux:
+	    # $SIG{CHLD} = sub { $done = 1 };
+	    # $p->poll;
 	    my $signal_script = "perl -e '".
 	    q{
 		use IO::Poll;
-		$SIG{CHLD} = sub {exit ($?&127 ? 128+($?&127) : 1+$?>>8)};
+                $SIG{CHLD} = sub { $done = 1 };
 		$p = IO::Poll->new;
 		$p->mask(STDOUT, POLLHUP);
 		$pid=fork; unless($pid) {setpgrp; exec $ENV{SHELL}, "-c", @ARGV; die "exec: $!\n"}
-		$p->poll;
+                $p->poll;
 		kill SIGHUP, -${pid} unless $done;
 		wait; exit ($?&127 ? 128+($?&127) : 1+$?>>8)
             } . "' ";
@@ -5600,7 +5614,7 @@ sub sshlogin_wrap {
 		($pre
 		 . "$sshcmd $ssh_options $serverlogin $parallel_env "
 		 . $remote_pre
-#		 . ::shell_quote_scalar($signal_script . ::shell_quote_scalar($next_command_line))
+#		 . ::shell_quote_scalar($signal_script . ::shell_quote_scalar($command))
 		 . ::shell_quote_scalar($command)
 		 . ";"
 		 . $post);

src/parallel.pod

@@ -2373,6 +2373,27 @@ can be written like this:
   cat list | parallel "do_something {} scale {.}.jpg ; do_step2 <{} {.}" |\
     process_output
 
+If the body of the loop is bigger, it improves readability to use a function:
+
+  (for x in `cat list` ; do
+    do_something $x
+    [... 100 lines that do something with $x ...]
+  done) | process_output
+
+  cat list | (while read x ; do
+    do_something $x
+    [... 100 lines that do something with $x ...]
+  done) | process_output
+
+can both be rewritten as:
+
+  doit() {
+    x=$1
+    do_something $x
+    [... 100 lines that do something with $x ...]
+  }
+  export -f doit
+  cat list | parallel doit
 
 =head1 EXAMPLE: Rewriting nested for-loops
 

src/sql

@@ -566,7 +566,7 @@ $Global::Initfile && unlink $Global::Initfile;
 exit ($err);
 
 sub parse_options {
-    $Global::version = 20141023;
+    $Global::version = 20141122;
     $Global::progname = 'sql';
 
     # This must be done first as this may exec myself

testsuite/tests-to-run/niceload03.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 # force load > 10
-while uptime | grep -v age:.[1-9][0-9].[0-9][0-9] >/dev/null ; do (timeout 5 nice burnP6 2>/dev/null &) done
+while uptime | grep -v age:.[1-9]\\+[0-9].[0-9][0-9] >/dev/null ; do (timeout 5 nice burnP6 2>/dev/null &) done
 
 int() {
   perl -pe 's/(\d+\.\d*)/int($1)/e'