Fixed bug #34955: Timed-out processes are not identified as failures.

Fixed bug #34954: --timeout should kill process groups.
Released as 20111209 alpha.

NEWS

@@ -241,7 +241,7 @@
 
 20110422
 
-* {#} now works an alias for $PARALLEL_PID.
+* {#} now works as an alias for $PARALLEL_PID.
 
 * --eta now estimates way more accurately.
 

README

@@ -5,35 +5,44 @@ Please send problems and feedback to bug-parallel@gnu.org.
 
 = Presentation of GNU Parallel =
 
-GNU parallel is a shell tool for executing jobs in parallel using one
+GNU Parallel is a shell tool for executing jobs in parallel using one
 or more computers. A job is can be a single command or a small script
 that has to be run for each of the lines in the input. The typical
 input is a list of files, a list of hosts, a list of users, a list of
 URLs, or a list of tables. A job can also be a command that reads from
-a pipe. GNU parallel can then split the input and pipe it into
+a pipe. GNU Parallel can then split the input and pipe it into
 commands in parallel.
 
-If you use xargs and tee today you will find GNU parallel very easy to
+If you use xargs and tee today you will find GNU Parallel very easy to
-use as GNU parallel is written to have the same options as xargs. If
+use as GNU Parallel is written to have the same options as xargs. If
-you write loops in shell, you will find GNU parallel may be able to
+you write loops in shell, you will find GNU Parallel may be able to
 replace most of the loops and make them run faster by running several
 jobs in parallel.
 
-GNU parallel makes sure output from the commands is the same output as
+GNU Parallel makes sure output from the commands is the same output as
 you would get had you run the commands sequentially. This makes it
-possible to use output from GNU parallel as input for other programs.
+possible to use output from GNU Parallel as input for other programs.
 
 
 = Installation =
 
-Installation of GNU Parallel is a simple as:
+Installation of GNU Parallel is as simple as:
 
   ./configure && make && make install
 
-If you are not root you can install in ~/bin and ~/share:
+If you are not root you can add ~/bin to your path and install in
+~/bin and ~/share:
 
   ./configure --prefix=$HOME && make && make install
 
+Or if your system lacks 'make' you can simply copy src/parallel
+src/sem src/niceload src/sql to a dir in your path.
+
+If you just need parallel in a pinch:
+
+  wget http://git.savannah.gnu.org/cgit/parallel.git/plain/src/parallel
+  chmod 755 parallel
+
 After this you should be able to do:
 
   parallel traceroute ::: foss.org.my gnu.org freenetproject.org
@@ -42,7 +51,7 @@ This will run traceroute to 3 different hosts in parallel and print
 the output when the traceroutes are done.
 
 Watch the intro video for a quick introduction:
-http://www.youtube.com/watch?v=OpaiGYxkSuQ or at
+https://www.youtube.com/playlist?list=PL284C9FF2488BC6D1 or at
 http://tinyogg.com/watch/TORaR/ and http://tinyogg.com/watch/hfxKj/
 
 

configure

@@ -1,6 +1,6 @@
 #! /bin/sh
 # Guess values for system-dependent variables and create Makefiles.
-# Generated by GNU Autoconf 2.68 for parallel 20111122.
+# Generated by GNU Autoconf 2.68 for parallel 20111209.
 #
 # Report bugs to <bug-parallel@gnu.org>.
 #
@@ -559,8 +559,8 @@ MAKEFLAGS=
 # Identity of this package.
 PACKAGE_NAME='parallel'
 PACKAGE_TARNAME='parallel'
-PACKAGE_VERSION='20111122'
+PACKAGE_VERSION='20111209'
-PACKAGE_STRING='parallel 20111122'
+PACKAGE_STRING='parallel 20111209'
 PACKAGE_BUGREPORT='bug-parallel@gnu.org'
 PACKAGE_URL=''
 
@@ -1176,7 +1176,7 @@ if test "$ac_init_help" = "long"; then
   # Omit some internal or obsolete options to make the list less imposing.
   # This message is too long to be a string in the A/UX 3.1 sh.
   cat <<_ACEOF
-\`configure' configures parallel 20111122 to adapt to many kinds of systems.
+\`configure' configures parallel 20111209 to adapt to many kinds of systems.
 
 Usage: $0 [OPTION]... [VAR=VALUE]...
 
@@ -1242,7 +1242,7 @@ fi
 
 if test -n "$ac_init_help"; then
   case $ac_init_help in
-     short | recursive ) echo "Configuration of parallel 20111122:";;
+     short | recursive ) echo "Configuration of parallel 20111209:";;
    esac
   cat <<\_ACEOF
 
@@ -1309,7 +1309,7 @@ fi
 test -n "$ac_init_help" && exit $ac_status
 if $ac_init_version; then
   cat <<\_ACEOF
-parallel configure 20111122
+parallel configure 20111209
 generated by GNU Autoconf 2.68
 
 Copyright (C) 2010 Free Software Foundation, Inc.
@@ -1326,7 +1326,7 @@ cat >config.log <<_ACEOF
 This file contains any messages produced by compilers while
 running configure, to aid debugging if configure makes a mistake.
 
-It was created by parallel $as_me 20111122, which was
+It was created by parallel $as_me 20111209, which was
 generated by GNU Autoconf 2.68.  Invocation command line was
 
   $ $0 $@
@@ -2141,7 +2141,7 @@ fi
 
 # Define the identity of the package.
  PACKAGE='parallel'
- VERSION='20111122'
+ VERSION='20111209'
 
 
 cat >>confdefs.h <<_ACEOF
@@ -2704,7 +2704,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
 # report actual input values of CONFIG_FILES etc. instead of their
 # values after options handling.
 ac_log="
-This file was extended by parallel $as_me 20111122, which was
+This file was extended by parallel $as_me 20111209, which was
 generated by GNU Autoconf 2.68.  Invocation command line was
 
   CONFIG_FILES    = $CONFIG_FILES
@@ -2766,7 +2766,7 @@ _ACEOF
 cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
 ac_cs_version="\\
-parallel config.status 20111122
+parallel config.status 20111209
 configured by $0, generated by GNU Autoconf 2.68,
   with options \\"\$ac_cs_config\\"
 

configure.ac

@@ -1,4 +1,4 @@
-AC_INIT([parallel], [20111122], [bug-parallel@gnu.org])
+AC_INIT([parallel], [20111209], [bug-parallel@gnu.org])
 AM_INIT_AUTOMAKE([-Wall -Werror foreign])
 AC_CONFIG_HEADERS([config.h])
 AC_CONFIG_FILES([

doc/FUTURE_IDEAS

@@ -1,3 +1,12 @@
+Luk filen ved EOF - lad være med bare at læse videre.
+
+> /tmp/ged; tail -f /tmp/ged| xargs -n1 -E eof & sleep 1; echo echo a >>/tmp/ged; echo eof >>/tmp/ged; seq 10 >>/tmp/ged; wait
+
+> /tmp/ged; tail -f /tmp/ged| parallel -n1 -E eof & sleep 1; echo echo a >>/tmp/ged; echo eof >>/tmp/ged; seq 10 >>/tmp/ged; wait
+
+
+Fhqwhgads som 20120122 pga 10 års jubliæum.
+
 --timeout auto: If a jobs takes > 3*average runtime then kill it. Only after job 3.
 
 niceload seeks last column:
@@ -182,6 +191,16 @@ sem
 
 single file installation
 
+  ./configure --prefix=$HOME && make && make install
+
+Or if your system lacks 'make' you can simply copy src/parallel
+src/sem src/niceload src/sql to a dir in your path.
+
+wget http://git.savannah.gnu.org/cgit/parallel.git/plain/src/parallel
+
+
+
+
 
 == Video: GNU Parallel 20110522 - The Pakistan Release ==
 

doc/promo

@@ -25,10 +25,10 @@ http://nd.gd/0s http://www.youtube.com/watch?v=OpaiGYxkSuQ
 http://nd.gd/0t http://en.wikipedia.org/wiki/Xargs#The_separator_problem
 http://nd.gd/3k http://www.gnu.org/software/parallel/man.html#differences_between_xargs_and_gnu_parallel
 http://nd.gd/po http://www.gnu.org/software/parallel/man.html#example__distributing_work_to_local_and_remote_computers
+http://nd.gd/039 http://www.youtube.com/playlist?list=PL284C9FF2488BC6D1
 
 If you like xargs you may love GNU Parallel: http://nd.gd/0s
 
-
 With GNU Parallel (http://nd.gd/0s) you can do:
 ls | grep jpeg | parallel mv {} {.}.jpg
 
@@ -55,27 +55,19 @@ problem http://en.wikipedia.org/wiki/Xargs#The_separator_problem
 GNU Parallel http://www.gnu.org/software/parallel/ does not have that
 problem.
 
-Watch the intro video for GNU Parallel:
-http://www.youtube.com/watch?v=OpaiGYxkSuQ
-
 If you have GNU Parallel http://www.gnu.org/software/parallel/ installed you can do this:
 
-Watch the intro video for GNU Parallel to learn more:
+
-http://www.youtube.com/watch?v=OpaiGYxkSuQ
+You can install GNU Parallel simply by:
+
+    wget http://git.savannah.gnu.org/cgit/parallel.git/plain/src/parallel
+    chmod 755 parallel
+    cp parallel sem
+
+Watch the intro videos for GNU Parallel to learn more: http://nd.gd/039
 
 
 GNU Parallel also makes it possible to run small scripts. Try this:
 
-ls *.zip | parallel ‘mkdir {.}; cd {.}; unzip ../{}’
+ls *.zip | parallel 'mkdir {.}; cd {.}; unzip ../{}'
 
-= Other people says =
-
-fawxtin‎: Using GNU Parallel to run things simultaneously, neat! ;)
-cowsandmilk‎: if you haven't quit xargs for gnu parallel yet, you're really missing out
-afader‎: GNU Parallel is cool
-ecyrd‎: Okay, why hasn't anyone told me of GNU parallel before?
-cashion‎: very cool! Did not know about GNU parallel.
-toorghezi‎: Writing shell scripts? You should know about GNU Parallel!
-Racecar564 : GNU Parallel rocks!
-arclight : Very nice! I especially like the {} -> {.}.bz2 syntax.
-nwp8861 : I surprised that it's so comfortable!

doc/release_new_version

@@ -140,7 +140,7 @@ http://freshmeat.net/projects/parallel/releases/new
 == Update Twitter ==
 
 New version of #GNU Parallel released. See what is new in this release
-http://nd.gd/2j Watch the intro video http://nd.gd/0s
+http://nd.gd/2j Watch the intro video http://nd.gd/039
 
 https://savannah.gnu.org/news/?group=parallel
 
@@ -182,9 +182,9 @@ cc:Sandro Cazzaniga <kharec@mandriva.org>,
    Ryoichiro Suzuki <ryoichiro.suzuki@gmail.com>, Jesse Alama
    <jesse.alama@gmail.com>
 
-Subject: GNU Parallel 20111122 ('Silvio') released
+Subject: GNU Parallel 20111222 ('?') released
 
-GNU Parallel 20111122 ('Silvio') has been released. It is
+GNU Parallel 20111122 ('?') has been released. It is
 available for download at: http://ftp.gnu.org/gnu/parallel/
 
 This is a bugfix release with no new features. Probably a good release
@@ -192,44 +192,8 @@ for stable long-term use.
 
 New in this release:
 
-* GNU Parallel package for Fedora.
+* Interview in Hacker Public Radio.
-  https://admin.fedoraproject.org/pkgdb/acls/name/parallel
+  http://hackerpublicradio.org/eps.php?id=0860
-
-* Using GNU Parallel with Mozilla cross referencing.
-  https://blog.mozilla.com/it/2011/11/15/mxr-improvements/
-
-* Neat flash presentation of GNU Parallel.
-  http://prezi.com/0nq74b2elgtx/about-gnu-parallel/
-
-* How to create tar files of dirs in parallel.
-  https://joneslee85.wordpress.com/2011/11/22/shell-tips-how-to-recursively-archive-many-folders/
-
-* Article in Admin Network&Security by Ben Martin.
-  http://www.admin-magazine.com/HPC/Articles/GNU-Parallel
-
-* Blog post in Polish.
-  http://blog.tiger.com.pl/2011/01/05/gnu-parallel/
-
-* Blog post on using GNU Parallel of Mac OS X.
-  http://www.0xcb0.com/2011/10/19/running-parallel-bash-tasks-on-os-x/
-
-* Blog post in English about using zip with GNU Parallel.
-  http://nuclear-imaging.info/site_content/2011/05/11/gnu-parallel/
-
-* Blog post in English by researcher.
-  http://zandyware.wordpress.com/2011/10/15/gnu-parallell-make-best-use-of-your-multicore-computer/
-
-* Blog entry in Chinese.
-  http://blog.csdn.net/tossense/article/details/6889026
-
-* Blog entry in Japanese on managing many servers.
-  http://hiroakis.com/blog/2011/11/13/gnu-parallel/
-
-* GNU Parallel is now part of ppbs.
-  http://physiology.med.cornell.edu/faculty/mason/lab/ppbs/introduction/introduction.html
-
-* How to use GNU Parallel with MCX.
-  http://mcx.sourceforge.net/cgi-bin/index.cgi?MMC/Doc/MMCCluster
 
 * Bug fixes and man page updates.
 

src/niceload

@@ -24,7 +24,7 @@
 use strict;
 use Getopt::Long;
 $Global::progname="niceload";
-$Global::version = 20111122;
+$Global::version = 20111209;
 Getopt::Long::Configure("bundling","require_order");
 get_options_from_array(\@ARGV) || die_usage();
 if($::opt_version) {

src/parallel

@@ -502,7 +502,7 @@ sub get_options_from_array {
 sub parse_options {
     # Returns: N/A
     # Defaults:
-    $Global::version = 20111122;
+    $Global::version = 20111209;
     $Global::progname = 'parallel';
     $Global::infinity = 2**31;
     $Global::debug = 0;
@@ -1691,6 +1691,7 @@ sub usage {
 	 "{3} {3.} {3/} {3/.}  Positional replacement strings",
 	 "",
 	 "-S sshlogin    Example: foo\@server.example.com",
+	 "--slf ..       Use ~/.parallel/sshloginfile as the list of sshlogins",
 	 "--trc {}.bar   Shorthand for --transfer --return {}.bar --cleanup",
 	 "--onall        Run the given command with argument on all sshlogins",
 	 "--nonall       Run the given command with no arguments on all sshlogins",
@@ -3033,19 +3034,44 @@ sub timedout {
 sub kill {
     # kill the jobs
     my $self = shift;
-    my $pid = $self->pid();
+    my @family_pids = $self->family_pids();
-    for my $signal ("TERM", "TERM", "KILL") {
+    # Record this jobs as failed
+    $self->set_exitstatus(1);
     # Send two TERMs to give time to clean up
+    for my $signal ("TERM", "TERM", "KILL") {
+	my $alive = 0;
+	for my $pid (@family_pids) {
 	    if(kill 0, $pid) {
 		# The job still running
 		kill $signal, $pid;
-	    if($signal eq "TERM") { ::usleep(200); }
+		$alive = 1;
-	} else {
-	    # The job is already finished
-	    last;
 	    }
 	}
+	# Wait 200 ms between TERMs - but only if any pids are alive
+	if($signal eq "TERM" and $alive) { ::usleep(200); }
     }
+}
+
+sub family_pids {
+    # Find the pids with this->pid as (grand)*parent
+    # TODO test this on different OS as 'ps' is known to be different
+    my $self = shift;
+    my $pid = $self->pid();
+    my $script = q{
+	family_pids() {
+	    for CHILDPID in `ps --ppid "$@" -o pid --no-headers`; do
+                family_pids $CHILDPID &
+            done
+	    echo "$@"
+	}
+    } .
+    "family_pids $pid; wait";
+    my @pids = qx{$script};
+    chomp(@pids);
+    return ($pid,@pids);
+}
+
+
 sub failed {
     # return number of times failed for this $sshlogin
     my $self = shift;
@@ -3584,7 +3610,14 @@ sub exitstatus {
 sub set_exitstatus {
     my $self = shift;
     my $exitstatus = shift;
+    if($exitstatus) {
+	# Overwrite status if non-zero
 	$self->{'exitstatus'} = $exitstatus;
+    } else {
+	# Set status but do not overwrite
+	# Status may have been set by --timeout
+	$self->{'exitstatus'} ||= $exitstatus;
+    }
 }
 
 sub exitsignal {

src/parallel.pod

@@ -460,18 +460,13 @@ B<--gnu> takes precedence.
 
 =item B<-g> (-g will be retired 20120122)
 
-Group output.  Output from each jobs is grouped together and is only
-printed when the command is finished. stderr (standard error) first
-followed by stdout (standard output).  B<--group> is the default. Can be
-reversed with B<-u>.
-
 Group output. Output from each jobs is grouped together and is only
 printed when the command is finished. stderr (standard error) first
 followed by stdout (standard output). This takes some CPU time. In
 rare situations GNU B<parallel> takes up lots of CPU time and if it is
-acceptable that the output from different commands are mixed together,
+acceptable that the outputs from different commands are mixed
-then disabling grouping with B<-u> can speedup GNU B<parallel> by a
+together, then disabling grouping with B<-u> can speedup GNU
-factor of 10.
+B<parallel> by a factor of 10.
 
 B<--group> is the default. Can be reversed with B<-u>.
 
@@ -902,8 +897,8 @@ If B<--recstart> is given I<startstring> will be used to split at record start.
 
 If B<--recend> is given I<endstring> will be used to split at record end.
 
-If both B<--recstart> and B<--recend> are given the string
+If both B<--recstart> and B<--recend> are given the combined string
-I<startstring>I<endstring> will have to match to find a split
+I<endstring>I<startstring> will have to match to find a split
 position. This is useful if either I<startstring> or I<endstring>
 match in the middle of a record.
 

src/sql

@@ -556,7 +556,7 @@ $Global::Initfile && unlink $Global::Initfile;
 exit ($err);
 
 sub parse_options {
-    $Global::version = 20111122;
+    $Global::version = 20111209;
     $Global::progname = 'sql';
 
     # This must be done first as this may exec myself

testsuite/tests-to-run/test15.sh

@@ -87,6 +87,9 @@ echo '### Test -x'
 echo '### Test -a and --arg-file: Read input from file instead of stdin'
 seq 1 10 >/tmp/$$-1; parallel -k -a /tmp/$$-1 echo
 seq 1 10 >/tmp/$$-2; parallel -k --arg-file /tmp/$$-2 echo
+
+echo '### Test killing children with --timeout and exit value (failed if timed out)'
+pstree |grep sleep | wc; parallel --timeout 3 'true {} ; for i in `seq 100 120`; do bash -c "(sleep $i)" & sleep $i & done; wait; echo No good' ::: 1000000000 1000000001 ; echo $?; pstree |grep sleep | wc
 EOF
 
 #echo '### Test bugfix if no command given'

testsuite/wanted-results/test15

@@ -172,6 +172,10 @@ xargs: argument line too long
 8
 9
 10
+### Test killing children with --timeout and exit value (failed if timed out)
+      0       0       0
+2
+      0       0       0
 xargs Expect: 3 1 2
 3
 1
@@ -288,8 +292,8 @@ line 2
 ### Test --no-run-if-empty and -r: This should give no output
 ### Test --help and -h: Help output (just check we get the same amount of lines)
 Output from -h and --help
-27
+28
-27
+28
 ### Test --version: Version output (just check we get the same amount of lines)
 12
 ### Test --verbose and -t