parallel: Added difference between paexec

configure

@@ -1,6 +1,6 @@
 #! /bin/sh
 # Guess values for system-dependent variables and create Makefiles.
-# Generated by GNU Autoconf 2.67 for parallel 20100906.
+# Generated by GNU Autoconf 2.67 for parallel 20100922.
 #
 # Report bugs to <bug-parallel@gnu.org>.
 #
@@ -551,8 +551,8 @@ MAKEFLAGS=
 # Identity of this package.
 PACKAGE_NAME='parallel'
 PACKAGE_TARNAME='parallel'
-PACKAGE_VERSION='20100906'
+PACKAGE_VERSION='20100922'
-PACKAGE_STRING='parallel 20100906'
+PACKAGE_STRING='parallel 20100922'
 PACKAGE_BUGREPORT='bug-parallel@gnu.org'
 PACKAGE_URL=''
 
@@ -1168,7 +1168,7 @@ if test "$ac_init_help" = "long"; then
   # Omit some internal or obsolete options to make the list less imposing.
   # This message is too long to be a string in the A/UX 3.1 sh.
   cat <<_ACEOF
-\`configure' configures parallel 20100906 to adapt to many kinds of systems.
+\`configure' configures parallel 20100922 to adapt to many kinds of systems.
 
 Usage: $0 [OPTION]... [VAR=VALUE]...
 
@@ -1234,7 +1234,7 @@ fi
 
 if test -n "$ac_init_help"; then
   case $ac_init_help in
-     short | recursive ) echo "Configuration of parallel 20100906:";;
+     short | recursive ) echo "Configuration of parallel 20100922:";;
    esac
   cat <<\_ACEOF
 
@@ -1301,7 +1301,7 @@ fi
 test -n "$ac_init_help" && exit $ac_status
 if $ac_init_version; then
   cat <<\_ACEOF
-parallel configure 20100906
+parallel configure 20100922
 generated by GNU Autoconf 2.67
 
 Copyright (C) 2010 Free Software Foundation, Inc.
@@ -1318,7 +1318,7 @@ cat >config.log <<_ACEOF
 This file contains any messages produced by compilers while
 running configure, to aid debugging if configure makes a mistake.
 
-It was created by parallel $as_me 20100906, which was
+It was created by parallel $as_me 20100922, which was
 generated by GNU Autoconf 2.67.  Invocation command line was
 
   $ $0 $@
@@ -2133,7 +2133,7 @@ fi
 
 # Define the identity of the package.
  PACKAGE='parallel'
- VERSION='20100906'
+ VERSION='20100922'
 
 
 cat >>confdefs.h <<_ACEOF
@@ -2684,7 +2684,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
 # report actual input values of CONFIG_FILES etc. instead of their
 # values after options handling.
 ac_log="
-This file was extended by parallel $as_me 20100906, which was
+This file was extended by parallel $as_me 20100922, which was
 generated by GNU Autoconf 2.67.  Invocation command line was
 
   CONFIG_FILES    = $CONFIG_FILES
@@ -2746,7 +2746,7 @@ _ACEOF
 cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
 ac_cs_version="\\
-parallel config.status 20100906
+parallel config.status 20100922
 configured by $0, generated by GNU Autoconf 2.67,
   with options \\"\$ac_cs_config\\"
 

configure.ac

@@ -1,4 +1,4 @@
-AC_INIT([parallel], [20100906], [bug-parallel@gnu.org])
+AC_INIT([parallel], [20100922], [bug-parallel@gnu.org])
 AM_INIT_AUTOMAKE([-Wall -Werror foreign])
 AC_CONFIG_HEADERS([config.h])
 AC_CONFIG_FILES([

doc/FUTURE_IDEAS

@@ -1,25 +1,10 @@
-== Retry ==
-
-Bugfix: --keeporder with -j100% mixed up order of first job
-parallel: Unittest for --retries
-
-== FSCONS ==
-
-Annoncer foredrag til FSCONS
-
-I will be at FSCONS 2010-11-07 talking about GNU Parallel.
-http://www.fscons.org/fs/gnu-parallel
-
 == SQL ==
 
 Example with %0a as newline
-sql :pg_foo?'\dt %0a SELECT * FROM users'
+sql :my_postgres?'\dt %0a SELECT * FROM users'
 
 cat ~/.sql/aliases | parallel --colsep '\s' sql {1} '"select 0.14159+3;" | grep -q 3.14159 || (echo dead: {1}; exit 1)'
 
---list-tables
-select OBJECT_NAME from user_objects where object_type = 'TABLE'; 
-
 == FEX ==
 
 fex syntax for splitting fields

doc/release_new_version

@@ -44,7 +44,7 @@ cp parallel-$YYYYMMDD.tar.bz2 /tmp
 cd /tmp
 tar xjvf parallel-$YYYYMMDD.tar.bz2
 cd parallel-$YYYYMMDD
-./configure && make && sudo make install
+./configure && make -j && sudo make -j install
 
 == Upload ==
 
@@ -70,6 +70,7 @@ make -j && sudo make -j install
 == Update OpenSUSE build system ==
 
 https://build.opensuse.org/package/show?package=parallel&project=home%3Atange
+cd ~/privat/parallel/packager/obs ; make
 
 == Update Savannah ==
 

packager/obs/Makefile

@@ -0,0 +1,7 @@
+all:
+#	cd ../debian/ && make
+#	cp ../debian/parallel_*.dsc ../debian/parallel_*.tar.gz home\:tange/parallel/
+	cp `ls ../../parallel-*.tar.bz2|tail -n1` home\:tange/parallel/
+	cd home\:tange/parallel/ && osc up && osc add *.dsc *.tar.gz && osc ci
+
+

packager/obs/README

@@ -0,0 +1 @@
+https://build.opensuse.org/package/show?package=parallel&project=home%3Atange

src/parallel

@@ -1542,8 +1542,8 @@ Outputs
  O2. Send stderr to stderr
  O3. Send stdout to stdout
  O4. Order of output can be same as order of input
- O5. Stdout only from the command
+ O5. Stdout only contains stdout from the command
- O6. Stderr only from the command
+ O6. Stderr only contains stdout from the command
 
 Inputs
  I1. Arguments can be read from stdin
@@ -1571,6 +1571,7 @@ Remote execution
  R6. No config files needed
  R7. Do not run more than SSHD's MaxStartup can handle
  R8. Configurable SSH command
+ R9. Retry if connection breaks occationally
 
 Semaphore
  S1. Possibility to work as a mutex
@@ -1582,37 +1583,47 @@ Legend
  ID = yes
 
 As every new version of the programs are not tested the table may be
-outdated. Please file a bug-report if you find errors.
+outdated. Please file a bug-report if you find errors (See REPORTING
+BUGS).
 
 parallel: E1 E2 E3 E4 E5 O1 O2 O3 O4 O5 O6 I1 I2 I3 I4 I5 I6 I7 M1 M2
-M3 M4 M5 M6 R1 R2 R3 R4 R5 R6 R7 R8 S1 S2
+M3 M4 M5 M6 R1 R2 R3 R4 R5 R6 R7 R8 R9 S1 S2
 
 xargs:    E1  -  -  -  -  - O2 O3  - O5 O6 I1 I2  -  -  -  -  - - M2 M3  -
- -  -  -  -  -  -  -  x  -  -  -
+ -  -  -  -  -  -  -  x  -  -  - -
 
 find -exec: - -  -  x  -  x O2 O3 O4 O5 O6  -  -  -  -  -  -  - - M2 M3  -
- -  -  -  -  -  -  -  -  -  x  x
+ -  -  -  -  -  -  -  -  -  - x  x
 
 make -j: E1 - - - E5 O1 O2 O3 - x O6 - - - - - - - - - - - - - - - - - -
-- - -
+- - - -
 
 ppss: E1 E2 ?E3 E4 - O1 - - x - - I1 I2 - - - - I7 M1 - M3 - - M6 R1 R2
- R3 R4 - - ?R7 - -
+ R3 R4 - - ?R7 ? ? - -
 
 pexec: E1 - - E4 - O1 O2 O3 - O5 O6 I1 I2 - I4 I5 - - M1 - M3 - - M6 R1
- - - - - R6 - S1 -
+ - - - - R6 - - - S1 -
 
-xjobs: TODO
+xjobs: TODO - Please file a bug-report if you know what features xjobs
+supports (See REPORTING BUGS).
 
-prll: TODO
+prll: TODO - Please file a bug-report if you know what features prll
+supports (See REPORTING BUGS).
 
-dxargs: TODO
+dxargs: TODO - Please file a bug-report if you know what features dxargs
+supports (See REPORTING BUGS).
 
-mdm/middelman: TODO
+mdm/middelman: TODO - Please file a bug-report if you know what
+features mdm/middelman supports (See REPORTING BUGS).
 
-xapply: TODO
+xapply: TODO - Please file a bug-report if you know what features xapply
+supports (See REPORTING BUGS).
 
-ClusterSSH: TODO
+paexec: TODO - Please file a bug-report if you know what features paexec
+supports (See REPORTING BUGS).
+
+ClusterSSH: TODO - Please file a bug-report if you know what features ClusterSSH
+supports (See REPORTING BUGS).
 
 
 =head2 DIFFERENCES BETWEEN xargs AND GNU Parallel
@@ -1876,7 +1887,7 @@ echo unzip 1.zip >> /var/run/my_named_pipe;
 echo tar cf /backup/myhome.tar /home/me >> /var/run/my_named_pipe
 
 
-=head2 DIFFERENCES BETWEEN prll AND GNU parallel
+=head2 DIFFERENCES BETWEEN prll AND GNU Parallel
 
 B<prll> is also a tool for running jobs in parallel. It does not
 support running jobs on remote computers.
@@ -1979,6 +1990,51 @@ B<11> xapply -f '[ -f %1 ] && echo %1' List | ...
 B<11> parallel '[ -f {} ] && echo {}' < List | ...
 
 
+=head2 DIFFERENCES BETWEEN paexec AND GNU Parallel
+
+B<paexec> can run jobs in parallel on both the local and remote computers.
+
+B<paexec> requires commands to print a blank line as the last
+output. This means you will have to write a wrapper for most programs.
+
+B<paexec> has a job dependency facility so a job can depend on another
+job to be executed successfully. Sort of a poor-man's B<make>.
+
+Here are the examples from B<paexec>'s example catalog with the equivalent
+using GNU B<parallel>:
+
+=over 1
+
+=item 1_div_X_run:
+
+  ../../paexec -s -l -c "`pwd`/1_div_X_cmd" -n +1 <<EOF [...]
+  parallel echo {} '|' `pwd`/1_div_X_cmd <<EOF [...]
+
+=item all_substr_run:
+
+  ../../paexec -lp -c "`pwd`/all_substr_cmd" -n +3 <<EOF [...]
+  parallel echo {} '|' `pwd`/all_substr_cmd <<EOF [...]
+
+=item cc_wrapper_run:
+
+  ../../paexec -c "env CC=gcc CFLAGS=-O2 `pwd`/cc_wrapper_cmd" \
+             -n 'host1 host2' \
+             -t '/usr/bin/ssh -x' <<EOF [...]
+  parallel echo {} '|' "env CC=gcc CFLAGS=-O2 `pwd`/cc_wrapper_cmd" \
+             -S host1,host2 <<EOF [...]
+  # This is not exactly the same, but avoids the wrapper
+  parallel gcc -O2 -c -o {.}.o {} \
+             -S host1,host2 <<EOF [...]
+
+=item toupper_run:
+
+  ../../paexec -lp -c "`pwd`/toupper_cmd" -n +10 <<EOF [...]
+  parallel echo {} '|' ./toupper_cmd <<EOF [...]
+  # Without the wrapper:
+  parallel echo {} '| awk {print\ toupper\(\$0\)}' <<EOF [...]
+
+=back
+
 =head2 DIFFERENCES BETWEEN ClusterSSH AND GNU Parallel
 
 ClusterSSH solves a different problem than GNU B<parallel>.
@@ -1993,7 +2049,7 @@ computing. If more than one machine is listed in B<-S> GNU B<parallel> may
 only use one of these (e.g. if there are 8 jobs to be run and one
 machine has 8 cores).
 
-GNU B<parallel> can be used as a poor-mans version of ClusterSSH:
+GNU B<parallel> can be used as a poor-man's version of ClusterSSH:
 
 B<cat hostlist | parallel ssh {} do_stuff>
 
@@ -2012,9 +2068,16 @@ echo 1,2,3 | parallel -vkd, echo a{}
 
 =head2 Startup speed
 
-GNU B<parallel> is slow at starting up. Half of the startup time is
+GNU B<parallel> is slow at starting up. Half of the startup time on
-spent finding the maximal length of a command line. Setting B<-s> will
+the local computer is spent finding the maximal length of a command
-remove this part of the startup time.
+line. Setting B<-s> will remove this part of the startup time.
+
+When using multiple computers GNU B<parallel> opens B<ssh> connections
+to them to figure out how many connections can be used reliably
+simultaneously (Namely SSHD's MaxStartup). This test is done for each
+host in serial, so if your --sshloginfile contains many hosts it may
+be slow.
+
 
 =head1 REPORTING BUGS
 
@@ -2215,7 +2278,7 @@ sub acquire_semaphore {
 sub parse_options {
     # Returns: N/A
     # Defaults:
-    $Global::version = 20100906;
+    $Global::version = 20100922;
     $Global::progname = 'parallel';
     $Global::debug = 0;
     $Global::verbose = 0;

src/sql

@@ -452,7 +452,7 @@ $Global::Initfile && unlink $Global::Initfile;
 exit ($err);
 
 sub parse_options {
-    $Global::version = 20100914;
+    $Global::version = 20100922;
     $Global::progname = 'sql';
 
     # This must be done first as this may exec myself

unittest/tests-to-run/test32.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+echo '### Test of --retries'
+seq 1 10 | parallel --retries 2 -v -S 4.3.2.1,: echo
+
+echo '### Test of --retries - it should run 13 jobs in total'
+seq 0 12 | parallel --progress -kj100% --retries 1 -S 12/nlv.pi.dk,1/:,parallel@server2 -vq \
+ perl -e 'sleep 1;print "job{}\n";exit({})' | tail -n1 | \
+ perl -ne '@a=(split /\//,$_); print $a[1]+$a[4]+$a[7],"\n"'
+
+echo '### Test of --retries - it should run 25 jobs in total'
+seq 0 12 | parallel --progress -kj100% --retries 2 -S 12/nlv.pi.dk,1/:,parallel@server2 -vq \
+ perl -e 'sleep 1;print "job{}\n";exit({})' | tail -n1 | \
+ perl -ne '@a=(split /\//,$_); print $a[1]+$a[4]+$a[7],"\n"'
+
+echo '### Test of --retries - it should run 49 jobs in total'
+seq 0 12 | parallel --progress -kj100% --retries 4 -S 12/nlv.pi.dk,1/:,parallel@server2 -vq \
+ perl -e 'sleep 1;print "job{}\n";exit({})' | tail -n1 | \
+ perl -ne '@a=(split /\//,$_); print $a[1]+$a[4]+$a[7],"\n"'
+