From e06f6d52ce7b08289e1e11c944abe5c9ef555805 Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Wed, 22 Sep 2010 01:17:05 +0200 Subject: [PATCH] parallel: Added difference between paexec --- configure | 20 +++--- configure.ac | 2 +- doc/FUTURE_IDEAS | 17 +----- doc/release_new_version | 3 +- packager/obs/Makefile | 7 +++ packager/obs/README | 1 + src/parallel | 105 +++++++++++++++++++++++++------- src/sql | 2 +- unittest/tests-to-run/test32.sh | 20 ++++++ 9 files changed, 127 insertions(+), 50 deletions(-) create mode 100644 packager/obs/Makefile create mode 100644 packager/obs/README create mode 100644 unittest/tests-to-run/test32.sh diff --git a/configure b/configure index fed30878..e28121ce 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.67 for parallel 20100906. +# Generated by GNU Autoconf 2.67 for parallel 20100922. # # Report bugs to . # @@ -551,8 +551,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='parallel' PACKAGE_TARNAME='parallel' -PACKAGE_VERSION='20100906' -PACKAGE_STRING='parallel 20100906' +PACKAGE_VERSION='20100922' +PACKAGE_STRING='parallel 20100922' PACKAGE_BUGREPORT='bug-parallel@gnu.org' PACKAGE_URL='' @@ -1168,7 +1168,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures parallel 20100906 to adapt to many kinds of systems. +\`configure' configures parallel 20100922 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1234,7 +1234,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of parallel 20100906:";; + short | recursive ) echo "Configuration of parallel 20100922:";; esac cat <<\_ACEOF @@ -1301,7 +1301,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -parallel configure 20100906 +parallel configure 20100922 generated by GNU Autoconf 2.67 Copyright (C) 2010 Free Software Foundation, Inc. @@ -1318,7 +1318,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by parallel $as_me 20100906, which was +It was created by parallel $as_me 20100922, which was generated by GNU Autoconf 2.67. Invocation command line was $ $0 $@ @@ -2133,7 +2133,7 @@ fi # Define the identity of the package. PACKAGE='parallel' - VERSION='20100906' + VERSION='20100922' cat >>confdefs.h <<_ACEOF @@ -2684,7 +2684,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by parallel $as_me 20100906, which was +This file was extended by parallel $as_me 20100922, which was generated by GNU Autoconf 2.67. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -2746,7 +2746,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -parallel config.status 20100906 +parallel config.status 20100922 configured by $0, generated by GNU Autoconf 2.67, with options \\"\$ac_cs_config\\" diff --git a/configure.ac b/configure.ac index 1c449a3f..e28b5fc0 100644 --- a/configure.ac +++ b/configure.ac @@ -1,4 +1,4 @@ -AC_INIT([parallel], [20100906], [bug-parallel@gnu.org]) +AC_INIT([parallel], [20100922], [bug-parallel@gnu.org]) AM_INIT_AUTOMAKE([-Wall -Werror foreign]) AC_CONFIG_HEADERS([config.h]) AC_CONFIG_FILES([ diff --git a/doc/FUTURE_IDEAS b/doc/FUTURE_IDEAS index 18567333..b0066a0a 100644 --- a/doc/FUTURE_IDEAS +++ b/doc/FUTURE_IDEAS @@ -1,25 +1,10 @@ -== Retry == - -Bugfix: --keeporder with -j100% mixed up order of first job -parallel: Unittest for --retries - -== FSCONS == - -Annoncer foredrag til FSCONS - -I will be at FSCONS 2010-11-07 talking about GNU Parallel. -http://www.fscons.org/fs/gnu-parallel - == SQL == Example with %0a as newline -sql :pg_foo?'\dt %0a SELECT * FROM users' +sql :my_postgres?'\dt %0a SELECT * FROM users' cat ~/.sql/aliases | parallel --colsep '\s' sql {1} '"select 0.14159+3;" | grep -q 3.14159 || (echo dead: {1}; exit 1)' ---list-tables -select OBJECT_NAME from user_objects where object_type = 'TABLE'; - == FEX == fex syntax for splitting fields diff --git a/doc/release_new_version b/doc/release_new_version index f8a0f45b..61417c68 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -44,7 +44,7 @@ cp parallel-$YYYYMMDD.tar.bz2 /tmp cd /tmp tar xjvf parallel-$YYYYMMDD.tar.bz2 cd parallel-$YYYYMMDD -./configure && make && sudo make install +./configure && make -j && sudo make -j install == Upload == @@ -70,6 +70,7 @@ make -j && sudo make -j install == Update OpenSUSE build system == https://build.opensuse.org/package/show?package=parallel&project=home%3Atange +cd ~/privat/parallel/packager/obs ; make == Update Savannah == diff --git a/packager/obs/Makefile b/packager/obs/Makefile new file mode 100644 index 00000000..58c925ba --- /dev/null +++ b/packager/obs/Makefile @@ -0,0 +1,7 @@ +all: +# cd ../debian/ && make +# cp ../debian/parallel_*.dsc ../debian/parallel_*.tar.gz home\:tange/parallel/ + cp `ls ../../parallel-*.tar.bz2|tail -n1` home\:tange/parallel/ + cd home\:tange/parallel/ && osc up && osc add *.dsc *.tar.gz && osc ci + + diff --git a/packager/obs/README b/packager/obs/README new file mode 100644 index 00000000..891e733b --- /dev/null +++ b/packager/obs/README @@ -0,0 +1 @@ +https://build.opensuse.org/package/show?package=parallel&project=home%3Atange diff --git a/src/parallel b/src/parallel index 8ae04f16..79a6f3cf 100755 --- a/src/parallel +++ b/src/parallel @@ -1542,8 +1542,8 @@ Outputs O2. Send stderr to stderr O3. Send stdout to stdout O4. Order of output can be same as order of input - O5. Stdout only from the command - O6. Stderr only from the command + O5. Stdout only contains stdout from the command + O6. Stderr only contains stdout from the command Inputs I1. Arguments can be read from stdin @@ -1571,6 +1571,7 @@ Remote execution R6. No config files needed R7. Do not run more than SSHD's MaxStartup can handle R8. Configurable SSH command + R9. Retry if connection breaks occationally Semaphore S1. Possibility to work as a mutex @@ -1582,37 +1583,47 @@ Legend ID = yes As every new version of the programs are not tested the table may be -outdated. Please file a bug-report if you find errors. +outdated. Please file a bug-report if you find errors (See REPORTING +BUGS). parallel: E1 E2 E3 E4 E5 O1 O2 O3 O4 O5 O6 I1 I2 I3 I4 I5 I6 I7 M1 M2 -M3 M4 M5 M6 R1 R2 R3 R4 R5 R6 R7 R8 S1 S2 +M3 M4 M5 M6 R1 R2 R3 R4 R5 R6 R7 R8 R9 S1 S2 xargs: E1 - - - - - O2 O3 - O5 O6 I1 I2 - - - - - - M2 M3 - - - - - - - - - x - - - + - - - - - - - x - - - - find -exec: - - - x - x O2 O3 O4 O5 O6 - - - - - - - - M2 M3 - - - - - - - - - - - x x + - - - - - - - - - - x x make -j: E1 - - - E5 O1 O2 O3 - x O6 - - - - - - - - - - - - - - - - - - -- - - +- - - - ppss: E1 E2 ?E3 E4 - O1 - - x - - I1 I2 - - - - I7 M1 - M3 - - M6 R1 R2 - R3 R4 - - ?R7 - - + R3 R4 - - ?R7 ? ? - - pexec: E1 - - E4 - O1 O2 O3 - O5 O6 I1 I2 - I4 I5 - - M1 - M3 - - M6 R1 - - - - - R6 - S1 - + - - - - R6 - - - S1 - -xjobs: TODO +xjobs: TODO - Please file a bug-report if you know what features xjobs +supports (See REPORTING BUGS). -prll: TODO +prll: TODO - Please file a bug-report if you know what features prll +supports (See REPORTING BUGS). -dxargs: TODO +dxargs: TODO - Please file a bug-report if you know what features dxargs +supports (See REPORTING BUGS). -mdm/middelman: TODO +mdm/middelman: TODO - Please file a bug-report if you know what +features mdm/middelman supports (See REPORTING BUGS). -xapply: TODO +xapply: TODO - Please file a bug-report if you know what features xapply +supports (See REPORTING BUGS). -ClusterSSH: TODO +paexec: TODO - Please file a bug-report if you know what features paexec +supports (See REPORTING BUGS). + +ClusterSSH: TODO - Please file a bug-report if you know what features ClusterSSH +supports (See REPORTING BUGS). =head2 DIFFERENCES BETWEEN xargs AND GNU Parallel @@ -1876,7 +1887,7 @@ echo unzip 1.zip >> /var/run/my_named_pipe; echo tar cf /backup/myhome.tar /home/me >> /var/run/my_named_pipe -=head2 DIFFERENCES BETWEEN prll AND GNU parallel +=head2 DIFFERENCES BETWEEN prll AND GNU Parallel B is also a tool for running jobs in parallel. It does not support running jobs on remote computers. @@ -1979,6 +1990,51 @@ B<11> xapply -f '[ -f %1 ] && echo %1' List | ... B<11> parallel '[ -f {} ] && echo {}' < List | ... +=head2 DIFFERENCES BETWEEN paexec AND GNU Parallel + +B can run jobs in parallel on both the local and remote computers. + +B requires commands to print a blank line as the last +output. This means you will have to write a wrapper for most programs. + +B has a job dependency facility so a job can depend on another +job to be executed successfully. Sort of a poor-man's B. + +Here are the examples from B's example catalog with the equivalent +using GNU B: + +=over 1 + +=item 1_div_X_run: + + ../../paexec -s -l -c "`pwd`/1_div_X_cmd" -n +1 <. @@ -1993,7 +2049,7 @@ computing. If more than one machine is listed in B<-S> GNU B may only use one of these (e.g. if there are 8 jobs to be run and one machine has 8 cores). -GNU B can be used as a poor-mans version of ClusterSSH: +GNU B can be used as a poor-man's version of ClusterSSH: B @@ -2012,9 +2068,16 @@ echo 1,2,3 | parallel -vkd, echo a{} =head2 Startup speed -GNU B is slow at starting up. Half of the startup time is -spent finding the maximal length of a command line. Setting B<-s> will -remove this part of the startup time. +GNU B is slow at starting up. Half of the startup time on +the local computer is spent finding the maximal length of a command +line. Setting B<-s> will remove this part of the startup time. + +When using multiple computers GNU B opens B connections +to them to figure out how many connections can be used reliably +simultaneously (Namely SSHD's MaxStartup). This test is done for each +host in serial, so if your --sshloginfile contains many hosts it may +be slow. + =head1 REPORTING BUGS @@ -2215,7 +2278,7 @@ sub acquire_semaphore { sub parse_options { # Returns: N/A # Defaults: - $Global::version = 20100906; + $Global::version = 20100922; $Global::progname = 'parallel'; $Global::debug = 0; $Global::verbose = 0; diff --git a/src/sql b/src/sql index ee06af9d..f5478e9d 100755 --- a/src/sql +++ b/src/sql @@ -452,7 +452,7 @@ $Global::Initfile && unlink $Global::Initfile; exit ($err); sub parse_options { - $Global::version = 20100914; + $Global::version = 20100922; $Global::progname = 'sql'; # This must be done first as this may exec myself diff --git a/unittest/tests-to-run/test32.sh b/unittest/tests-to-run/test32.sh new file mode 100644 index 00000000..50e6d635 --- /dev/null +++ b/unittest/tests-to-run/test32.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +echo '### Test of --retries' +seq 1 10 | parallel --retries 2 -v -S 4.3.2.1,: echo + +echo '### Test of --retries - it should run 13 jobs in total' +seq 0 12 | parallel --progress -kj100% --retries 1 -S 12/nlv.pi.dk,1/:,parallel@server2 -vq \ + perl -e 'sleep 1;print "job{}\n";exit({})' | tail -n1 | \ + perl -ne '@a=(split /\//,$_); print $a[1]+$a[4]+$a[7],"\n"' + +echo '### Test of --retries - it should run 25 jobs in total' +seq 0 12 | parallel --progress -kj100% --retries 2 -S 12/nlv.pi.dk,1/:,parallel@server2 -vq \ + perl -e 'sleep 1;print "job{}\n";exit({})' | tail -n1 | \ + perl -ne '@a=(split /\//,$_); print $a[1]+$a[4]+$a[7],"\n"' + +echo '### Test of --retries - it should run 49 jobs in total' +seq 0 12 | parallel --progress -kj100% --retries 4 -S 12/nlv.pi.dk,1/:,parallel@server2 -vq \ + perl -e 'sleep 1;print "job{}\n";exit({})' | tail -n1 | \ + perl -ne '@a=(split /\//,$_); print $a[1]+$a[4]+$a[7],"\n"' +