diff --git a/autom4te.cache/output.0 b/autom4te.cache/output.0 index 62d8b0f4..2812890a 100644 --- a/autom4te.cache/output.0 +++ b/autom4te.cache/output.0 @@ -1,6 +1,6 @@ @%:@! /bin/sh @%:@ Guess values for system-dependent variables and create Makefiles. -@%:@ Generated by GNU Autoconf 2.65 for parallel 20100428. +@%:@ Generated by GNU Autoconf 2.65 for parallel 20100601. @%:@ @%:@ Report bugs to . @%:@ @@ -551,8 +551,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='parallel' PACKAGE_TARNAME='parallel' -PACKAGE_VERSION='20100428' -PACKAGE_STRING='parallel 20100428' +PACKAGE_VERSION='20100601' +PACKAGE_STRING='parallel 20100601' PACKAGE_BUGREPORT='bug-parallel@gnu.org' PACKAGE_URL='' @@ -1167,7 +1167,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures parallel 20100428 to adapt to many kinds of systems. +\`configure' configures parallel 20100601 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1233,7 +1233,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of parallel 20100428:";; + short | recursive ) echo "Configuration of parallel 20100601:";; esac cat <<\_ACEOF @@ -1300,7 +1300,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -parallel configure 20100428 +parallel configure 20100601 generated by GNU Autoconf 2.65 Copyright (C) 2009 Free Software Foundation, Inc. @@ -1317,7 +1317,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by parallel $as_me 20100428, which was +It was created by parallel $as_me 20100601, which was generated by GNU Autoconf 2.65. Invocation command line was $ $0 $@ @@ -2125,7 +2125,7 @@ fi # Define the identity of the package. PACKAGE='parallel' - VERSION='20100428' + VERSION='20100601' cat >>confdefs.h <<_ACEOF @@ -2675,7 +2675,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by parallel $as_me 20100428, which was +This file was extended by parallel $as_me 20100601, which was generated by GNU Autoconf 2.65. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -2737,7 +2737,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -parallel config.status 20100428 +parallel config.status 20100601 configured by $0, generated by GNU Autoconf 2.65, with options \\"\$ac_cs_config\\" diff --git a/autom4te.cache/output.1 b/autom4te.cache/output.1 index 62d8b0f4..2812890a 100644 --- a/autom4te.cache/output.1 +++ b/autom4te.cache/output.1 @@ -1,6 +1,6 @@ @%:@! /bin/sh @%:@ Guess values for system-dependent variables and create Makefiles. -@%:@ Generated by GNU Autoconf 2.65 for parallel 20100428. +@%:@ Generated by GNU Autoconf 2.65 for parallel 20100601. @%:@ @%:@ Report bugs to . @%:@ @@ -551,8 +551,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='parallel' PACKAGE_TARNAME='parallel' -PACKAGE_VERSION='20100428' -PACKAGE_STRING='parallel 20100428' +PACKAGE_VERSION='20100601' +PACKAGE_STRING='parallel 20100601' PACKAGE_BUGREPORT='bug-parallel@gnu.org' PACKAGE_URL='' @@ -1167,7 +1167,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures parallel 20100428 to adapt to many kinds of systems. +\`configure' configures parallel 20100601 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1233,7 +1233,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of parallel 20100428:";; + short | recursive ) echo "Configuration of parallel 20100601:";; esac cat <<\_ACEOF @@ -1300,7 +1300,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -parallel configure 20100428 +parallel configure 20100601 generated by GNU Autoconf 2.65 Copyright (C) 2009 Free Software Foundation, Inc. @@ -1317,7 +1317,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by parallel $as_me 20100428, which was +It was created by parallel $as_me 20100601, which was generated by GNU Autoconf 2.65. Invocation command line was $ $0 $@ @@ -2125,7 +2125,7 @@ fi # Define the identity of the package. PACKAGE='parallel' - VERSION='20100428' + VERSION='20100601' cat >>confdefs.h <<_ACEOF @@ -2675,7 +2675,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by parallel $as_me 20100428, which was +This file was extended by parallel $as_me 20100601, which was generated by GNU Autoconf 2.65. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -2737,7 +2737,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -parallel config.status 20100428 +parallel config.status 20100601 configured by $0, generated by GNU Autoconf 2.65, with options \\"\$ac_cs_config\\" diff --git a/autom4te.cache/traces.1 b/autom4te.cache/traces.1 index ab012615..5c12546e 100644 --- a/autom4te.cache/traces.1 +++ b/autom4te.cache/traces.1 @@ -1,4 +1,4 @@ -m4trace:configure.ac:1: -1- AC_INIT([parallel], [20100428], [bug-parallel@gnu.org]) +m4trace:configure.ac:1: -1- AC_INIT([parallel], [20100601], [bug-parallel@gnu.org]) m4trace:configure.ac:1: -1- m4_pattern_forbid([^_?A[CHUM]_]) m4trace:configure.ac:1: -1- m4_pattern_forbid([_AC_]) m4trace:configure.ac:1: -1- m4_pattern_forbid([^LIBOBJS$], [do not use LIBOBJS directly, use AC_LIBOBJ (see section `AC_LIBOBJ vs LIBOBJS']) diff --git a/configure b/configure index b912d353..188dc472 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.65 for parallel 20100428. +# Generated by GNU Autoconf 2.65 for parallel 20100601. # # Report bugs to . # @@ -551,8 +551,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='parallel' PACKAGE_TARNAME='parallel' -PACKAGE_VERSION='20100428' -PACKAGE_STRING='parallel 20100428' +PACKAGE_VERSION='20100601' +PACKAGE_STRING='parallel 20100601' PACKAGE_BUGREPORT='bug-parallel@gnu.org' PACKAGE_URL='' @@ -1167,7 +1167,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures parallel 20100428 to adapt to many kinds of systems. +\`configure' configures parallel 20100601 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1233,7 +1233,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of parallel 20100428:";; + short | recursive ) echo "Configuration of parallel 20100601:";; esac cat <<\_ACEOF @@ -1300,7 +1300,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -parallel configure 20100428 +parallel configure 20100601 generated by GNU Autoconf 2.65 Copyright (C) 2009 Free Software Foundation, Inc. @@ -1317,7 +1317,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by parallel $as_me 20100428, which was +It was created by parallel $as_me 20100601, which was generated by GNU Autoconf 2.65. Invocation command line was $ $0 $@ @@ -2125,7 +2125,7 @@ fi # Define the identity of the package. PACKAGE='parallel' - VERSION='20100428' + VERSION='20100601' cat >>confdefs.h <<_ACEOF @@ -2675,7 +2675,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by parallel $as_me 20100428, which was +This file was extended by parallel $as_me 20100601, which was generated by GNU Autoconf 2.65. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -2737,7 +2737,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -parallel config.status 20100428 +parallel config.status 20100601 configured by $0, generated by GNU Autoconf 2.65, with options \\"\$ac_cs_config\\" diff --git a/configure.ac b/configure.ac index 5a29d391..2fef886c 100644 --- a/configure.ac +++ b/configure.ac @@ -1,4 +1,4 @@ -AC_INIT([parallel], [20100428], [bug-parallel@gnu.org]) +AC_INIT([parallel], [20100601], [bug-parallel@gnu.org]) AM_INIT_AUTOMAKE([-Wall -Werror foreign]) AC_CONFIG_HEADERS([config.h]) AC_CONFIG_FILES([ diff --git a/doc/FUTURE_IDEAS b/doc/FUTURE_IDEAS index 239fa4ff..6e89a550 100644 --- a/doc/FUTURE_IDEAS +++ b/doc/FUTURE_IDEAS @@ -1,9 +1,12 @@ =head1 IDEAS -One char options not used: F G J K P Q Y +Kan vi lave flere ssh'er, hvis vi venter lidt? +En ssh med 20% loss og 900 ms delay, så kan login nås på 15 sek. Test if -0 works on filenames ending in '\n' +Transfer scriptfile before first job. Remove it when last job done. + monitor to see which jobs are currently running http://code.google.com/p/ppss/ @@ -15,22 +18,13 @@ hard, as all foreground processes get the INT from the shell. If there are nomore jobs (STDIN is closed) then make sure to distribute the arguments evenly if running -X. - -Reuse ssh connection (-M and -S) if not using own ssh command. - -SEED=$RANDOM -ssh -MST /tmp/ssh-%r@%h:%p-$SEED elvis -rsync --rsh="ssh -S /tmp/ssh-%r@%h:%p-$SEED" gitup elvis:/tmp/ -ssh -S /tmp/ssh-%r@%h:%p-$SEED elvis hostname - - -http://www.semicomplete.com/blog/geekery/distributed-xargs.html?source=rss20 - Parallelize so this can be done: mdm.screen find dir -execdir mdm-run cmd {} \; Maybe: find dir -execdir par$ --communication-file /tmp/comfile cmd {} \; +find dir -execdir mutex -j4 -b cmd {} \; + =head2 Comfile This will put a lock on /tmp/comfile. The number of locks is the number of running commands. @@ -41,12 +35,30 @@ par$ --wait /tmp/comfile will wait until no more locks on the file =head2 mutex -mutex -n -l lockid -m max_locks [command] +mutex -b -n -l lockid -m max_locks [command] mutex -u lockid +-b run command in background -l lockfile will lock using the lockid -n nonblocking -m maximal number of locks (default 1) -u unlock If command given works like: mutex -l lockfile -n number_of_locks ; command; mutex -u lockfile +If -b given works like: mutex -l lockfile -n number_of_locks ; (command; mutex -u lockfile)& + +Kan vi finde på lockid som giver mening? + +=head1 options + +One char options not used: F G J K P Q Y + +Skilletegn i sshlogin: +#=item B<--sshlogin> I<[ncpu/]sshlogin[,[ncpu/]sshlogin[,...]]> (beta testing) +# Skilletegn: +# No: "#!&()?\<>|;*'~ shellspecial +# No: @.- part of user@i.p.n.r i.p.n.r host-name +# No: , separates different sshlogins +# No: space Will make it hard to do: 8/server1,server2 +# Maybe: / 8//usr/bin/myssh,//usr/bin/ssh +# %/=:_^ diff --git a/doc/parallel.directive b/doc/parallel.directive index bd4c0e17..c0518fef 100644 --- a/doc/parallel.directive +++ b/doc/parallel.directive @@ -1,4 +1,4 @@ version: 1.1 directory: parallel filename: parallel-20100424.tar.bz2 -comment: Initial release of GNU Parallel +comment: New release diff --git a/doc/pod2savannah_publicinfo b/doc/pod2savannah_publicinfo index 14f51d1c..395faf9d 100755 --- a/doc/pod2savannah_publicinfo +++ b/doc/pod2savannah_publicinfo @@ -15,7 +15,8 @@ while(<>) { } else { if(/^\s*$/) { $line=~s/B<(.*?)>/*$1*/gs; - $line=~s/I<(.*?)>/_$1_/gs; + # =_text_ -> = _text_ + $line=~s/I<(.*?)>/ _$1_ /gs; push(@lines,$line."\n\n"); $line=""; next; diff --git a/doc/release_new_version b/doc/release_new_version new file mode 100644 index 00000000..ca2d2b8d --- /dev/null +++ b/doc/release_new_version @@ -0,0 +1,94 @@ += Notes on how to release new version = + +== Update version == + +=== configure.ac === +configure.ac: AC_INIT([parallel], [20100422], [bug-parallel@gnu.org]) + +YYYYMMDD=`yyyymmdd` +perl -i -pe "s/20\d\d\d\d\d\d/$YYYYMMDD/" configure.ac + +=== src/parallel === + +src/parallel: $Global::version = 20100422; + +YYYYMMDD=`yyyymmdd` +perl -i -pe "/version/ and s/20\d\d\d\d\d\d/$YYYYMMDD/" src/parallel + +=== Autoconf/automake === + +rm -fr autom4te.cache aclocal.m4 config.h config.h.in config.log Makefile.in missing install-sh +rm -rf src/Makefile.in +autoreconf --install -W gnu +./configure +make && sudo make install + +== Unittest == + +cd unittest; make unittest + +== Package == + +./configure +make dist +make dist-bzip2 + +== Upload == + +YYYYMMDD=`yyyymmdd` +cp doc/parallel.directive parallel-$YYYYMMDD.tar.bz2.directive +perl -i -pe "/version/ and s/20\d\d\d\d\d\d/$YYYYMMDD/" parallel-*.tar.*directive +gpg --clearsign parallel-$YYYYMMDD.tar.bz2.directive + +YYYYMMDD=`yyyymmdd` +gpg -b parallel-$YYYYMMDD.tar.bz2 + +YYYYMMDD=`yyyymmdd` +echo put parallel-$YYYYMMDD.tar.bz2{,.sig,*asc} | ncftp ftp://ftp-upload.gnu.org/incoming/ftp/ + +== Update Savannah == + +doc/pod2savannah_publicinfo src/parallel | klipper-stdin +https://savannah.gnu.org/project/admin/editgroupinfo.php?group=parallel + +== Update Freshmeat == + +http://freshmeat.net/projects/parallel/releases/new + +== Send announce == + +<<<<< +to:parallel@gnu.org, bug-parallel@gnu.org, info-gnu@gnu.org, bug-directory@gnu.org + +Subject: GNU Parallel 20100601 released + +GNU Parallel 20100601 has been released today. It is available for +download at: http://ftp.gnu.org/gnu/parallel/ + +New in this release: + +* GNU Parallel now has support for running jobs on remote machines + += About GNU Parallel = + +GNU Parallel is a shell tool for executing jobs in parallel using one +or more machines. A job is typically a single command or a small +script that has to be run for each of the lines in the input. The +typical input is a list of files, a list of hosts, a list of users, a +list of URLs, or a list of tables. + +If you use xargs today you will find GNU Parallel very easy to use as +GNU Parallel is written to have the same options as xargs. If you +write loops in shell, you will find GNU Parallel may be able to +replace most of the loops and make them run faster by running several +jobs in parallel. If you use ppss or pexec you will find GNU Parallel +will often make the command easier to read. + +GNU Parallel makes sure output from the commands is the same output as +you would get had you run the commands sequentially. This makes it +possible to use output from GNU Parallel as input for other programs. + + +You can find more about GNU Parallel at: +http://www.gnu.org/software/parallel/ +>>>>> diff --git a/src/parallel b/src/parallel index f751a070..816c1fc9 100755 --- a/src/parallel +++ b/src/parallel @@ -647,15 +647,15 @@ B>B<{.}.bz2 && rm {}"> for-loops like this: -B< (for x in `cat list` ; do + (for x in `cat list` ; do do_something $x - done) | process_output> + done) | process_output and while-loops like this: -B< cat list | (while read x ; do + cat list | (while read x ; do do_something $x - done) | process_output> + done) | process_output can be written like this: @@ -663,19 +663,19 @@ B If the processing requires more steps the for-loop like this: -B< (for x in `cat list` ; do + (for x in `cat list` ; do no_extension=${x%.png}; do_something $x scale $no_extension.jpg do_step2 <$x $no_extension - done) | process_output> + done) | process_output and while-loops like this: -B< cat list | (while read x ; do + cat list | (while read x ; do no_extension=${x%.png}; do_something $x scale $no_extension.jpg do_step2 <$x $no_extension - done) | process_output> + done) | process_output can be written like this: @@ -1129,6 +1129,13 @@ ls *jpg | parallel -j8 'mutex -m blockread cat {} | jpegtopnm |' \ 'pnmscale 0.5 | pnmtojpeg | mutex -m blockwrite cat > th_{}' +=head1 DIFFERENCES BETWEEN dxargs AND parallel + +B does not deal well with more simultaneous jobs than SSHD's +MaxStartup. B is only built for remote run jobs, but does not +support transferring of files. + + =head1 DIFFERENCES BETWEEN mdm/middleman AND parallel middleman(mdm) is also a tool for running jobs in parallel. @@ -1149,13 +1156,6 @@ options need to be split into groups in which only the last option takes an argument. Each group of options should be put on a line of its own. -=head1 INIT FILE (RC FILE) - -The file ~/.parallelrc will be read if it exists. It should be -formatted like the environment variable $PARALLEL. Lines starting with -'#' will be ignored. - - =head2 EXAMPLE cat list | parallel -j1 -k -v ls @@ -1175,6 +1175,14 @@ Notice the newline in the middel is needed because both B<-S> and B<-j> take an argument and thus both need to be at the end of a group. +=head1 INIT FILE (RC FILE) + +The file ~/.parallelrc will be read if it exists. It should be +formatted like the environment variable $PARALLEL. Lines starting with +'#' will be ignored. + + + =head1 BUGS Filenames beginning with '-' can cause some commands to give @@ -1338,7 +1346,7 @@ drain_job_queue(); sub parse_options { # Defaults: - $Global::version = 20100529; + $Global::version = 20100601; $Global::progname = 'parallel'; $Global::debug = 0; $Global::verbose = 0; @@ -2669,12 +2677,3 @@ $Global::control_path = 0; # TODO Debian package # TODO transfer a script to be run - -#=item B<--sshlogin> I<[ncpu/]sshlogin[,[ncpu/]sshlogin[,...]]> (beta testing) -# Skilletegn: -# No: "#!&()?\<>|;*'~ shellspecial -# No: @.- part of user@i.p.n.r i.p.n.r host-name -# No: , separates different sshlogins -# No: space Will make it hard to do: 8/server1,server2 -# Maybe: / 8//usr/bin/myssh,//usr/bin/ssh -# %/=:_^