From 82550070eede3fbc5f082d3f600ff3509f4f442e Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Sat, 23 Aug 2014 00:39:59 +0200 Subject: [PATCH] Released as 20140822 ('Williams'). --- NEWS | 40 +++++++ README | 12 +-- configure | 20 ++-- configure.ac | 2 +- doc/release_new_version | 6 +- .../obs/home:tange/parallel/parallel.spec | 4 +- src/niceload | 2 +- src/parallel | 2 +- src/parallel.pod | 102 ++++++++++-------- src/sql | 2 +- 10 files changed, 123 insertions(+), 69 deletions(-) diff --git a/NEWS b/NEWS index bbcc0cf3..e18720de 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,43 @@ +20140822 + +* GNU Parallel now uses the same shell it was started from as the + command shell for local jobs. So if GNU Parallel is started from + tcsh it will use tcsh as its shell even if the login $SHELL is + different. For remote jobs the login $SHELL will be used. + +* The whole current environment in bash can be copied by using a shell + wrapper function (Search manual for env_parallel). + +* --plus adds the replacement strings {+/} {+.} {+..} {+...} {..} + {...} {/..} {/...}. The idea being that '+foo' matches the opposite + of 'foo' and {} = {+/}/{/} = {.}.{+.} = {+/}/{/.}.{+.} = {..}.{+..} + = {+/}/{/..}.{+..} = {...}.{+...} = {+/}/{/...}.{+...} + +* GNU Parallel now deals correctly with the combination + rsync-3.1.X-client and rsync-2.5.7-server + +* GNU Parallel was cited in: A Web Service for Scholarly Big Data + Information Extraction + http://patshih.ist.psu.edu/publications/Williams-CiteSeerExtractor-ICWS14.pdf + +* Comparison of the speed of different GNU Parallel versions: + http://lists.gnu.org/archive/html/parallel/2014-08/msg00030.html + +* GNU Parallel was covered in the webcast 2014-08-20: Data Science at + the Command Line http://www.oreilly.com/pub/e/3115 + +* Distributed processing with GNU parallel + http://kazjote.eu/2014/08/11/distributed-processing-with-gnu-parallel + +* A Peek into GNU Parallel + http://blog.dataweave.in/post/94238943763/a-peek-into-gnu-parallel + +* Сборка GNU parallel для CentOS/RHEL + http://www.stableit.ru/2014/07/gnu-parallel-centosrhel.html + +* Bug fixes and man page updates. + + 20140722 * {= perl expression =} can be used as replacement string. The diff --git a/README b/README index a665f8d6..abb6f2b8 100644 --- a/README +++ b/README @@ -40,9 +40,9 @@ document. Full installation of GNU Parallel is as simple as: - wget http://ftpmirror.gnu.org/parallel/parallel-20140815.tar.bz2 - bzip2 -dc parallel-20140815.tar.bz2 | tar xvf - - cd parallel-20140815 + wget http://ftpmirror.gnu.org/parallel/parallel-20140822.tar.bz2 + bzip2 -dc parallel-20140822.tar.bz2 | tar xvf - + cd parallel-20140822 ./configure && make && make install @@ -51,9 +51,9 @@ Full installation of GNU Parallel is as simple as: If you are not root you can add ~/bin to your path and install in ~/bin and ~/share: - wget http://ftpmirror.gnu.org/parallel/parallel-20140815.tar.bz2 - bzip2 -dc parallel-20140815.tar.bz2 | tar xvf - - cd parallel-20140815 + wget http://ftpmirror.gnu.org/parallel/parallel-20140822.tar.bz2 + bzip2 -dc parallel-20140822.tar.bz2 | tar xvf - + cd parallel-20140822 ./configure --prefix=$HOME && make && make install Or if your system lacks 'make' you can simply copy src/parallel diff --git a/configure b/configure index 606bc5ec..e4ae80bd 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.69 for parallel 20140815. +# Generated by GNU Autoconf 2.69 for parallel 20140822. # # Report bugs to . # @@ -579,8 +579,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='parallel' PACKAGE_TARNAME='parallel' -PACKAGE_VERSION='20140815' -PACKAGE_STRING='parallel 20140815' +PACKAGE_VERSION='20140822' +PACKAGE_STRING='parallel 20140822' PACKAGE_BUGREPORT='bug-parallel@gnu.org' PACKAGE_URL='' @@ -1203,7 +1203,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures parallel 20140815 to adapt to many kinds of systems. +\`configure' configures parallel 20140822 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1269,7 +1269,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of parallel 20140815:";; + short | recursive ) echo "Configuration of parallel 20140822:";; esac cat <<\_ACEOF @@ -1345,7 +1345,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -parallel configure 20140815 +parallel configure 20140822 generated by GNU Autoconf 2.69 Copyright (C) 2012 Free Software Foundation, Inc. @@ -1362,7 +1362,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by parallel $as_me 20140815, which was +It was created by parallel $as_me 20140822, which was generated by GNU Autoconf 2.69. Invocation command line was $ $0 $@ @@ -2225,7 +2225,7 @@ fi # Define the identity of the package. PACKAGE='parallel' - VERSION='20140815' + VERSION='20140822' cat >>confdefs.h <<_ACEOF @@ -2867,7 +2867,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by parallel $as_me 20140815, which was +This file was extended by parallel $as_me 20140822, which was generated by GNU Autoconf 2.69. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -2929,7 +2929,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -parallel config.status 20140815 +parallel config.status 20140822 configured by $0, generated by GNU Autoconf 2.69, with options \\"\$ac_cs_config\\" diff --git a/configure.ac b/configure.ac index 4cec91aa..7540a9e4 100644 --- a/configure.ac +++ b/configure.ac @@ -1,4 +1,4 @@ -AC_INIT([parallel], [20140815], [bug-parallel@gnu.org]) +AC_INIT([parallel], [20140822], [bug-parallel@gnu.org]) AM_INIT_AUTOMAKE([-Wall -Werror foreign]) AC_CONFIG_HEADERS([config.h]) AC_CONFIG_FILES([ diff --git a/doc/release_new_version b/doc/release_new_version index 9866bb88..7143bf35 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -234,10 +234,12 @@ New in this release: * The whole current environment in bash can be copied by using a shell wrapper function (Search manual for env_parallel). -* GNU Parallel was cited in: A Web Service for Scholarly Big Data Information Extraction http://patshih.ist.psu.edu/publications/Williams-CiteSeerExtractor-ICWS14.pdf - * --plus adds the replacement strings {+/} {+.} {+..} {+...} {..} {...} {/..} {/...}. The idea being that '+foo' matches the opposite of 'foo' and {} = {+/}/{/} = {.}.{+.} = {+/}/{/.}.{+.} = {..}.{+..} = {+/}/{/..}.{+..} = {...}.{+...} = {+/}/{/...}.{+...} +* GNU Parallel now deals correctly with the combination rsync-3.1.X-client and rsync-2.5.7-server + +* GNU Parallel was cited in: A Web Service for Scholarly Big Data Information Extraction http://patshih.ist.psu.edu/publications/Williams-CiteSeerExtractor-ICWS14.pdf + * Comparison of the speed of different GNU Parallel versions: http://lists.gnu.org/archive/html/parallel/2014-08/msg00030.html * GNU Parallel was covered in the webcast 2014-08-20: Data Science at the Command Line http://www.oreilly.com/pub/e/3115 diff --git a/packager/obs/home:tange/parallel/parallel.spec b/packager/obs/home:tange/parallel/parallel.spec index 6a20c79b..b60d1b9f 100644 --- a/packager/obs/home:tange/parallel/parallel.spec +++ b/packager/obs/home:tange/parallel/parallel.spec @@ -1,11 +1,11 @@ Summary: Shell tool for executing jobs in parallel Name: parallel -Version: 20140722 +Version: 20140822 Release: 1 License: GPL Group: Productivity/File utilities URL: ftp://ftp.gnu.org/gnu/parallel -Source0: %{name}-%{version}.tar.bz2 +Source0: %{name}_%{version}.tar.gz BuildArch: noarch BuildArchitectures: noarch BuildRoot: %{_tmppath}/%{name}-%{version}-buildroot diff --git a/src/niceload b/src/niceload index 5f9c8c88..f2114f9b 100755 --- a/src/niceload +++ b/src/niceload @@ -24,7 +24,7 @@ use strict; use Getopt::Long; $Global::progname="niceload"; -$Global::version = 20140815; +$Global::version = 20140822; Getopt::Long::Configure("bundling","require_order"); get_options_from_array(\@ARGV) || die_usage(); if($opt::version) { diff --git a/src/parallel b/src/parallel index 0c17e284..b16534b2 100755 --- a/src/parallel +++ b/src/parallel @@ -721,7 +721,7 @@ sub get_options_from_array { sub parse_options { # Returns: N/A # Defaults: - $Global::version = 20140815; + $Global::version = 20140822; $Global::progname = 'parallel'; $Global::infinity = 2**31; $Global::debug = 0; diff --git a/src/parallel.pod b/src/parallel.pod index 0d0f0cc2..07236dd1 100644 --- a/src/parallel.pod +++ b/src/parallel.pod @@ -97,7 +97,7 @@ B to export and to set $SHELL to bash: The command cannot contain the character \257 (macron: ¯). -=item B<{}> (beta testing) +=item B<{}> (alpha testing) Input line. This replacement string will be replaced by a full line read from the input source. The input source is normally stdin @@ -110,7 +110,7 @@ If the command line contains no replacement strings then B<{}> will be appended to the command line. -=item B<{.}> (beta testing) +=item B<{.}> (alpha testing) Input line without extension. This replacement string will be replaced by the input with the extension removed. If the input line contains @@ -126,7 +126,7 @@ The replacement string B<{.}> can be changed with B<--er>. To understand replacement strings see B<{}>. -=item B<{/}> (beta testing) +=item B<{/}> (alpha testing) Basename of input line. This replacement string will be replaced by the input with the directory part removed. @@ -137,7 +137,7 @@ B<--basenamereplace>. To understand replacement strings see B<{}>. -=item B<{//}> (beta testing) +=item B<{//}> (alpha testing) Dirname of input line. This replacement string will be replaced by the dir of the input line. See B(1). @@ -148,7 +148,7 @@ B<--dirnamereplace>. To understand replacement strings see B<{}>. -=item B<{/.}> (beta testing) +=item B<{/.}> (alpha testing) Basename of input line without extension. This replacement string will be replaced by the input with the directory and extension part @@ -160,7 +160,7 @@ B<--basenameextensionreplace>. To understand replacement strings see B<{}>. -=item B<{#}> (beta testing) +=item B<{#}> (alpha testing) Sequence number of the job to run. This replacement string will be replaced by the sequence number of the job being run. It contains the @@ -171,7 +171,7 @@ The replacement string B<{#}> can be changed with B<--seqreplace>. To understand replacement strings see B<{}>. -=item B<{%}> (beta testing) +=item B<{%}> (alpha testing) Job slot number. This replacement string will be replaced by the job's slot number between 1 and number of jobs to run in parallel. There @@ -183,7 +183,7 @@ The replacement string B<{%}> can be changed with B<--slotreplace>. To understand replacement strings see B<{}>. -=item B<{>IB<}> (beta testing) +=item B<{>IB<}> (alpha testing) Argument from input source I or the I'th argument. This positional replacement string will be replaced by the input from input @@ -194,7 +194,7 @@ I'th last argument. To understand replacement strings see B<{}>. -=item B<{>I.B<}> (beta testing) +=item B<{>I.B<}> (alpha testing) Argument from input source I or the I'th argument without extension. It is a combination of B<{>IB<}> and B<{.}>. @@ -207,7 +207,7 @@ extension removed. To understand positional replacement strings see B<{>IB<}>. -=item B<{>I/B<}> (beta testing) +=item B<{>I/B<}> (alpha testing) Basename of argument from input source I or the I'th argument. It is a combination of B<{>IB<}> and B<{/}>. @@ -220,7 +220,7 @@ directory (if any) removed. To understand positional replacement strings see B<{>IB<}>. -=item B<{>I//B<}> (beta testing) +=item B<{>I//B<}> (alpha testing) Dirname of argument from input source I or the I'th argument. It is a combination of B<{>IB<}> and B<{//}>. @@ -232,7 +232,7 @@ the I'th argument (when used with B<-N>). See B(1). To understand positional replacement strings see B<{>IB<}>. -=item B<{>I/.B<}> (beta testing) +=item B<{>I/.B<}> (alpha testing) Basename of argument from input source I or the I'th argument without extension. It is a combination of B<{>IB<}>, B<{/}>, and @@ -246,7 +246,7 @@ directory (if any) and extension removed. To understand positional replacement strings see B<{>IB<}>. -=item B<{=>IB<=}> (beta testing) +=item B<{=>IB<=}> (alpha testing) Replace with calculated I. B<$_> will contain the same as B<{}>. After evaluating I B<$_> will be used @@ -259,7 +259,7 @@ The B<{=>IB<=}> must be given as a single string. See also: B<--rpl> B<--parens> -=item B<{=>I IB<=}> (beta testing) +=item B<{=>I IB<=}> (alpha testing) Positional equivalent to B<{= perl expression =}>. To understand positional replacement strings see B<{>IB<}>. @@ -321,9 +321,9 @@ Use NUL as delimiter. Normally input lines will end in \n for processing arguments that may contain \n (newline). -=item B<--arg-file> I (alpha testing) +=item B<--arg-file> I (beta testing) -=item B<-a> I (alpha testing) +=item B<-a> I (beta testing) Use I as input source. If you use this option, stdin (standard input) is given to the first process run. Otherwise, stdin @@ -370,9 +370,9 @@ It is compatible with B: seq 1000 | parallel -j30 --bar '(echo {};sleep 0.1)' 2> >(zenity --progress --auto-kill) | wc -=item B<--basefile> I +=item B<--basefile> I (alpha testing) -=item B<--bf> I +=item B<--bf> I (alpha testing) I will be transferred to each sshlogin before a jobs is started. It will be removed if B<--cleanup> is active. The file may be @@ -413,9 +413,9 @@ Print the BibTeX entry for GNU B and disable citation notice. -=item B<--block> I (alpha testing) +=item B<--block> I (beta testing) -=item B<--block-size> I (alpha testing) +=item B<--block-size> I (beta testing) Size of block in bytes. The size can be postfixed with K, M, G, T, P, k, m, g, t, or p which would multiply the size with 1024, 1048576, @@ -523,7 +523,7 @@ Use the replacement string I instead of B<{//}> for dirname of input line. -=item B<-E> I +=item B<-E> I (alpha testing) Set the end of file string to eof-str. If the end of file string occurs as a line of input, the rest of the input is ignored. If @@ -556,7 +556,7 @@ If I is omitted, there is no end of file string. If neither B<-E> nor B<-e> is used, no end of file string is used. -=item B<--env> I +=item B<--env> I (alpha testing) Copy environment variable I. This will copy I to the environment that the command is run in. This is especially useful for @@ -588,10 +588,23 @@ cannot be exported: parallel --env my_importer \ 'my_importer; echo "{}" "${indexed[{}]}" "${assoc[${indexed[{}]}]}"' ::: "${!indexed[@]}" +To copy the full environment use this function: + + env_parallel() { + export parallel_bash_environment='() { + '"$(echo "shopt -s expand_aliases 2>/dev/null"; alias;typeset -p | grep -vFf <(readonly; echo GROUPS; echo FUNCNAME; echo DIRSTACK; echo _; echo PIPESTATUS; echo USERNAME) | grep -v BASH_;typeset -f)"' + }' + # Run as: env_parallel parallel_bash_environment "2>/dev/null;" ... + `which parallel` "$@" + unset parallel_bash_environment + } + # call as: + env_parallel ... + See also: B<--record-env>. -=item B<--eta> (alpha testing) +=item B<--eta> (beta testing) Show the estimated number of seconds before finishing. This forces GNU B to read all jobs before starting to find the number of @@ -661,9 +674,9 @@ See also: B<--line-buffer> B<--ungroup> Print a summary of the options to GNU B and exit. -=item B<--halt-on-error> I (beta testing) +=item B<--halt-on-error> I (alpha testing) -=item B<--halt> I (beta testing) +=item B<--halt> I (alpha testing) How should GNU B terminate if one of more jobs fail? @@ -713,9 +726,9 @@ If I is a number, it will match that many lines. Use the replacement string I instead of {}. -=item B<--replace>[=I] +=item B<--replace>[=I] (alpha testing) -=item B<-i>[I] +=item B<-i>[I] (alpha testing) This option is a synonym for B<-I>I if I is specified, and for B<-I>{} otherwise. This option is deprecated; @@ -784,9 +797,8 @@ B<--use-cpus-instead-of-cores>. =item B<-P> I% (alpha testing) -Multiply N% with the number of CPU cores. Run this many jobs in parallel. -If the evaluated number is less than 1 then 1 will be used. See also -B<--use-cpus-instead-of-cores>. +Multiply N% with the number of CPU cores. Run this many jobs in +parallel. See also B<--use-cpus-instead-of-cores>. =item B<--jobs> I @@ -875,9 +887,9 @@ the same syntax as B<--jobs>, so I<100%> for one per CPU is a valid setting. Only difference is 0 which is interpreted as 0.01. -=item B<--controlmaster> +=item B<--controlmaster> (alpha testing) -=item B<-M> +=item B<-M> (alpha testing) Use ssh's ControlMaster to make ssh connections faster. Useful if jobs run remote and are very fast to run. This is disabled for sshlogins @@ -987,7 +999,7 @@ B<--files> is often used with B<--pipe>. See also: B<--recstart>, B<--recend>, B<--fifo>, B<--cat>, B<--pipepart>. -=item B<--pipepart> (alpha testing) +=item B<--pipepart> (beta testing) Pipe parts of a physical file. B<--pipepart> works similar to B<--pipe>, but is much faster. It has a few limitations: @@ -1118,7 +1130,7 @@ from the terminal. Only run the command line if the response starts with 'y' or 'Y'. Implies B<-t>. -=item B<--parens> I (beta testing) +=item B<--parens> I Use to define start and end parenthesis for B<{= perl expression =}>. The left and the right parenthesis can be multiple characters and are @@ -1308,7 +1320,7 @@ useful if some jobs fail for no apparent reason (such as network failure). -=item B<--return> I +=item B<--return> I (alpha testing) Transfer files from remote computers. B<--return> is used with B<--sshlogin> when the arguments are files on the remote computers. When @@ -1356,7 +1368,7 @@ B<--keep-order> will not work with B<--round-robin> as it is impossible to track which input block corresponds to which output. -=item B<--rpl> 'I I' (beta testing) +=item B<--rpl> 'I I' (alpha testing) Use I as a replacement string for I. This makes it possible to define your own replacement strings. GNU B's @@ -1461,9 +1473,9 @@ Use the replacement string I instead of B<{#}> for job sequence number. -=item B<--shebang> (beta testing) +=item B<--shebang> -=item B<--hashbang> (beta testing) +=item B<--hashbang> GNU B can be called as a shebang (#!) command as the first line of a script. The content of the file will be treated as @@ -1488,7 +1500,7 @@ On FreeBSD B is needed: freenetproject.org -=item B<--shebang-wrap> (beta testing) +=item B<--shebang-wrap> GNU B can parallelize scripts by wrapping the shebang line. If the program can be run like this: @@ -1537,9 +1549,9 @@ I seconds after starting each ssh. I can be less than 1 seconds. -=item B<-S> I<[ncpu/]sshlogin[,[ncpu/]sshlogin[,...]]> +=item B<-S> I<[ncpu/]sshlogin[,[ncpu/]sshlogin[,...]]> (alpha testing) -=item B<--sshlogin> I<[ncpu/]sshlogin[,[ncpu/]sshlogin[,...]]> +=item B<--sshlogin> I<[ncpu/]sshlogin[,[ncpu/]sshlogin[,...]]> (alpha testing) Distribute jobs to remote computers. The jobs will be run on a list of remote computers. GNU B will determine the number of CPU @@ -1575,9 +1587,9 @@ B<--sshlogin> is often used with B<--transfer>, B<--return>, B<--cleanup>, and B<--trc>. -=item B<--sshloginfile> I +=item B<--sshloginfile> I (alpha testing) -=item B<--slf> I +=item B<--slf> I (alpha testing) File with sshlogins. The file consists of sshlogins on separate lines. Empty lines and lines starting with '#' are ignored. Example: @@ -1662,7 +1674,7 @@ different dir for the files. Setting B<--tmpdir> is equivalent to setting $TMPDIR. -=item B<--tmux> (alpha testing) +=item B<--tmux> (beta testing) Use B for output. Start a B session and run each job in a window in that session. No other output will be produced. @@ -1688,7 +1700,7 @@ Print the job to be run on stderr (standard error). See also B<-v>, B<-p>. -=item B<--transfer> +=item B<--transfer> (alpha testing) Transfer files to remote computers. B<--transfer> is used with B<--sshlogin> when the arguments are files and should be transferred diff --git a/src/sql b/src/sql index 46531e4c..b8e42fd0 100755 --- a/src/sql +++ b/src/sql @@ -566,7 +566,7 @@ $Global::Initfile && unlink $Global::Initfile; exit ($err); sub parse_options { - $Global::version = 20140815; + $Global::version = 20140822; $Global::progname = 'sql'; # This must be done first as this may exec myself