diff --git a/NEWS b/NEWS index 469f7427..7234c611 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,26 @@ +20151222 + +* --transfer is now an alias for --transferfile {}. + +* --transferfile works like --transfer, but takes an argument like --return. This makes it possible to combine transferring files with multiple input sources: parallel -S server --tf {1} wc {2} {1} ::: * ::: -l -w -c + +* total_jobs() can now be used in {= =}: parallel echo job {#} of '{= $_=total_jobs() =}' ::: {1..50} + +* GNU Parallel was cited in: Evolution and Learning in Heterogeneous Environments http://research.gold.ac.uk/15078/1/COM_thesis_JonesD_2015.pdf + +* GNU Parallel was cited in: Contrasting genetic architectures of schizophrenia and other complex diseases using fast variance-components analysis http://www.nature.com/ng/journal/v47/n12/full/ng.3431.html + +* GNU Parallel was cited in: Efficient Retrieval of Key Material for Inspecting Potentially Malicious Traffic in the Cloud http://www.cs.bham.ac.uk/~bxb/Papres/2015.1.pdf + +* GNU Parallel was cited in: Achieving Consistent Doppler Measurements from SDO/HMI Vector Field Inversions http://arxiv.org/pdf/1511.06500.pdf + +* Flo uses GNU Parallel: https://github.com/wurmlab/flo + +* 使用 GNU parallel 來平行運算http://mutolisp.logdown.com/posts/316959-using-gnu-parallel-to-parallel-computing + +* Bug fixes and man page updates. + + 20151122 * GNU Parallel packaged for CERN CentOS: diff --git a/README b/README index e3937e40..250beba5 100644 --- a/README +++ b/README @@ -40,9 +40,9 @@ document. Full installation of GNU Parallel is as simple as: - wget http://ftpmirror.gnu.org/parallel/parallel-20151122.tar.bz2 - bzip2 -dc parallel-20151122.tar.bz2 | tar xvf - - cd parallel-20151122 + wget http://ftpmirror.gnu.org/parallel/parallel-20151222.tar.bz2 + bzip2 -dc parallel-20151222.tar.bz2 | tar xvf - + cd parallel-20151222 ./configure && make && sudo make install @@ -51,9 +51,9 @@ Full installation of GNU Parallel is as simple as: If you are not root you can add ~/bin to your path and install in ~/bin and ~/share: - wget http://ftpmirror.gnu.org/parallel/parallel-20151122.tar.bz2 - bzip2 -dc parallel-20151122.tar.bz2 | tar xvf - - cd parallel-20151122 + wget http://ftpmirror.gnu.org/parallel/parallel-20151222.tar.bz2 + bzip2 -dc parallel-20151222.tar.bz2 | tar xvf - + cd parallel-20151222 ./configure --prefix=$HOME && make && make install Or if your system lacks 'make' you can simply copy src/parallel diff --git a/configure b/configure index 0652f398..01b791ed 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.69 for parallel 20151122. +# Generated by GNU Autoconf 2.69 for parallel 20151222. # # Report bugs to . # @@ -579,8 +579,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='parallel' PACKAGE_TARNAME='parallel' -PACKAGE_VERSION='20151122' -PACKAGE_STRING='parallel 20151122' +PACKAGE_VERSION='20151222' +PACKAGE_STRING='parallel 20151222' PACKAGE_BUGREPORT='bug-parallel@gnu.org' PACKAGE_URL='' @@ -1203,7 +1203,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures parallel 20151122 to adapt to many kinds of systems. +\`configure' configures parallel 20151222 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1269,7 +1269,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of parallel 20151122:";; + short | recursive ) echo "Configuration of parallel 20151222:";; esac cat <<\_ACEOF @@ -1345,7 +1345,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -parallel configure 20151122 +parallel configure 20151222 generated by GNU Autoconf 2.69 Copyright (C) 2012 Free Software Foundation, Inc. @@ -1362,7 +1362,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by parallel $as_me 20151122, which was +It was created by parallel $as_me 20151222, which was generated by GNU Autoconf 2.69. Invocation command line was $ $0 $@ @@ -2225,7 +2225,7 @@ fi # Define the identity of the package. PACKAGE='parallel' - VERSION='20151122' + VERSION='20151222' cat >>confdefs.h <<_ACEOF @@ -2867,7 +2867,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by parallel $as_me 20151122, which was +This file was extended by parallel $as_me 20151222, which was generated by GNU Autoconf 2.69. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -2929,7 +2929,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -parallel config.status 20151122 +parallel config.status 20151222 configured by $0, generated by GNU Autoconf 2.69, with options \\"\$ac_cs_config\\" diff --git a/configure.ac b/configure.ac index ebc569a5..0edb3d93 100644 --- a/configure.ac +++ b/configure.ac @@ -1,4 +1,4 @@ -AC_INIT([parallel], [20151122], [bug-parallel@gnu.org]) +AC_INIT([parallel], [20151222], [bug-parallel@gnu.org]) AM_INIT_AUTOMAKE([-Wall -Werror foreign]) AC_CONFIG_HEADERS([config.h]) AC_CONFIG_FILES([ diff --git a/doc/release_new_version b/doc/release_new_version index 64fdc97c..ef44512c 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -220,11 +220,19 @@ GNU Parallel 20151222 ('') <<[stable]>> has been released. It is available for d Haiku of the month: -<<>> - -- Ole Tange + ||||| + ||||||| + Use GNU Parallel + -- Malcolm Cook New in this release: +* --transfer is now an alias for --transferfile {}. + +* --transferfile works like --transfer, but takes an argument like --return. This makes it possible to combine transferring files with multiple input sources: parallel -S server --tf {1} wc {2} {1} ::: * ::: -l -w -c + +* total_jobs() can now be used in {= =}: parallel echo job {#} of '{= $_=total_jobs() =}' ::: {1..50} + * << kontakt GNU Parallel was used (unfortunately without citation) in: Instrumentation and Trace Analysis for Ad-hoc Python Workflows in Cloud Environments http://ieeexplore.ieee.org/xpl/articleDetails.jsp?arnumber=7214035>> * <> GNU Parallel was used (unfortunately without citation) in: MUGBAS: a species free gene-based programme suite for post-GWAS analysis http://www.ncbi.nlm.nih.gov/pubmed/25765345 @@ -249,6 +257,8 @@ for Big Data Applications https://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumb * GNU Parallel was cited in: Achieving Consistent Doppler Measurements from SDO/HMI Vector Field Inversions http://arxiv.org/pdf/1511.06500.pdf +* Flo uses GNU Parallel: https://github.com/wurmlab/flo + * 使用 GNU parallel 來平行運算http://mutolisp.logdown.com/posts/316959-using-gnu-parallel-to-parallel-computing * Bug fixes and man page updates. diff --git a/packager/obs/home:tange/parallel/parallel.spec b/packager/obs/home:tange/parallel/parallel.spec index 317fb4e4..40c598a7 100644 --- a/packager/obs/home:tange/parallel/parallel.spec +++ b/packager/obs/home:tange/parallel/parallel.spec @@ -1,6 +1,6 @@ Summary: Shell tool for executing jobs in parallel Name: parallel -Version: 20151122 +Version: 20151222 Release: 1 License: GPL Group: Productivity/File utilities diff --git a/src/niceload b/src/niceload index 084cfa18..ff0f4847 100755 --- a/src/niceload +++ b/src/niceload @@ -24,7 +24,7 @@ use strict; use Getopt::Long; $Global::progname="niceload"; -$Global::version = 20151122; +$Global::version = 20151222; Getopt::Long::Configure("bundling","require_order"); get_options_from_array(\@ARGV) || die_usage(); if($opt::version) { diff --git a/src/parallel b/src/parallel index 20b52c81..73c52f56 100755 --- a/src/parallel +++ b/src/parallel @@ -1162,7 +1162,7 @@ sub check_invalid_option_combinations { sub init_globals { # Defaults: - $Global::version = 20151213; + $Global::version = 20151222; $Global::progname = 'parallel'; $Global::infinity = 2**31; $Global::debug = 0; diff --git a/src/parallel.pod b/src/parallel.pod index f6ce46ee..eea52090 100644 --- a/src/parallel.pod +++ b/src/parallel.pod @@ -312,20 +312,35 @@ Replace with calculated I. B<$_> will contain the same as B<{}>. After evaluating I B<$_> will be used as the value. It is recommended to only change $_ but you have full access to all of GNU B's internal functions and data -structures. You can use the function BIB<)> to shell quote -a string. +structures. A few convenience functions have been made: -The B<{=>IB<=}> must be given as a single string. +=over 15 + +=item Z<> BIB<)> + +shell quote a string + +=item Z<> B + +number of jobs in total (alpha testing) + +=back + +Example: + + seq 10 | parallel echo {} + 1 is {= '$_++' =} + parallel csh -c {= '$_="mkdir ".Q($_)' =} ::: '12" dir' + seq 50 | parallel echo job {#} of {= '$_=total_jobs()' =} See also: B<--rpl> B<--parens> =item B<{=>I IB<=}> -Positional equivalent to B<{= perl expression =}>. To understand +Positional equivalent to B<{=perl expression=}>. To understand positional replacement strings see B<{>IB<}>. -See also: B<{= perl expression =}> B<{>IB<}>. +See also: B<{=perl expression=}> B<{>IB<}>. =item B<:::> I @@ -437,9 +452,9 @@ It is compatible with B: 2> >(zenity --progress --auto-kill) | wc -=item B<--basefile> I +=item B<--basefile> I (alpha testing) -=item B<--bf> I +=item B<--bf> I (alpha testing) I will be transferred to each sshlogin before a jobs is started. It will be removed if B<--cleanup> is active. The file may be @@ -525,7 +540,7 @@ Implies B<--pipe> unless B<--pipepart> is used. See also B<--fifo>. -=item B<--cleanup> +=item B<--cleanup> (alpha testing) Remove transferred files. B<--cleanup> will remove the transferred files on the remote computer after processing is done. @@ -713,6 +728,11 @@ See also B<--cat>. Remove down hosts. For each remote host: check that login through ssh works. If not: do not use this host. +For performance reasons, this check is performed only at the start and +every time B<--sshloginfile> is changed. If an host goes down after +the first check, it will go undetected until B<--sshloginfile> is +changed; B<--retries> can be used to mitigate this. + Currently you can I put B<--filter-hosts> in a profile, $PARALLEL, /etc/parallel/config or similar. This is because GNU B uses GNU B to compute this, so you will get an @@ -1040,9 +1060,9 @@ the same syntax as B<--jobs>, so I<100%> for one per CPU is a valid setting. Only difference is 0 which is interpreted as 0.01. -=item B<--controlmaster> +=item B<--controlmaster> (alpha testing) -=item B<-M> +=item B<-M> (alpha testing) Use ssh's ControlMaster to make ssh connections faster. Useful if jobs run remote and are very fast to run. This is disabled for sshlogins @@ -1080,7 +1100,7 @@ See also B<-X> for context replace. If in doubt use B<-X> as that will most likely do what is needed. -=item B<--memfree> I +=item B<--memfree> I (beta testing) Minimum memory free when starting another job. The I can be postfixed with K, M, G, T, P, k, m, g, t, or p which would multiply @@ -1116,7 +1136,7 @@ This is useful for running the same command (e.g. uptime) on a list of servers. -=item B<--onall> +=item B<--onall> (alpha testing) Run all the jobs on all computers given with B<--sshlogin>. GNU B will log into B<--jobs> number of computers in parallel @@ -1141,9 +1161,9 @@ of each job is saved in a file and the filename is then printed. See also: B<--results> -=item B<--pipe> +=item B<--pipe> (alpha testing) -=item B<--spreadstdin> +=item B<--spreadstdin> (alpha testing) Spread input to jobs on stdin (standard input). Read a block of data from stdin (standard input) and give one block of data as input to one @@ -1504,7 +1524,7 @@ the joblog. See also B<--joblog>, B<--resume>, B<--resume-failed>. -=item B<--retries> I +=item B<--retries> I (alpha testing) If a job fails, retry it on another computer on which it has not failed. Do this I times. If there are fewer than I computers in @@ -1513,7 +1533,7 @@ useful if some jobs fail for no apparent reason (such as network failure). -=item B<--return> I +=item B<--return> I (alpha testing) Transfer files from remote computers. B<--return> is used with B<--sshlogin> when the arguments are files on the remote computers. When @@ -1528,6 +1548,12 @@ This will transfer the file I<$HOME/foo/bar.out> from the computer I to the file I after running B on I. + parallel -S server --trc out/./{}.out touch {}.out ::: in/file + +This will transfer the file I from the computer +I to the files I after running +B on I. + echo /tmp/foo/bar.txt | parallel --return {.}.out \ --sshlogin server.example.com touch {.}.out @@ -1535,7 +1561,7 @@ This will transfer the file I from the computer I to the file I after running B on I. -Multiple files can be transferred by repeating the options multiple +Multiple files can be transferred by repeating the option multiple times: echo /tmp/foo/bar.txt | parallel \ @@ -2004,10 +2030,17 @@ B on I. echo /tmp/foo/bar.txt | parallel \ --sshlogin server.example.com --transferfile {} wc -This will transfer the file I to the computer +This will transfer the file I to the computer I to the file I before running B on I. + echo /tmp/./foo/bar.txt | parallel \ + --sshlogin server.example.com --transferfile {} wc + +This will transfer the file I to the computer +I to the file I before running +B on I. + B<--transferfile> is often used with B<--return> and B<--cleanup>. A shorthand for B<--transferfile {}> is B<--transfer>. diff --git a/src/sql b/src/sql index ae99d96a..46f1f53e 100755 --- a/src/sql +++ b/src/sql @@ -566,7 +566,7 @@ $Global::Initfile && unlink $Global::Initfile; exit ($err); sub parse_options { - $Global::version = 20151122; + $Global::version = 20151222; $Global::progname = 'sql'; # This must be done first as this may exec myself diff --git a/testsuite/tests-to-run/parallel-freebsd.sh b/testsuite/tests-to-run/parallel-freebsd.sh index 2fd7fe69..4a5f6114 100644 --- a/testsuite/tests-to-run/parallel-freebsd.sh +++ b/testsuite/tests-to-run/parallel-freebsd.sh @@ -9,7 +9,7 @@ ping -c 1 freebsd7.tange.dk >/dev/null 2>&1 ssh freebsd7.tange.dk touch .parallel/will-cite scp -q .*/src/{parallel,sem,sql,niceload} freebsd7.tange.dk:bin/ -cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj9 -k --joblog /tmp/jl-`basename $0` -L1 -S freebsd7.tange.dk +cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj9 -k --joblog /tmp/jl-`basename $0` -L1 --retries 3 -S freebsd7.tange.dk echo 'bug #40136: FreeBSD: No more processes' sem --jobs 3 --id my_id -u 'echo First started; sleep 10; echo The first finished;echo' && sem --jobs 3 --id my_id -u 'echo Second started; sleep 11; echo The second finished;echo' && diff --git a/testsuite/wanted-results/parallel-local-ssh4 b/testsuite/wanted-results/parallel-local-ssh4 index 891ae249..c0fb1d78 100644 --- a/testsuite/wanted-results/parallel-local-ssh4 +++ b/testsuite/wanted-results/parallel-local-ssh4 @@ -142,7 +142,7 @@ Funky- 0 updates are security updates. -zsh:129: command not found: alias_echo +zsh:130: command not found: alias_echo function_works myvar works array_val2 @@ -150,7 +150,7 @@ assoc_val_a Funky-   !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~-funky -zsh:129: command not found: alias_echo +zsh:130: command not found: alias_echo function_works_over_ssh myvar works array_val2