Released as 20151222 ('ParisAgreement')

NEWS

@@ -1,3 +1,26 @@
+20151222
+
+* --transfer is now an alias for --transferfile {}.
+
+* --transferfile works like --transfer, but takes an argument like --return. This makes it possible to combine transferring files with multiple input sources: parallel -S server --tf {1} wc {2} {1} ::: * ::: -l -w -c
+
+* total_jobs() can now be used in {= =}: parallel echo job {#} of '{= $_=total_jobs() =}' ::: {1..50}
+
+* GNU Parallel was cited in: Evolution and Learning in Heterogeneous Environments http://research.gold.ac.uk/15078/1/COM_thesis_JonesD_2015.pdf
+
+* GNU Parallel was cited in: Contrasting genetic architectures of schizophrenia and other complex diseases using fast variance-components analysis http://www.nature.com/ng/journal/v47/n12/full/ng.3431.html
+
+* GNU Parallel was cited in: Efficient Retrieval of Key Material for Inspecting Potentially Malicious Traffic in the Cloud http://www.cs.bham.ac.uk/~bxb/Papres/2015.1.pdf
+
+* GNU Parallel was cited in: Achieving Consistent Doppler Measurements from SDO/HMI Vector Field Inversions http://arxiv.org/pdf/1511.06500.pdf
+
+* Flo uses GNU Parallel: https://github.com/wurmlab/flo
+
+* 使用 GNU parallel 來平行運算http://mutolisp.logdown.com/posts/316959-using-gnu-parallel-to-parallel-computing
+
+* Bug fixes and man page updates.
+
+
 20151122
 
 * GNU Parallel packaged for CERN CentOS:

README

@@ -40,9 +40,9 @@ document.
 
 Full installation of GNU Parallel is as simple as:
 
-    wget http://ftpmirror.gnu.org/parallel/parallel-20151122.tar.bz2
+    wget http://ftpmirror.gnu.org/parallel/parallel-20151222.tar.bz2
-    bzip2 -dc parallel-20151122.tar.bz2 | tar xvf -
+    bzip2 -dc parallel-20151222.tar.bz2 | tar xvf -
-    cd parallel-20151122
+    cd parallel-20151222
     ./configure && make && sudo make install
 
 
@@ -51,9 +51,9 @@ Full installation of GNU Parallel is as simple as:
 If you are not root you can add ~/bin to your path and install in
 ~/bin and ~/share:
 
-    wget http://ftpmirror.gnu.org/parallel/parallel-20151122.tar.bz2
+    wget http://ftpmirror.gnu.org/parallel/parallel-20151222.tar.bz2
-    bzip2 -dc parallel-20151122.tar.bz2 | tar xvf -
+    bzip2 -dc parallel-20151222.tar.bz2 | tar xvf -
-    cd parallel-20151122
+    cd parallel-20151222
     ./configure --prefix=$HOME && make && make install
 
 Or if your system lacks 'make' you can simply copy src/parallel

configure

@@ -1,6 +1,6 @@
 #! /bin/sh
 # Guess values for system-dependent variables and create Makefiles.
-# Generated by GNU Autoconf 2.69 for parallel 20151122.
+# Generated by GNU Autoconf 2.69 for parallel 20151222.
 #
 # Report bugs to <bug-parallel@gnu.org>.
 #
@@ -579,8 +579,8 @@ MAKEFLAGS=
 # Identity of this package.
 PACKAGE_NAME='parallel'
 PACKAGE_TARNAME='parallel'
-PACKAGE_VERSION='20151122'
+PACKAGE_VERSION='20151222'
-PACKAGE_STRING='parallel 20151122'
+PACKAGE_STRING='parallel 20151222'
 PACKAGE_BUGREPORT='bug-parallel@gnu.org'
 PACKAGE_URL=''
 
@@ -1203,7 +1203,7 @@ if test "$ac_init_help" = "long"; then
   # Omit some internal or obsolete options to make the list less imposing.
   # This message is too long to be a string in the A/UX 3.1 sh.
   cat <<_ACEOF
-\`configure' configures parallel 20151122 to adapt to many kinds of systems.
+\`configure' configures parallel 20151222 to adapt to many kinds of systems.
 
 Usage: $0 [OPTION]... [VAR=VALUE]...
 
@@ -1269,7 +1269,7 @@ fi
 
 if test -n "$ac_init_help"; then
   case $ac_init_help in
-     short | recursive ) echo "Configuration of parallel 20151122:";;
+     short | recursive ) echo "Configuration of parallel 20151222:";;
    esac
   cat <<\_ACEOF
 
@@ -1345,7 +1345,7 @@ fi
 test -n "$ac_init_help" && exit $ac_status
 if $ac_init_version; then
   cat <<\_ACEOF
-parallel configure 20151122
+parallel configure 20151222
 generated by GNU Autoconf 2.69
 
 Copyright (C) 2012 Free Software Foundation, Inc.
@@ -1362,7 +1362,7 @@ cat >config.log <<_ACEOF
 This file contains any messages produced by compilers while
 running configure, to aid debugging if configure makes a mistake.
 
-It was created by parallel $as_me 20151122, which was
+It was created by parallel $as_me 20151222, which was
 generated by GNU Autoconf 2.69.  Invocation command line was
 
   $ $0 $@
@@ -2225,7 +2225,7 @@ fi
 
 # Define the identity of the package.
  PACKAGE='parallel'
- VERSION='20151122'
+ VERSION='20151222'
 
 
 cat >>confdefs.h <<_ACEOF
@@ -2867,7 +2867,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
 # report actual input values of CONFIG_FILES etc. instead of their
 # values after options handling.
 ac_log="
-This file was extended by parallel $as_me 20151122, which was
+This file was extended by parallel $as_me 20151222, which was
 generated by GNU Autoconf 2.69.  Invocation command line was
 
   CONFIG_FILES    = $CONFIG_FILES
@@ -2929,7 +2929,7 @@ _ACEOF
 cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
 ac_cs_version="\\
-parallel config.status 20151122
+parallel config.status 20151222
 configured by $0, generated by GNU Autoconf 2.69,
   with options \\"\$ac_cs_config\\"
 

configure.ac

@@ -1,4 +1,4 @@
-AC_INIT([parallel], [20151122], [bug-parallel@gnu.org])
+AC_INIT([parallel], [20151222], [bug-parallel@gnu.org])
 AM_INIT_AUTOMAKE([-Wall -Werror foreign])
 AC_CONFIG_HEADERS([config.h])
 AC_CONFIG_FILES([

doc/release_new_version

@@ -220,11 +220,19 @@ GNU Parallel 20151222 ('') <<[stable]>> has been released. It is available for d
 
 Haiku of the month:
 
-<<>>
+  |||||
-    -- Ole Tange
+  |||||||
+  Use GNU Parallel
+    -- Malcolm Cook
 
 New in this release:
 
+* --transfer is now an alias for --transferfile {}.
+
+* --transferfile works like --transfer, but takes an argument like --return. This makes it possible to combine transferring files with multiple input sources: parallel -S server --tf {1} wc {2} {1} ::: * ::: -l -w -c
+
+* total_jobs() can now be used in {= =}: parallel echo job {#} of '{= $_=total_jobs() =}' ::: {1..50}
+
 * << kontakt GNU Parallel was used (unfortunately without citation) in: Instrumentation and Trace Analysis for Ad-hoc Python Workflows in Cloud Environments http://ieeexplore.ieee.org/xpl/articleDetails.jsp?arnumber=7214035>>
 
 * <<Har angiveligt submittet ny version - afventer opdatering>> GNU Parallel was used (unfortunately without citation) in: MUGBAS: a species free gene-based programme suite for post-GWAS analysis http://www.ncbi.nlm.nih.gov/pubmed/25765345
@@ -249,6 +257,8 @@ for Big Data Applications https://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumb
 
 * GNU Parallel was cited in: Achieving Consistent Doppler Measurements from SDO/HMI Vector Field Inversions http://arxiv.org/pdf/1511.06500.pdf
 
+* Flo uses GNU Parallel: https://github.com/wurmlab/flo
+
 * 使用 GNU parallel 來平行運算http://mutolisp.logdown.com/posts/316959-using-gnu-parallel-to-parallel-computing
 
 * Bug fixes and man page updates.

packager/obs/home:tange/parallel/parallel.spec

@@ -1,6 +1,6 @@
 Summary:	Shell tool for executing jobs in parallel
 Name: 		parallel
-Version: 	20151122
+Version: 	20151222
 Release: 	1
 License: 	GPL
 Group: 		Productivity/File utilities

src/niceload

@@ -24,7 +24,7 @@
 use strict;
 use Getopt::Long;
 $Global::progname="niceload";
-$Global::version = 20151122;
+$Global::version = 20151222;
 Getopt::Long::Configure("bundling","require_order");
 get_options_from_array(\@ARGV) || die_usage();
 if($opt::version) {

src/parallel

@@ -1162,7 +1162,7 @@ sub check_invalid_option_combinations {
 
 sub init_globals {
     # Defaults:
-    $Global::version = 20151213;
+    $Global::version = 20151222;
     $Global::progname = 'parallel';
     $Global::infinity = 2**31;
     $Global::debug = 0;

src/parallel.pod

@@ -312,10 +312,25 @@ Replace with calculated I<perl expression>. B<$_> will contain the
 same as B<{}>. After evaluating I<perl expression> B<$_> will be used
 as the value. It is recommended to only change $_ but you have full
 access to all of GNU B<parallel>'s internal functions and data
-structures. You can use the function B<Q(>I<string>B<)> to shell quote
+structures. A few convenience functions have been made:
-a string.
 
-The B<{=>I<perl expression>B<=}> must be given as a single string.
+=over 15
+
+=item Z<> B<Q(>I<string>B<)>
+
+shell quote a string
+
+=item Z<> B<total_jobs()>
+
+number of jobs in total (alpha testing)
+
+=back
+
+Example:
+
+  seq 10 | parallel echo {} + 1 is {= '$_++' =}
+  parallel csh -c {= '$_="mkdir ".Q($_)' =} ::: '12" dir'
+  seq 50 | parallel echo job {#} of {= '$_=total_jobs()' =}
 
 See also: B<--rpl> B<--parens>
 
@@ -437,9 +452,9 @@ It is compatible with B<zenity>:
     2> >(zenity --progress --auto-kill) | wc
 
 
-=item B<--basefile> I<file>
+=item B<--basefile> I<file> (alpha testing)
 
-=item B<--bf> I<file>
+=item B<--bf> I<file> (alpha testing)
 
 I<file> will be transferred to each sshlogin before a jobs is
 started. It will be removed if B<--cleanup> is active. The file may be
@@ -525,7 +540,7 @@ Implies B<--pipe> unless B<--pipepart> is used.
 See also B<--fifo>.
 
 
-=item B<--cleanup>
+=item B<--cleanup> (alpha testing)
 
 Remove transferred files. B<--cleanup> will remove the transferred
 files on the remote computer after processing is done.
@@ -713,6 +728,11 @@ See also B<--cat>.
 Remove down hosts. For each remote host: check that login through ssh
 works. If not: do not use this host.
 
+For performance reasons, this check is performed only at the start and
+every time B<--sshloginfile> is changed. If an host goes down after
+the first check, it will go undetected until B<--sshloginfile> is
+changed; B<--retries> can be used to mitigate this.
+
 Currently you can I<not> put B<--filter-hosts> in a profile,
 $PARALLEL, /etc/parallel/config or similar. This is because GNU
 B<parallel> uses GNU B<parallel> to compute this, so you will get an
@@ -1040,9 +1060,9 @@ the same syntax as B<--jobs>, so I<100%> for one per CPU is a valid
 setting. Only difference is 0 which is interpreted as 0.01.
 
 
-=item B<--controlmaster>
+=item B<--controlmaster> (alpha testing)
 
-=item B<-M>
+=item B<-M> (alpha testing)
 
 Use ssh's ControlMaster to make ssh connections faster. Useful if jobs
 run remote and are very fast to run. This is disabled for sshlogins
@@ -1080,7 +1100,7 @@ See also B<-X> for context replace. If in doubt use B<-X> as that will
 most likely do what is needed.
 
 
-=item B<--memfree> I<size>
+=item B<--memfree> I<size> (beta testing)
 
 Minimum memory free when starting another job. The I<size> can be
 postfixed with K, M, G, T, P, k, m, g, t, or p which would multiply
@@ -1116,7 +1136,7 @@ This is useful for running the same command (e.g. uptime) on a list of
 servers.
 
 
-=item B<--onall>
+=item B<--onall> (alpha testing)
 
 Run all the jobs on all computers given with B<--sshlogin>. GNU
 B<parallel> will log into B<--jobs> number of computers in parallel
@@ -1141,9 +1161,9 @@ of each job is saved in a file and the filename is then printed.
 
 See also: B<--results>
 
-=item B<--pipe>
+=item B<--pipe> (alpha testing)
 
-=item B<--spreadstdin>
+=item B<--spreadstdin> (alpha testing)
 
 Spread input to jobs on stdin (standard input). Read a block of data
 from stdin (standard input) and give one block of data as input to one
@@ -1504,7 +1524,7 @@ the joblog.
 See also B<--joblog>, B<--resume>, B<--resume-failed>.
 
 
-=item B<--retries> I<n>
+=item B<--retries> I<n> (alpha testing)
 
 If a job fails, retry it on another computer on which it has not
 failed. Do this I<n> times. If there are fewer than I<n> computers in
@@ -1513,7 +1533,7 @@ useful if some jobs fail for no apparent reason (such as network
 failure).
 
 
-=item B<--return> I<filename>
+=item B<--return> I<filename> (alpha testing)
 
 Transfer files from remote computers. B<--return> is used with
 B<--sshlogin> when the arguments are files on the remote computers. When
@@ -1528,6 +1548,12 @@ This will transfer the file I<$HOME/foo/bar.out> from the computer
 I<server.example.com> to the file I<foo/bar.out> after running
 B<touch foo/bar.out> on I<server.example.com>.
 
+  parallel -S server --trc out/./{}.out touch {}.out ::: in/file
+
+This will transfer the file I<in/file.out> from the computer
+I<server.example.com> to the files I<out/in/file.out> after running
+B<touch in/file.out> on I<server>.
+
   echo /tmp/foo/bar.txt | parallel --return {.}.out \
     --sshlogin server.example.com touch {.}.out
 
@@ -1535,7 +1561,7 @@ This will transfer the file I</tmp/foo/bar.out> from the computer
 I<server.example.com> to the file I</tmp/foo/bar.out> after running
 B<touch /tmp/foo/bar.out> on I<server.example.com>.
 
-Multiple files can be transferred by repeating the options multiple
+Multiple files can be transferred by repeating the option multiple
 times:
 
   echo /tmp/foo/bar.txt | parallel \
@@ -2004,10 +2030,17 @@ B<wc foo/bar.txt> on I<server.example.com>.
   echo /tmp/foo/bar.txt | parallel \
     --sshlogin server.example.com --transferfile {} wc
 
-This will transfer the file I<foo/bar.txt> to the computer
+This will transfer the file I</tmp/foo/bar.txt> to the computer
 I<server.example.com> to the file I</tmp/foo/bar.txt> before running
 B<wc /tmp/foo/bar.txt> on I<server.example.com>.
 
+  echo /tmp/./foo/bar.txt | parallel \
+    --sshlogin server.example.com --transferfile {} wc
+
+This will transfer the file I</tmp/foo/bar.txt> to the computer
+I<server.example.com> to the file I<foo/bar.txt> before running
+B<wc /tmp/./foo/bar.txt> on I<server.example.com>.
+
 B<--transferfile> is often used with B<--return> and B<--cleanup>. A
 shorthand for B<--transferfile {}> is B<--transfer>.
 

src/sql

@@ -566,7 +566,7 @@ $Global::Initfile && unlink $Global::Initfile;
 exit ($err);
 
 sub parse_options {
-    $Global::version = 20151122;
+    $Global::version = 20151222;
     $Global::progname = 'sql';
 
     # This must be done first as this may exec myself

testsuite/tests-to-run/parallel-freebsd.sh

@@ -9,7 +9,7 @@ ping -c 1 freebsd7.tange.dk >/dev/null 2>&1
 ssh freebsd7.tange.dk touch .parallel/will-cite
 scp -q .*/src/{parallel,sem,sql,niceload} freebsd7.tange.dk:bin/
 
-cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj9 -k --joblog /tmp/jl-`basename $0` -L1 -S freebsd7.tange.dk
+cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj9 -k --joblog /tmp/jl-`basename $0` -L1 --retries 3 -S freebsd7.tange.dk
 echo 'bug #40136: FreeBSD: No more processes'
   sem --jobs 3 --id my_id -u 'echo First started; sleep 10; echo The first finished;echo' && 
   sem --jobs 3 --id my_id -u 'echo Second started; sleep 11; echo The second finished;echo' && 

testsuite/wanted-results/parallel-local-ssh4

@@ -142,7 +142,7 @@ Funky-
 
 0 updates are security updates.
 
-zsh:129: command not found: alias_echo
+zsh:130: command not found: alias_echo
 function_works
 myvar  works
 array_val2
@@ -150,7 +150,7 @@ assoc_val_a
 Funky-
 	
 
 !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~€亗儎厗噲墛媽崕彁憭摂晼棙櫄洔潪煚、￥ウЖ┆<D096><E29486><EFBFBD>辈炒刀犯购患骄坷谅媚牌侨墒颂臀闲岩釉罩棕仝圮蒉哙徕沅彐玷殛腱眍镳耱篝貊鼬<E8B28A><E9BCAC><EFBFBD><EFBFBD>-funky
-zsh:129: command not found: alias_echo
+zsh:130: command not found: alias_echo
 function_works_over_ssh
 myvar  works
 array_val2