tange/parallel
1
0
Fork 0
mirror of https://git.savannah.gnu.org/git/parallel.git synced 2024-11-22 14:07:55 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

parallel: {choose_k} added to --plus.

Browse source
This commit is contained in:
Ole Tange 2017-09-22 00:50:39 +02:00
parent 68114b93e7
commit f09e0384ff
11 changed files with 87 additions and 44 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
README
View file

@ -44,9 +44,9 @@ document.
Full installation of GNU Parallel is as simple as: Full installation of GNU Parallel is as simple as:
wget http://ftpmirror.gnu.org/parallel/parallel-20170822.tar.bz2 wget http://ftpmirror.gnu.org/parallel/parallel-20170922.tar.bz2
bzip2 -dc parallel-20170822.tar.bz2 | tar xvf - bzip2 -dc parallel-20170922.tar.bz2 | tar xvf -
cd parallel-20170822 cd parallel-20170922
./configure && make && sudo make install ./configure && make && sudo make install
@ -55,9 +55,9 @@ Full installation of GNU Parallel is as simple as:
If you are not root you can add ~/bin to your path and install in If you are not root you can add ~/bin to your path and install in
~/bin and ~/share: ~/bin and ~/share:
wget http://ftpmirror.gnu.org/parallel/parallel-20170822.tar.bz2 wget http://ftpmirror.gnu.org/parallel/parallel-20170922.tar.bz2
bzip2 -dc parallel-20170822.tar.bz2 | tar xvf - bzip2 -dc parallel-20170922.tar.bz2 | tar xvf -
cd parallel-20170822 cd parallel-20170922
./configure --prefix=$HOME && make && make install ./configure --prefix=$HOME && make && make install
Or if your system lacks 'make' you can simply copy src/parallel Or if your system lacks 'make' you can simply copy src/parallel

20
configure vendored
View file

@ -1,6 +1,6 @@
#! /bin/sh #! /bin/sh
# Guess values for system-dependent variables and create Makefiles. # Guess values for system-dependent variables and create Makefiles.
# Generated by GNU Autoconf 2.69 for parallel 20170822. # Generated by GNU Autoconf 2.69 for parallel 20170922.
# #
# Report bugs to <bug-parallel@gnu.org>. # Report bugs to <bug-parallel@gnu.org>.
# #
@ -579,8 +579,8 @@ MAKEFLAGS=
# Identity of this package. # Identity of this package.
PACKAGE_NAME='parallel' PACKAGE_NAME='parallel'
PACKAGE_TARNAME='parallel' PACKAGE_TARNAME='parallel'
PACKAGE_VERSION='20170822' PACKAGE_VERSION='20170922'
PACKAGE_STRING='parallel 20170822' PACKAGE_STRING='parallel 20170922'
PACKAGE_BUGREPORT='bug-parallel@gnu.org' PACKAGE_BUGREPORT='bug-parallel@gnu.org'
PACKAGE_URL='' PACKAGE_URL=''
@ -1214,7 +1214,7 @@ if test "$ac_init_help" = "long"; then
# Omit some internal or obsolete options to make the list less imposing. # Omit some internal or obsolete options to make the list less imposing.
# This message is too long to be a string in the A/UX 3.1 sh. # This message is too long to be a string in the A/UX 3.1 sh.
cat <<_ACEOF cat <<_ACEOF
\`configure' configures parallel 20170822 to adapt to many kinds of systems. \`configure' configures parallel 20170922 to adapt to many kinds of systems.
Usage: $0 [OPTION]... [VAR=VALUE]... Usage: $0 [OPTION]... [VAR=VALUE]...
@ -1281,7 +1281,7 @@ fi
if test -n "$ac_init_help"; then if test -n "$ac_init_help"; then
case $ac_init_help in case $ac_init_help in
short | recursive ) echo "Configuration of parallel 20170822:";; short | recursive ) echo "Configuration of parallel 20170922:";;
esac esac
cat <<\_ACEOF cat <<\_ACEOF
@ -1357,7 +1357,7 @@ fi
test -n "$ac_init_help" && exit $ac_status test -n "$ac_init_help" && exit $ac_status
if $ac_init_version; then if $ac_init_version; then
cat <<\_ACEOF cat <<\_ACEOF
parallel configure 20170822 parallel configure 20170922
generated by GNU Autoconf 2.69 generated by GNU Autoconf 2.69
Copyright (C) 2012 Free Software Foundation, Inc. Copyright (C) 2012 Free Software Foundation, Inc.
@ -1374,7 +1374,7 @@ cat >config.log <<_ACEOF
This file contains any messages produced by compilers while This file contains any messages produced by compilers while
running configure, to aid debugging if configure makes a mistake. running configure, to aid debugging if configure makes a mistake.
It was created by parallel $as_me 20170822, which was It was created by parallel $as_me 20170922, which was
generated by GNU Autoconf 2.69. Invocation command line was generated by GNU Autoconf 2.69. Invocation command line was
$ $0 $@ $ $0 $@
@ -2237,7 +2237,7 @@ fi
# Define the identity of the package. # Define the identity of the package.
PACKAGE='parallel' PACKAGE='parallel'
VERSION='20170822' VERSION='20170922'
cat >>confdefs.h <<_ACEOF cat >>confdefs.h <<_ACEOF
@ -2880,7 +2880,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
# report actual input values of CONFIG_FILES etc. instead of their # report actual input values of CONFIG_FILES etc. instead of their
# values after options handling. # values after options handling.
ac_log=" ac_log="
This file was extended by parallel $as_me 20170822, which was This file was extended by parallel $as_me 20170922, which was
generated by GNU Autoconf 2.69. Invocation command line was generated by GNU Autoconf 2.69. Invocation command line was
CONFIG_FILES = $CONFIG_FILES CONFIG_FILES = $CONFIG_FILES
@ -2942,7 +2942,7 @@ _ACEOF
cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
ac_cs_version="\\ ac_cs_version="\\
parallel config.status 20170822 parallel config.status 20170922
configured by $0, generated by GNU Autoconf 2.69, configured by $0, generated by GNU Autoconf 2.69,
with options \\"\$ac_cs_config\\" with options \\"\$ac_cs_config\\"

2
configure.ac
View file

@ -1,4 +1,4 @@
AC_INIT([parallel], [20170822], [bug-parallel@gnu.org]) AC_INIT([parallel], [20170922], [bug-parallel@gnu.org])
AM_INIT_AUTOMAKE([-Wall -Werror foreign]) AM_INIT_AUTOMAKE([-Wall -Werror foreign])
AC_CONFIG_HEADERS([config.h]) AC_CONFIG_HEADERS([config.h])
AC_CONFIG_FILES([ AC_CONFIG_FILES([

16
doc/release_new_version
View file

@ -198,20 +198,26 @@ to:parallel@gnu.org, bug-parallel@gnu.org
stable-bcc: Jesse Alama <jessealama@fastmail.fm> stable-bcc: Jesse Alama <jessealama@fastmail.fm>
Subject: GNU Parallel 20170922 ('Harvey/Peter Madsen') released <<[stable]>> Subject: GNU Parallel 20170922 ('Mexico City') released <<[stable]>>
GNU Parallel 20170922 ('') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/ GNU Parallel 20170922 ('Mexico City') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/
<<No new functionality was introduced so this is a good candidate for a stable release.>> <<No new functionality was introduced so this is a good candidate for a stable release.>>
Haiku of the month: Haiku of the month:
<<>> --limit can
-- Ole Tange limit jobs dynamic'ly
given a command
--ole-tange
New in this release: New in this release:
https://til.hashrocket.com/posts/ggt1jaes4y-download-all-of-humble-bundle-books-in-parallel * Use '--limit myprog' to make a dynamic job limit. Just return 0 to spawn another job, 1 to not spawn another job, and 2 to kill the youngest job.
* PARALLEL_RSYNC_OPTS and --rsync-opts sets the options for rsync (Default: -rlDzR).
* Download all of humble bundle books in parallel https://til.hashrocket.com/posts/ggt1jaes4y-download-all-of-humble-bundle-books-in-parallel
<<Citation not OK: BAMClipper: removing primers from alignments to minimize false-negative mutations in amplicon next-generation sequencing https://www.nature.com/articles/s41598-017-01703-6>> <<Citation not OK: BAMClipper: removing primers from alignments to minimize false-negative mutations in amplicon next-generation sequencing https://www.nature.com/articles/s41598-017-01703-6>>

2
src/niceload
View file

@ -24,7 +24,7 @@
use strict; use strict;
use Getopt::Long; use Getopt::Long;
$Global::progname="niceload"; $Global::progname="niceload";
$Global::version = 20170822; $Global::version = 20170922;
Getopt::Long::Configure("bundling","require_order"); Getopt::Long::Configure("bundling","require_order");
get_options_from_array(\@ARGV) || die_usage(); get_options_from_array(\@ARGV) || die_usage();
if($opt::version) { if($opt::version) {

10
src/parallel
View file

@ -1381,7 +1381,7 @@ sub check_invalid_option_combinations {
sub init_globals { sub init_globals {
# Defaults: # Defaults:
$Global::version = 20170822; $Global::version = 20170922;
$Global::progname = 'parallel'; $Global::progname = 'parallel';
$Global::infinity = 2**31; $Global::infinity = 2**31;
$Global::debug = 0; $Global::debug = 0;
@ -1415,6 +1415,7 @@ sub init_globals {
'{...}' => 's:\.[^/.]+$::; s:\.[^/.]+$::; s:\.[^/.]+$::', '{...}' => 's:\.[^/.]+$::; s:\.[^/.]+$::; s:\.[^/.]+$::',
'{/..}' => 's:.*/::; s:\.[^/.]+$::; s:\.[^/.]+$::', '{/..}' => 's:.*/::; s:\.[^/.]+$::; s:\.[^/.]+$::',
'{/...}' => 's:.*/::; s:\.[^/.]+$::; s:\.[^/.]+$::; s:\.[^/.]+$::', '{/...}' => 's:.*/::; s:\.[^/.]+$::; s:\.[^/.]+$::; s:\.[^/.]+$::',
'{choose_k}' => 'for $t (2..$#arg){ if($arg[$t-1] ge $arg[$t]) { skip() } }',
# {##} = number of jobs # {##} = number of jobs
'{##}' => '$_=total_jobs()', '{##}' => '$_=total_jobs()',
# Bash ${a:-myval} # Bash ${a:-myval}
@ -2925,7 +2926,7 @@ sub progress {
# $pctcomplete = percent of jobs completed # $pctcomplete = percent of jobs completed
# $avgtime = averaged time # $avgtime = averaged time
# $eta = smoothed eta # $eta = smoothed eta
$total ||= $Global::JobQueue->total_jobs(); $total = $Global::JobQueue->total_jobs();
my $completed = $Global::total_completed; my $completed = $Global::total_completed;
my $left = $total - $completed; my $left = $total - $completed;
if(not $completed) { if(not $completed) {
@ -5156,7 +5157,7 @@ sub limit {
my ($cmd,@args) = split /\s+/,$opt::limit; my ($cmd,@args) = split /\s+/,$opt::limit;
if($limitscripts{$cmd}) { if($limitscripts{$cmd}) {
my $tmpfile = ::tmpname("parlmt"); my $tmpfile = ::tmpname("parlmt");
$Global::unlink{$tmpfile}; ++$Global::unlink{$tmpfile};
$self->{'limitscript'} = $self->{'limitscript'} =
::spacefree(1, sprintf($limitscripts{$cmd}, ::spacefree(1, sprintf($limitscripts{$cmd},
::multiply_binary_prefix(@args),$tmpfile)); ::multiply_binary_prefix(@args),$tmpfile));
@ -9832,7 +9833,8 @@ sub new {
if($posrpl =~ s/^\{//) { if($posrpl =~ s/^\{//) {
# Only do this if the shorthand start with { # Only do this if the shorthand start with {
$prefix=~s/^\{//; $prefix=~s/^\{//;
while(s{( (?: ^|\257> ) (?: (?! \257[<>]). )*? ) # Don't replace after \257 unless \257> # Don't replace after \257 unless \257>
while(s{( (?: ^|\257> ) (?: (?! \257[<>]). )*? )
\{(-?\d+) \s* \Q$prefix\E $grp_regexp \Q$postfix\E} \{(-?\d+) \s* \Q$prefix\E $grp_regexp \Q$postfix\E}
{ {
# The start remains the same # The start remains the same

23
src/parallel.pod
View file

@ -1328,6 +1328,11 @@ Activate additional replacement strings: {+/} {+.} {+..} {+...} {..}
B<{##}> is the number of jobs to be run. It is incompatible with B<{##}> is the number of jobs to be run. It is incompatible with
B<-X>/B<-m>/B<--xargs>. B<-X>/B<-m>/B<--xargs>.
B<{choose_k}> is inspired by n choose k: Given a list of n elements,
choose k. k is the number of input sources and n is the number of
arguments in an input source. The content of the input sources must
be the same and the arguments must be unique.
The following dynamic replacement strings are also activated. They are The following dynamic replacement strings are also activated. They are
inspired by bash's parameter expansion: inspired by bash's parameter expansion:
@ -3098,6 +3103,24 @@ This also works if the input file is a file with columns:
parallel --colsep '\t' --header : echo {Name} {E-mail address} parallel --colsep '\t' --header : echo {Name} {E-mail address}
=head1 EXAMPLE: All combinations in a list
GNU B<parallel> makes all combinations when given two lists.
To make all combinations in a single list with unique values, you
repeat the list and use replacement string with a Perl expression that
skips the job if the value from input source 1 is greater than or
equal to the value from input source 2:
parallel echo {= 'if($arg[1] ge $arg[2]) { skip() }' =} ::: A B C D ::: A B C D
Or more generally:
parallel echo \
'{= for $t (2..$#arg){ if($arg[$t-1] ge $arg[$t]) { skip() } } =}' \
::: A B C D ::: A B C D ::: A B C D
=head1 EXAMPLE: From a to b and b to c =head1 EXAMPLE: From a to b and b to c
Assume you have input like: Assume you have input like:

32
src/parallel_alternatives.pod
View file

@ -1269,20 +1269,6 @@ The failed commands can be resumed with:
=back =back
=head2 DIFFERENCES BETWEEN k-bx/par AND GNU Parallel
kbx requires Haskell to work. This limits the number of platforms this
can work on.
Todo https://github.com/k-bx/par
=head2 DIFFERENCES BETWEEN machma AND GNU Parallel
Todo. Requires Go >= 1.7.
=head2 DIFFERENCES BETWEEN ClusterSSH AND GNU Parallel =head2 DIFFERENCES BETWEEN ClusterSSH AND GNU Parallel
ClusterSSH solves a different problem than GNU B<parallel>. ClusterSSH solves a different problem than GNU B<parallel>.
@ -1303,6 +1289,24 @@ GNU B<parallel> can be used as a poor-man's version of ClusterSSH:
B<parallel --nonall -S server-a,server-b do_stuff foo bar> B<parallel --nonall -S server-a,server-b do_stuff foo bar>
=head2 Todo
machma. Requires Go >= 1.7.
https://github.com/k-bx/par requires Haskell to work. This limits the
number of platforms this can work on.
https://github.com/otonvm/Parallel
https://github.com/flesler/parallel
https://github.com/kou1okada/lesser-parallel
https://github.com/Julian/Verge
https://github.com/amattn/paral
=head1 TESTING OTHER TOOLS =head1 TESTING OTHER TOOLS
There are certain issues that are very common on parallelizing There are certain issues that are very common on parallelizing

2
src/sql
View file

@ -576,7 +576,7 @@ $Global::Initfile && unlink $Global::Initfile;
exit ($err); exit ($err);
sub parse_options { sub parse_options {
$Global::version = 20170822; $Global::version = 20170922;
$Global::progname = 'sql'; $Global::progname = 'sql';
# This must be done first as this may exec myself # This must be done first as this may exec myself

9
testsuite/tests-to-run/parallel-local-ssh6.sh
View file

@ -124,8 +124,15 @@ par_remote_symlink_dir() {
ssh parallel@lo 'mkdir -p tmp; rm -rf wd; ln -s tmp wd' ssh parallel@lo 'mkdir -p tmp; rm -rf wd; ln -s tmp wd'
mkdir -p wd mkdir -p wd
touch wd/testfile touch wd/testfile
parallel --nonall --rsync-opts '--keep-dirlinks -rlDzR' -S parallel@lo --basefile wd/testfile
ssh parallel@lo rm wd && echo OK: wd is still a symlink with --rsync-opts
ssh parallel@lo 'mkdir -p tmp; rm -rf wd; ln -s tmp wd'
mkdir -p wd
touch wd/testfile
export PARALLEL_RSYNC_OPTS='--keep-dirlinks -rlDzR'
parallel --nonall -S parallel@lo --basefile wd/testfile parallel --nonall -S parallel@lo --basefile wd/testfile
ssh parallel@lo rm wd && echo OK: wd is still a symlink ssh parallel@lo rm wd && echo OK: wd is still a symlink with PARALLEL_RSYNC_OPTS
} }
export -f $(compgen -A function | grep par_) export -f $(compgen -A function | grep par_)

3
testsuite/wanted-results/parallel-local-ssh6
View file

@ -24,7 +24,8 @@ par_read_sshloginfile_from_stdin foo
par_read_sshloginfile_from_stdin /home/parallel par_read_sshloginfile_from_stdin /home/parallel
par_read_sshloginfile_from_stdin foo par_read_sshloginfile_from_stdin foo
par_remote_symlink_dir bug #51293: parallel does not preserve symlinked directory structure on remote par_remote_symlink_dir bug #51293: parallel does not preserve symlinked directory structure on remote
par_remote_symlink_dir OK: wd is still a symlink par_remote_symlink_dir OK: wd is still a symlink with --rsync-opts
par_remote_symlink_dir OK: wd is still a symlink with PARALLEL_RSYNC_OPTS
par_test_nonall ### Test --nonall par_test_nonall ### Test --nonall
par_test_nonall /home/csh par_test_nonall /home/csh
par_test_nonall /home/parallel par_test_nonall /home/parallel