mirror of
https://git.savannah.gnu.org/git/parallel.git
synced 2024-12-22 20:57:53 +00:00
parallel: {choose_k} added to --plus.
This commit is contained in:
Ole Tange
parent
68114b93e7
commit
f09e0384ff
12
README
12
README
|
|
@ -44,9 +44,9 @@ document.
|
|||
|
||||
Full installation of GNU Parallel is as simple as:
|
||||
|
||||
wget http://ftpmirror.gnu.org/parallel/parallel-20170822.tar.bz2
|
||||
bzip2 -dc parallel-20170822.tar.bz2 | tar xvf -
|
||||
cd parallel-20170822
|
||||
wget http://ftpmirror.gnu.org/parallel/parallel-20170922.tar.bz2
|
||||
bzip2 -dc parallel-20170922.tar.bz2 | tar xvf -
|
||||
cd parallel-20170922
|
||||
./configure && make && sudo make install
|
||||
|
||||
|
||||
|
|
@ -55,9 +55,9 @@ Full installation of GNU Parallel is as simple as:
|
|||
If you are not root you can add ~/bin to your path and install in
|
||||
~/bin and ~/share:
|
||||
|
||||
wget http://ftpmirror.gnu.org/parallel/parallel-20170822.tar.bz2
|
||||
bzip2 -dc parallel-20170822.tar.bz2 | tar xvf -
|
||||
cd parallel-20170822
|
||||
wget http://ftpmirror.gnu.org/parallel/parallel-20170922.tar.bz2
|
||||
bzip2 -dc parallel-20170922.tar.bz2 | tar xvf -
|
||||
cd parallel-20170922
|
||||
./configure --prefix=$HOME && make && make install
|
||||
|
||||
Or if your system lacks 'make' you can simply copy src/parallel
|
||||
|
|
|
|||
20
configure
vendored
20
configure
vendored
|
|
@ -1,6 +1,6 @@
|
|||
#! /bin/sh
|
||||
# Guess values for system-dependent variables and create Makefiles.
|
||||
# Generated by GNU Autoconf 2.69 for parallel 20170822.
|
||||
# Generated by GNU Autoconf 2.69 for parallel 20170922.
|
||||
#
|
||||
# Report bugs to <bug-parallel@gnu.org>.
|
||||
#
|
||||
|
|
@ -579,8 +579,8 @@ MAKEFLAGS=
|
|||
# Identity of this package.
|
||||
PACKAGE_NAME='parallel'
|
||||
PACKAGE_TARNAME='parallel'
|
||||
PACKAGE_VERSION='20170822'
|
||||
PACKAGE_STRING='parallel 20170822'
|
||||
PACKAGE_VERSION='20170922'
|
||||
PACKAGE_STRING='parallel 20170922'
|
||||
PACKAGE_BUGREPORT='bug-parallel@gnu.org'
|
||||
PACKAGE_URL=''
|
||||
|
||||
|
|
@ -1214,7 +1214,7 @@ if test "$ac_init_help" = "long"; then
|
|||
# Omit some internal or obsolete options to make the list less imposing.
|
||||
# This message is too long to be a string in the A/UX 3.1 sh.
|
||||
cat <<_ACEOF
|
||||
\`configure' configures parallel 20170822 to adapt to many kinds of systems.
|
||||
\`configure' configures parallel 20170922 to adapt to many kinds of systems.
|
||||
|
||||
Usage: $0 [OPTION]... [VAR=VALUE]...
|
||||
|
||||
|
|
@ -1281,7 +1281,7 @@ fi
|
|||
|
||||
if test -n "$ac_init_help"; then
|
||||
case $ac_init_help in
|
||||
short | recursive ) echo "Configuration of parallel 20170822:";;
|
||||
short | recursive ) echo "Configuration of parallel 20170922:";;
|
||||
esac
|
||||
cat <<\_ACEOF
|
||||
|
||||
|
|
@ -1357,7 +1357,7 @@ fi
|
|||
test -n "$ac_init_help" && exit $ac_status
|
||||
if $ac_init_version; then
|
||||
cat <<\_ACEOF
|
||||
parallel configure 20170822
|
||||
parallel configure 20170922
|
||||
generated by GNU Autoconf 2.69
|
||||
|
||||
Copyright (C) 2012 Free Software Foundation, Inc.
|
||||
|
|
@ -1374,7 +1374,7 @@ cat >config.log <<_ACEOF
|
|||
This file contains any messages produced by compilers while
|
||||
running configure, to aid debugging if configure makes a mistake.
|
||||
|
||||
It was created by parallel $as_me 20170822, which was
|
||||
It was created by parallel $as_me 20170922, which was
|
||||
generated by GNU Autoconf 2.69. Invocation command line was
|
||||
|
||||
$ $0 $@
|
||||
|
|
@ -2237,7 +2237,7 @@ fi
|
|||
|
||||
# Define the identity of the package.
|
||||
PACKAGE='parallel'
|
||||
VERSION='20170822'
|
||||
VERSION='20170922'
|
||||
|
||||
|
||||
cat >>confdefs.h <<_ACEOF
|
||||
|
|
@ -2880,7 +2880,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
|
|||
# report actual input values of CONFIG_FILES etc. instead of their
|
||||
# values after options handling.
|
||||
ac_log="
|
||||
This file was extended by parallel $as_me 20170822, which was
|
||||
This file was extended by parallel $as_me 20170922, which was
|
||||
generated by GNU Autoconf 2.69. Invocation command line was
|
||||
|
||||
CONFIG_FILES = $CONFIG_FILES
|
||||
|
|
@ -2942,7 +2942,7 @@ _ACEOF
|
|||
cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
|
||||
ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
|
||||
ac_cs_version="\\
|
||||
parallel config.status 20170822
|
||||
parallel config.status 20170922
|
||||
configured by $0, generated by GNU Autoconf 2.69,
|
||||
with options \\"\$ac_cs_config\\"
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
AC_INIT([parallel], [20170822], [bug-parallel@gnu.org])
|
||||
AC_INIT([parallel], [20170922], [bug-parallel@gnu.org])
|
||||
AM_INIT_AUTOMAKE([-Wall -Werror foreign])
|
||||
AC_CONFIG_HEADERS([config.h])
|
||||
AC_CONFIG_FILES([
|
||||
|
|
|
|||
|
|
@ -198,20 +198,26 @@ to:parallel@gnu.org, bug-parallel@gnu.org
|
|||
stable-bcc: Jesse Alama <jessealama@fastmail.fm>
|
||||
|
||||
|
||||
Subject: GNU Parallel 20170922 ('Harvey/Peter Madsen') released <<[stable]>>
|
||||
Subject: GNU Parallel 20170922 ('Mexico City') released <<[stable]>>
|
||||
|
||||
GNU Parallel 20170922 ('') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/
|
||||
GNU Parallel 20170922 ('Mexico City') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/
|
||||
|
||||
<<No new functionality was introduced so this is a good candidate for a stable release.>>
|
||||
|
||||
Haiku of the month:
|
||||
|
||||
<<>>
|
||||
-- Ole Tange
|
||||
--limit can
|
||||
limit jobs dynamic'ly
|
||||
given a command
|
||||
--ole-tange
|
||||
|
||||
New in this release:
|
||||
|
||||
https://til.hashrocket.com/posts/ggt1jaes4y-download-all-of-humble-bundle-books-in-parallel
|
||||
* Use '--limit myprog' to make a dynamic job limit. Just return 0 to spawn another job, 1 to not spawn another job, and 2 to kill the youngest job.
|
||||
|
||||
* PARALLEL_RSYNC_OPTS and --rsync-opts sets the options for rsync (Default: -rlDzR).
|
||||
|
||||
* Download all of humble bundle books in parallel https://til.hashrocket.com/posts/ggt1jaes4y-download-all-of-humble-bundle-books-in-parallel
|
||||
|
||||
<<Citation not OK: BAMClipper: removing primers from alignments to minimize false-negative mutations in amplicon next-generation sequencing https://www.nature.com/articles/s41598-017-01703-6>>
|
||||
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@
|
|||
use strict;
|
||||
use Getopt::Long;
|
||||
$Global::progname="niceload";
|
||||
$Global::version = 20170822;
|
||||
$Global::version = 20170922;
|
||||
Getopt::Long::Configure("bundling","require_order");
|
||||
get_options_from_array(\@ARGV) || die_usage();
|
||||
if($opt::version) {
|
||||
|
|
|
|||
10
src/parallel
10
src/parallel
|
|
@ -1381,7 +1381,7 @@ sub check_invalid_option_combinations {
|
|||
|
||||
sub init_globals {
|
||||
# Defaults:
|
||||
$Global::version = 20170822;
|
||||
$Global::version = 20170922;
|
||||
$Global::progname = 'parallel';
|
||||
$Global::infinity = 2**31;
|
||||
$Global::debug = 0;
|
||||
|
|
@ -1415,6 +1415,7 @@ sub init_globals {
|
|||
'{...}' => 's:\.[^/.]+$::; s:\.[^/.]+$::; s:\.[^/.]+$::',
|
||||
'{/..}' => 's:.*/::; s:\.[^/.]+$::; s:\.[^/.]+$::',
|
||||
'{/...}' => 's:.*/::; s:\.[^/.]+$::; s:\.[^/.]+$::; s:\.[^/.]+$::',
|
||||
'{choose_k}' => 'for $t (2..$#arg){ if($arg[$t-1] ge $arg[$t]) { skip() } }',
|
||||
# {##} = number of jobs
|
||||
'{##}' => '$_=total_jobs()',
|
||||
# Bash ${a:-myval}
|
||||
|
|
@ -2925,7 +2926,7 @@ sub progress {
|
|||
# $pctcomplete = percent of jobs completed
|
||||
# $avgtime = averaged time
|
||||
# $eta = smoothed eta
|
||||
$total ||= $Global::JobQueue->total_jobs();
|
||||
$total = $Global::JobQueue->total_jobs();
|
||||
my $completed = $Global::total_completed;
|
||||
my $left = $total - $completed;
|
||||
if(not $completed) {
|
||||
|
|
@ -5156,7 +5157,7 @@ sub limit {
|
|||
my ($cmd,@args) = split /\s+/,$opt::limit;
|
||||
if($limitscripts{$cmd}) {
|
||||
my $tmpfile = ::tmpname("parlmt");
|
||||
$Global::unlink{$tmpfile};
|
||||
++$Global::unlink{$tmpfile};
|
||||
$self->{'limitscript'} =
|
||||
::spacefree(1, sprintf($limitscripts{$cmd},
|
||||
::multiply_binary_prefix(@args),$tmpfile));
|
||||
|
|
@ -9832,7 +9833,8 @@ sub new {
|
|||
if($posrpl =~ s/^\{//) {
|
||||
# Only do this if the shorthand start with {
|
||||
$prefix=~s/^\{//;
|
||||
while(s{( (?: ^|\257> ) (?: (?! \257[<>]). )*? ) # Don't replace after \257 unless \257>
|
||||
# Don't replace after \257 unless \257>
|
||||
while(s{( (?: ^|\257> ) (?: (?! \257[<>]). )*? )
|
||||
\{(-?\d+) \s* \Q$prefix\E $grp_regexp \Q$postfix\E}
|
||||
{
|
||||
# The start remains the same
|
||||
|
|
|
|||
|
|
@ -1328,6 +1328,11 @@ Activate additional replacement strings: {+/} {+.} {+..} {+...} {..}
|
|||
B<{##}> is the number of jobs to be run. It is incompatible with
|
||||
B<-X>/B<-m>/B<--xargs>.
|
||||
|
||||
B<{choose_k}> is inspired by n choose k: Given a list of n elements,
|
||||
choose k. k is the number of input sources and n is the number of
|
||||
arguments in an input source. The content of the input sources must
|
||||
be the same and the arguments must be unique.
|
||||
|
||||
The following dynamic replacement strings are also activated. They are
|
||||
inspired by bash's parameter expansion:
|
||||
|
||||
|
|
@ -3098,6 +3103,24 @@ This also works if the input file is a file with columns:
|
|||
parallel --colsep '\t' --header : echo {Name} {E-mail address}
|
||||
|
||||
|
||||
=head1 EXAMPLE: All combinations in a list
|
||||
|
||||
GNU B<parallel> makes all combinations when given two lists.
|
||||
|
||||
To make all combinations in a single list with unique values, you
|
||||
repeat the list and use replacement string with a Perl expression that
|
||||
skips the job if the value from input source 1 is greater than or
|
||||
equal to the value from input source 2:
|
||||
|
||||
parallel echo {= 'if($arg[1] ge $arg[2]) { skip() }' =} ::: A B C D ::: A B C D
|
||||
|
||||
Or more generally:
|
||||
|
||||
parallel echo \
|
||||
'{= for $t (2..$#arg){ if($arg[$t-1] ge $arg[$t]) { skip() } } =}' \
|
||||
::: A B C D ::: A B C D ::: A B C D
|
||||
|
||||
|
||||
=head1 EXAMPLE: From a to b and b to c
|
||||
|
||||
Assume you have input like:
|
||||
|
|
|
|||
|
|
@ -1269,20 +1269,6 @@ The failed commands can be resumed with:
|
|||
=back
|
||||
|
||||
|
||||
=head2 DIFFERENCES BETWEEN k-bx/par AND GNU Parallel
|
||||
|
||||
kbx requires Haskell to work. This limits the number of platforms this
|
||||
can work on.
|
||||
|
||||
Todo https://github.com/k-bx/par
|
||||
|
||||
|
||||
|
||||
=head2 DIFFERENCES BETWEEN machma AND GNU Parallel
|
||||
|
||||
Todo. Requires Go >= 1.7.
|
||||
|
||||
|
||||
=head2 DIFFERENCES BETWEEN ClusterSSH AND GNU Parallel
|
||||
|
||||
ClusterSSH solves a different problem than GNU B<parallel>.
|
||||
|
|
@ -1303,6 +1289,24 @@ GNU B<parallel> can be used as a poor-man's version of ClusterSSH:
|
|||
B<parallel --nonall -S server-a,server-b do_stuff foo bar>
|
||||
|
||||
|
||||
=head2 Todo
|
||||
|
||||
machma. Requires Go >= 1.7.
|
||||
|
||||
https://github.com/k-bx/par requires Haskell to work. This limits the
|
||||
number of platforms this can work on.
|
||||
|
||||
https://github.com/otonvm/Parallel
|
||||
|
||||
https://github.com/flesler/parallel
|
||||
|
||||
https://github.com/kou1okada/lesser-parallel
|
||||
|
||||
https://github.com/Julian/Verge
|
||||
|
||||
https://github.com/amattn/paral
|
||||
|
||||
|
||||
=head1 TESTING OTHER TOOLS
|
||||
|
||||
There are certain issues that are very common on parallelizing
|
||||
|
|
|
|||
2
src/sql
2
src/sql
|
|
@ -576,7 +576,7 @@ $Global::Initfile && unlink $Global::Initfile;
|
|||
exit ($err);
|
||||
|
||||
sub parse_options {
|
||||
$Global::version = 20170822;
|
||||
$Global::version = 20170922;
|
||||
$Global::progname = 'sql';
|
||||
|
||||
# This must be done first as this may exec myself
|
||||
|
|
|
|||
|
|
@ -124,8 +124,15 @@ par_remote_symlink_dir() {
|
|||
ssh parallel@lo 'mkdir -p tmp; rm -rf wd; ln -s tmp wd'
|
||||
mkdir -p wd
|
||||
touch wd/testfile
|
||||
parallel --nonall --rsync-opts '--keep-dirlinks -rlDzR' -S parallel@lo --basefile wd/testfile
|
||||
ssh parallel@lo rm wd && echo OK: wd is still a symlink with --rsync-opts
|
||||
|
||||
ssh parallel@lo 'mkdir -p tmp; rm -rf wd; ln -s tmp wd'
|
||||
mkdir -p wd
|
||||
touch wd/testfile
|
||||
export PARALLEL_RSYNC_OPTS='--keep-dirlinks -rlDzR'
|
||||
parallel --nonall -S parallel@lo --basefile wd/testfile
|
||||
ssh parallel@lo rm wd && echo OK: wd is still a symlink
|
||||
ssh parallel@lo rm wd && echo OK: wd is still a symlink with PARALLEL_RSYNC_OPTS
|
||||
}
|
||||
|
||||
export -f $(compgen -A function | grep par_)
|
||||
|
|
|
|||
|
|
@ -24,7 +24,8 @@ par_read_sshloginfile_from_stdin foo
|
|||
par_read_sshloginfile_from_stdin /home/parallel
|
||||
par_read_sshloginfile_from_stdin foo
|
||||
par_remote_symlink_dir bug #51293: parallel does not preserve symlinked directory structure on remote
|
||||
par_remote_symlink_dir OK: wd is still a symlink
|
||||
par_remote_symlink_dir OK: wd is still a symlink with --rsync-opts
|
||||
par_remote_symlink_dir OK: wd is still a symlink with PARALLEL_RSYNC_OPTS
|
||||
par_test_nonall ### Test --nonall
|
||||
par_test_nonall /home/csh
|
||||
par_test_nonall /home/parallel
|
||||
|
|
|
|||
Loading…
Reference in a new issue