mirror of
https://git.savannah.gnu.org/git/parallel.git
synced 2024-10-18 04:49:40 +00:00
parallel: Format replacement string.
This commit is contained in:
Ole Tange
parent
b4b3da4b0f
commit
67cfb6b79f
|
|
@ -156,8 +156,8 @@ am__define_uniq_tagged_files = \
|
|||
if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \
|
||||
done | $(am__uniquify_input)`
|
||||
DIST_SUBDIRS = $(SUBDIRS)
|
||||
am__DIST_COMMON = $(srcdir)/Makefile.in $(srcdir)/config.h.in COPYING \
|
||||
NEWS README TODO install-sh missing
|
||||
am__DIST_COMMON = $(srcdir)/Makefile.in $(srcdir)/config.h.in NEWS \
|
||||
README TODO install-sh missing
|
||||
DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST)
|
||||
distdir = $(PACKAGE)-$(VERSION)
|
||||
top_distdir = $(distdir)
|
||||
|
|
|
|||
|
|
@ -4,6 +4,12 @@
|
|||
|
||||
Quote of the month:
|
||||
|
||||
Recently executed a flawless live data migration of ~2.4pb using GNU parallel for scale and bash scripts.
|
||||
-- @mechanicker@twitter Dhruva
|
||||
|
||||
by extreme do you mean extremely slow? you could do it at least 100x faster using awk + grep + gnu parallel
|
||||
-- @ObssessedDev@twitter
|
||||
|
||||
gnu parallel is amazing
|
||||
-- Tarun Agarwal @axrawal
|
||||
|
||||
|
|
|
|||
|
|
@ -268,34 +268,30 @@ from:tange@gnu.org
|
|||
to:parallel@gnu.org, bug-parallel@gnu.org
|
||||
stable-bcc: Jesse Alama <jessealama@fastmail.fm>
|
||||
|
||||
Subject: GNU Parallel 20240722 ('Assange') released [stable]
|
||||
Subject: GNU Parallel 20240822 ('Southport/<<>>') released <<[stable]>>
|
||||
|
||||
GNU Parallel 20240722 ('Assange') has been released. It is available for download at: lbry://@GnuParallel:4
|
||||
GNU Parallel 20240822 ('<<>>') has been released. It is available for download at: lbry://@GnuParallel:4
|
||||
|
||||
Quote of the month:
|
||||
|
||||
parallel is frickin great for launching jobs on multiple
|
||||
machines. Ansible and Jenkins and others may be good too but I was
|
||||
able to jump right in with parallel.
|
||||
-- dwhite21787@reddit
|
||||
<<>>
|
||||
|
||||
New in this release:
|
||||
|
||||
* No new features. This is a candidate for a stable release.
|
||||
<<>>
|
||||
<<* No new features. This is a candidate for a stable release.>>
|
||||
|
||||
* Bug fixes and man page updates.
|
||||
|
||||
News about GNU Parallel:
|
||||
|
||||
* Scientific Workflows at Scale using GNU Parallel https://web.cvent.com/event/f318e73c-2230-432a-a044-b75625020543/websitePage:afd80266-008e-414b-9f94-2fd9b4dd1924?session=fe79a785-ec60-414c-8d2b-c29208f53d4c&shareLink=true
|
||||
https://contentbase.com/blog/increase-file-transfer-speed-parallel-rsync/
|
||||
|
||||
* Use GNU Parallel to render blender movies distributed by a bunch of nodes https://github.com/tfmoraes/blender_gnu_parallel_render
|
||||
https://bytefreaks.net/2024/07/27
|
||||
|
||||
* Lessons Learned from Scaling to Multi-Terabyte Datasets https://v2thegreat.com/2024/06/19/lessons-learned-from-scaling-to-multi-terabyte-datasets/
|
||||
https://medium.com/box-developer-blog/turbocharging-the-box-cli-with-gnu-parallel-ee44c48811c0
|
||||
|
||||
* Efisiensi Maksimal: Cara Paralelisasi Perintah di CLI Linux https://medium.com/@nfrozi/efisiensi-maksimal-cara-paralelisasi-perintah-di-cli-linux-f4fda3afe2a0
|
||||
|
||||
* Introduction to GNU parallel https://datascience.101workbook.org/06-hpc/06-parallel/01-intro-to-gnu-parallel/#gsc.tab=0
|
||||
<<>>
|
||||
|
||||
|
||||
GNU Parallel - For people who live life in the parallel lane.
|
||||
|
|
|
|||
|
|
@ -601,7 +601,7 @@ _parset_main() {
|
|||
fi
|
||||
if [ "$_parset_NAME" = "--version" ] ; then
|
||||
# shellcheck disable=SC2006
|
||||
echo "parset 20240722 (GNU parallel `parallel --minversion 1`)"
|
||||
echo "parset 20240723 (GNU parallel `parallel --minversion 1`)"
|
||||
echo "Copyright (C) 2007-2024 Ole Tange, http://ole.tange.dk and Free Software"
|
||||
echo "Foundation, Inc."
|
||||
echo "License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>"
|
||||
|
|
|
|||
|
|
@ -605,7 +605,7 @@ _parset_main() {
|
|||
fi
|
||||
if [ "$_parset_NAME" = "--version" ] ; then
|
||||
# shellcheck disable=SC2006
|
||||
echo "parset 20240722 (GNU parallel `parallel --minversion 1`)"
|
||||
echo "parset 20240723 (GNU parallel `parallel --minversion 1`)"
|
||||
echo "Copyright (C) 2007-2024 Ole Tange, http://ole.tange.dk and Free Software"
|
||||
echo "Foundation, Inc."
|
||||
echo "License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>"
|
||||
|
|
|
|||
|
|
@ -601,7 +601,7 @@ _parset_main() {
|
|||
fi
|
||||
if [ "$_parset_NAME" = "--version" ] ; then
|
||||
# shellcheck disable=SC2006
|
||||
echo "parset 20240722 (GNU parallel `parallel --minversion 1`)"
|
||||
echo "parset 20240723 (GNU parallel `parallel --minversion 1`)"
|
||||
echo "Copyright (C) 2007-2024 Ole Tange, http://ole.tange.dk and Free Software"
|
||||
echo "Foundation, Inc."
|
||||
echo "License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>"
|
||||
|
|
|
|||
|
|
@ -601,7 +601,7 @@ _parset_main() {
|
|||
fi
|
||||
if [ "$_parset_NAME" = "--version" ] ; then
|
||||
# shellcheck disable=SC2006
|
||||
echo "parset 20240722 (GNU parallel `parallel --minversion 1`)"
|
||||
echo "parset 20240723 (GNU parallel `parallel --minversion 1`)"
|
||||
echo "Copyright (C) 2007-2024 Ole Tange, http://ole.tange.dk and Free Software"
|
||||
echo "Foundation, Inc."
|
||||
echo "License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>"
|
||||
|
|
|
|||
|
|
@ -570,7 +570,7 @@ _parset_main() {
|
|||
fi
|
||||
if [ "$_parset_NAME" = "--version" ] ; then
|
||||
# shellcheck disable=SC2006
|
||||
echo "parset 20240722 (GNU parallel `parallel --minversion 1`)"
|
||||
echo "parset 20240723 (GNU parallel `parallel --minversion 1`)"
|
||||
echo "Copyright (C) 2007-2024 Ole Tange, http://ole.tange.dk and Free Software"
|
||||
echo "Foundation, Inc."
|
||||
echo "License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>"
|
||||
|
|
|
|||
|
|
@ -601,7 +601,7 @@ _parset_main() {
|
|||
fi
|
||||
if [ "$_parset_NAME" = "--version" ] ; then
|
||||
# shellcheck disable=SC2006
|
||||
echo "parset 20240722 (GNU parallel `parallel --minversion 1`)"
|
||||
echo "parset 20240723 (GNU parallel `parallel --minversion 1`)"
|
||||
echo "Copyright (C) 2007-2024 Ole Tange, http://ole.tange.dk and Free Software"
|
||||
echo "Foundation, Inc."
|
||||
echo "License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>"
|
||||
|
|
|
|||
|
|
@ -601,7 +601,7 @@ _parset_main() {
|
|||
fi
|
||||
if [ "$_parset_NAME" = "--version" ] ; then
|
||||
# shellcheck disable=SC2006
|
||||
echo "parset 20240722 (GNU parallel `parallel --minversion 1`)"
|
||||
echo "parset 20240723 (GNU parallel `parallel --minversion 1`)"
|
||||
echo "Copyright (C) 2007-2024 Ole Tange, http://ole.tange.dk and Free Software"
|
||||
echo "Foundation, Inc."
|
||||
echo "License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>"
|
||||
|
|
|
|||
|
|
@ -26,7 +26,7 @@
|
|||
use strict;
|
||||
use Getopt::Long;
|
||||
$Global::progname="niceload";
|
||||
$Global::version = 20240722;
|
||||
$Global::version = 20240723;
|
||||
Getopt::Long::Configure("bundling","require_order");
|
||||
get_options_from_array(\@ARGV) || die_usage();
|
||||
if($opt::version) {
|
||||
|
|
|
|||
383
src/parallel
383
src/parallel
|
|
@ -2793,7 +2793,7 @@ sub check_invalid_option_combinations() {
|
|||
|
||||
sub init_globals() {
|
||||
# Defaults:
|
||||
$Global::version = 20240722;
|
||||
$Global::version = 20240723;
|
||||
$Global::progname = 'parallel';
|
||||
$::name = "GNU Parallel";
|
||||
$Global::infinity = 2**31;
|
||||
|
|
@ -13282,6 +13282,251 @@ sub skip($) {
|
|||
package CommandLineQueue;
|
||||
|
||||
sub new($) {
|
||||
sub merge_rpl_parts(@) {
|
||||
# '{=' 'perlexpr' '=}' => '{= perlexpr =}'
|
||||
# Input:
|
||||
# @in = the @command as given by the user
|
||||
# Uses:
|
||||
# $Global::parensleft
|
||||
# $Global::parensright
|
||||
# Returns:
|
||||
# @command with parts merged to keep {= and =} as one
|
||||
my @in = @_;
|
||||
my @out;
|
||||
my $l = quotemeta($Global::parensleft);
|
||||
my $r = quotemeta($Global::parensright);
|
||||
|
||||
while(@in) {
|
||||
my $s = shift @in;
|
||||
$_ = $s;
|
||||
# Remove matching (right most) parens
|
||||
while(s/(.*)$l.*?$r/$1/os) {}
|
||||
if(/$l/o) {
|
||||
# Missing right parens
|
||||
while(@in) {
|
||||
$s .= " ".shift @in;
|
||||
$_ = $s;
|
||||
while(s/(.*)$l.*?$r/$1/os) {}
|
||||
if(not /$l/o) {
|
||||
last;
|
||||
}
|
||||
}
|
||||
}
|
||||
push @out, $s;
|
||||
}
|
||||
return @out;
|
||||
}
|
||||
|
||||
sub escape_177($) {
|
||||
# Escape \177 => \177\176
|
||||
my $s = shift;
|
||||
$Global::escape_string_present += $s =~ s/\177/\177\176/g;
|
||||
return $s;
|
||||
}
|
||||
|
||||
sub replace_parens($) {
|
||||
# Needs to match rightmost left parens (Perl defaults to leftmost)
|
||||
# to deal with: {={==} and {={==}=}
|
||||
# Replace {= -> \177< and =} -> \177>
|
||||
#
|
||||
# Complex way to do:
|
||||
# s/{=(.*)=}/\177<$1\177>/g
|
||||
# which would not work
|
||||
my $s = shift;
|
||||
$s =~ s[\Q$Global::parensleft\E # Match {=
|
||||
# Match . unless the next string is {= or =}
|
||||
# needed to force matching the shortest {= =}
|
||||
((?:(?! \Q$Global::parensleft\E|\Q$Global::parensright\E ).)*?)
|
||||
\Q$Global::parensright\E ] # Match =}
|
||||
{\177<$1\177>}gxs;
|
||||
# Now {= perlexpr =} => \177< perlexpr \177>
|
||||
return $s;
|
||||
}
|
||||
|
||||
sub replace_rpl_def($) {
|
||||
my $s = shift;
|
||||
# Replace rpl-definitions with the corresponding perl code
|
||||
for my $rpl (sort { length $b <=> length $a } keys %Global::rpl) {
|
||||
# Replace long --rpl's before short ones, as a short may be a
|
||||
# substring of a long:
|
||||
# --rpl '% s/a/b/' --rpl '%% s/b/a/'
|
||||
#
|
||||
# Replace the shorthand string (--rpl)
|
||||
# with the {= perl expr =}
|
||||
#
|
||||
# Avoid searching for shorthand strings inside existing {= perl expr =}
|
||||
#
|
||||
# Replace $$1 in {= perl expr =} with groupings in shorthand string
|
||||
#
|
||||
# parallel --rpl '{/(\.\S+)/(\.\S+)} s/$$1/$$2_REPLACE/g;' echo {/.gz/.lz} ::: UU.tar.gz
|
||||
#
|
||||
# {/.gz/.lz} =>
|
||||
# \177< $_pAr_gRp1 = ".gz";$_pAr_gRp2 = ".lz";s/${_pAr_gRp1}/${_pAr_gRp2}_REPLACE/g; \177>
|
||||
# {/.gz/.lz:%8.2s} =>
|
||||
# \177< $_pAr_gRp1 = ".gz";$_pAr_gRp2 = ".lz";s/${_pAr_gRp1}/${_pAr_gRp2}_REPLACE/g;; $_ = sprintf("%8.2f",$_); \177>
|
||||
#
|
||||
sub replacer_format {
|
||||
my $rpl = shift;
|
||||
my $unchanged = shift;
|
||||
my $position = shift;
|
||||
my $grp_regexp = shift;
|
||||
my $grp_string = shift;
|
||||
my $formatstring = shift;
|
||||
|
||||
$grp_string =~ /^${grp_regexp}$/ or
|
||||
::die_bug("Match failed: '$grp_regexp' on $grp_string");
|
||||
# Dummy entry to start $grp[n] at 1.
|
||||
my @grp = (1);
|
||||
if($] >= 5.010) {
|
||||
@grp = (1, @{^CAPTURE});
|
||||
} else {
|
||||
for(my $i = 1; defined $grp[$#grp]; $i++) {
|
||||
push @grp, eval '$'.$i;
|
||||
}
|
||||
}
|
||||
my $rv = $Global::rpl{$rpl};
|
||||
# replace $$1 with $_pAr_gRp1, $$2 with $_pAr_gRp2
|
||||
# in the code to be executed
|
||||
$rv =~ s/\$\$ (\d+)/\$_pAr_gRp$1/gx;
|
||||
# prepend with $_pAr_gRp1 = perlquote($1),
|
||||
my $set_args = "";
|
||||
for(my $i = 1;defined $grp[$i]; $i++) {
|
||||
$set_args .= "\$_pAr_gRp$i = \"" .
|
||||
::perl_quote_scalar($grp[$i]) . "\";";
|
||||
}
|
||||
# :%8.2f => %8.2f
|
||||
$formatstring =~ s/^://;
|
||||
my $formatcode = "";
|
||||
if(length $formatstring > 0) {
|
||||
$formatcode = ";\$_ = sprintf('$formatstring',\$_);";
|
||||
}
|
||||
::debug("rpl","match: $rpl ¤ $unchanged ¤ $position ¤ ".
|
||||
"$grp_regexp ¤ $formatstring\n");
|
||||
return($unchanged . "\177<" . $position . $set_args .
|
||||
$rv . $formatcode. "\177>");
|
||||
}
|
||||
sub replacer_no_format {
|
||||
replacer_format(@_);
|
||||
}
|
||||
sub replacer {
|
||||
replacer_format(@_);
|
||||
}
|
||||
if($rpl =~ /^\{/) {
|
||||
my ($prefix,$grp_regexp,$postfix) =
|
||||
# Ignore { and }
|
||||
$rpl =~ /^ \{ # {
|
||||
( [^(]* ) # Prefix (no '{' ) - e.g. %%
|
||||
( \(.*\) )? # Group capture regexp - e.g (.*)
|
||||
( [^)]* ) # Postfix (no '}' ) - e.g. end
|
||||
\} $ # }
|
||||
/xs;
|
||||
my $format_regexp = ":%.*?";
|
||||
q {
|
||||
# Regexp using named captures - kept for documentation
|
||||
# It is easier to understand than the backward compatible version
|
||||
# Look for: { position prefix group format postfix }
|
||||
while($s =~
|
||||
s{(?<unchanged> (?: ^|\177> ) (?: [^\177]*|[\177][^<>] )*?)
|
||||
\{
|
||||
(?<position> -?\d+)?
|
||||
\Q$prefix\E \s*
|
||||
(?<grp>$grp_regexp)
|
||||
\Q$postfix\E
|
||||
(?<format> $format_regexp)
|
||||
\}
|
||||
}
|
||||
{
|
||||
replacer_format($rpl, $+{unchanged}, $+{position},
|
||||
$grp_regexp, $+{grp}, $+{format});
|
||||
}gsex){};
|
||||
# Look for: { position prefix group postfix }
|
||||
while($s =~
|
||||
s{(?<unchanged> (?: ^|\177> ) (?: [^\177]*|[\177][^<>] )*?)
|
||||
\{
|
||||
(?<position> -?\d+)?
|
||||
\Q$prefix\E \s*
|
||||
(?<grp>$grp_regexp)
|
||||
\Q$postfix\E
|
||||
\}
|
||||
}
|
||||
{
|
||||
replacer_no_format($rpl, $+{unchanged},
|
||||
$+{position}, $grp_regexp,
|
||||
$+{grp});
|
||||
}gsex){}
|
||||
};
|
||||
{
|
||||
# This a rewrite of the above to perl 5.8
|
||||
# (does not use $+{...} which was introduced in 5.010
|
||||
# Look for: { position prefix group format postfix }
|
||||
while($s =~
|
||||
s{( (?: ^|\177> ) (?: [^\177]*|[\177][^<>] )*?)
|
||||
\{
|
||||
(-?\d+)?
|
||||
\Q$prefix\E \s*
|
||||
($grp_regexp)
|
||||
\Q$postfix\E
|
||||
($format_regexp)
|
||||
\}
|
||||
}
|
||||
{
|
||||
replacer_format($rpl, $1, $2,
|
||||
$grp_regexp, $3, $+);
|
||||
}gsex){}
|
||||
# Look for: { position prefix group postfix }
|
||||
while($s =~
|
||||
s{( (?: ^|\177> ) (?: [^\177]*|[\177][^<>] )*?)
|
||||
\{
|
||||
(-?\d+)?
|
||||
\Q$prefix\E \s*
|
||||
($grp_regexp)
|
||||
\Q$postfix\E
|
||||
\}
|
||||
}
|
||||
{
|
||||
replacer_no_format($rpl, $1, $2,
|
||||
$grp_regexp, $3);
|
||||
}gsex){}
|
||||
}
|
||||
} else {
|
||||
my ($prefix,$grp_regexp,$postfix) =
|
||||
$rpl =~ /^( [^(]* ) # Prefix - e.g. {%%
|
||||
( \(.*\) )? # Group capture regexp - e.g (.*)
|
||||
( [^)]* )$ # Postfix - e.g }
|
||||
/xs;
|
||||
q {
|
||||
# Regexp using named captures - kept for documentation
|
||||
# Look for: prefix group postfix
|
||||
while($s =~
|
||||
s{(?<unchanged> (?: ^|\177> ) (?: [^\177]*|[\177][^<>] )*?)
|
||||
(?<position>)
|
||||
\Q$prefix\E \s*
|
||||
(?<grp>$grp_regexp)
|
||||
\Q$postfix\E
|
||||
}
|
||||
{
|
||||
replacer($rpl, $+{unchanged}, $+{position},
|
||||
$grp_regexp, $+{grp});
|
||||
}gsex){};
|
||||
};
|
||||
{
|
||||
# This a rewrite of the above to perl 5.8
|
||||
# (does not use $+{...} which was introduced in 5.010
|
||||
# Look for: prefix group postfix
|
||||
while($s =~
|
||||
s{( (?: ^|\177> ) (?: [^\177]*|[\177][^<>] )*?)
|
||||
()
|
||||
\Q$prefix\E \s*
|
||||
($grp_regexp)
|
||||
\Q$postfix\E
|
||||
}
|
||||
{ replacer($rpl, $1, $2, $grp_regexp, $3); }gsex){};
|
||||
}
|
||||
}
|
||||
}
|
||||
return $s;
|
||||
}
|
||||
|
||||
my $class = shift;
|
||||
my $commandref = shift;
|
||||
my $read_from = shift;
|
||||
|
|
@ -13296,6 +13541,7 @@ sub new($) {
|
|||
my ($replacecount_ref, $len_ref);
|
||||
my @command = @$commandref;
|
||||
my $seq = 1;
|
||||
|
||||
# Replace replacement strings with {= perl expr =}
|
||||
# '{=' 'perlexpr' '=}' => '{= perlexpr =}'
|
||||
@command = merge_rpl_parts(@command);
|
||||
|
|
@ -13312,103 +13558,11 @@ sub new($) {
|
|||
# Skip if undefined
|
||||
defined($_) or next;
|
||||
# Escape \177 => \177\176
|
||||
$Global::escape_string_present += s/\177/\177\176/g;
|
||||
# Needs to match rightmost left parens (Perl defaults to leftmost)
|
||||
# to deal with: {={==} and {={==}=}
|
||||
# Replace {= -> \177< and =} -> \177>
|
||||
#
|
||||
# Complex way to do:
|
||||
# s/{=(.*)=}/\177<$1\177>/g
|
||||
# which would not work
|
||||
s[\Q$Global::parensleft\E # Match {=
|
||||
# Match . unless the next string is {= or =}
|
||||
# needed to force matching the shortest {= =}
|
||||
((?:(?! \Q$Global::parensleft\E|\Q$Global::parensright\E ).)*?)
|
||||
\Q$Global::parensright\E ] # Match =}
|
||||
{\177<$1\177>}gxs;
|
||||
for my $rpl (sort { length $b <=> length $a } keys %Global::rpl) {
|
||||
# Replace long --rpl's before short ones, as a short may be a
|
||||
# substring of a long:
|
||||
# --rpl '% s/a/b/' --rpl '%% s/b/a/'
|
||||
#
|
||||
# Replace the shorthand string (--rpl)
|
||||
# with the {= perl expr =}
|
||||
#
|
||||
# Avoid searching for shorthand strings inside existing {= perl expr =}
|
||||
#
|
||||
# Replace $$1 in {= perl expr =} with groupings in shorthand string
|
||||
#
|
||||
# --rpl '{/(\.\S+)/(\.\S+)} s/$$1/$$2/g;'
|
||||
# echo {/.tar/.gz} ::: UU.tar.gz
|
||||
my ($prefix,$grp_regexp,$postfix) =
|
||||
$rpl =~ /^( [^(]* ) # Prefix - e.g. {%%
|
||||
( \(.*\) )? # Group capture regexp - e.g (.*)
|
||||
( [^)]* )$ # Postfix - e.g }
|
||||
/xs;
|
||||
$grp_regexp ||= '';
|
||||
my $rplval = $Global::rpl{$rpl};
|
||||
while(s{( (?: ^|\177> ) (?: [^\177]*|[\177][^<>] )*? )
|
||||
# Don't replace after \177 unless \177>
|
||||
\Q$prefix\E $grp_regexp \Q$postfix\E}
|
||||
{
|
||||
# The start remains the same
|
||||
my $unchanged = $1;
|
||||
# Dummy entry to start at 1.
|
||||
my @grp = (1);
|
||||
# $2 = first ()-group in $grp_regexp
|
||||
# Put $2 in $grp[1], Put $3 in $grp[2]
|
||||
# so first ()-group in $grp_regexp is $grp[1];
|
||||
for(my $i = 2; defined $grp[$#grp]; $i++) {
|
||||
push @grp, eval '$'.$i;
|
||||
}
|
||||
my $rv = $rplval;
|
||||
# replace $$1 with $_pAr_gRp1, $$2 with $_pAr_gRp2
|
||||
# in the code to be executed
|
||||
$rv =~ s/\$\$ (\d+)/\$_pAr_gRp$1/gx;
|
||||
# prepend with $_pAr_gRp1 = perlquote($1),
|
||||
my $set_args = "";
|
||||
for(my $i = 1;defined $grp[$i]; $i++) {
|
||||
$set_args .= "\$_pAr_gRp$i = \"" .
|
||||
::perl_quote_scalar($grp[$i]) . "\";";
|
||||
}
|
||||
$unchanged . "\177<" . $set_args . $rv . "\177>"
|
||||
}gxes) {
|
||||
}
|
||||
# Do the same for the positional replacement strings
|
||||
$posrpl = $rpl;
|
||||
if($posrpl =~ s/^\{//) {
|
||||
# Only do this if the shorthand start with {
|
||||
$prefix=~s/^\{//;
|
||||
# Don't replace after \177 unless \177>
|
||||
while(s{( (?: ^|\177> ) (?: [^\177]*|[\177][^<>] )*? )
|
||||
\{(-?\d+) \s* \Q$prefix\E $grp_regexp \Q$postfix\E}
|
||||
{
|
||||
# The start remains the same
|
||||
my $unchanged = $1;
|
||||
my $position = $2;
|
||||
# Dummy entry to start at 1.
|
||||
my @grp = (1);
|
||||
# $3 = first ()-group in $grp_regexp
|
||||
# Put $3 in $grp[1], Put $4 in $grp[2]
|
||||
# so first ()-group in $grp_regexp is $grp[1];
|
||||
for(my $i = 3; defined $grp[$#grp]; $i++) {
|
||||
push @grp, eval '$'.$i;
|
||||
}
|
||||
my $rv = $rplval;
|
||||
# replace $$1 with $_pAr_gRp1, $$2 with $_pAr_gRp2
|
||||
# in the code to be executed
|
||||
$rv =~ s/\$\$ (\d+)/\$_pAr_gRp$1/gx;
|
||||
# prepend with $_pAr_gRp1 = perlquote($1),
|
||||
my $set_args = "";
|
||||
for(my $i = 1;defined $grp[$i]; $i++) {
|
||||
$set_args .= "\$_pAr_gRp$i = \"" .
|
||||
::perl_quote_scalar($grp[$i]) . "\";";
|
||||
}
|
||||
$unchanged . "\177<" . $position . $set_args . $rv . "\177>"
|
||||
}gxes) {
|
||||
}
|
||||
}
|
||||
}
|
||||
$_ = escape_177($_);
|
||||
# {= perl expr =} => \177< perl expr \177>
|
||||
$_ = replace_parens($_);
|
||||
# Replace rpl-definitions with the corresponding perl code
|
||||
$_ = replace_rpl_def($_);
|
||||
}
|
||||
# Add {} if no replacement strings in @command
|
||||
($replacecount_ref, $len_ref, @command) =
|
||||
|
|
@ -13445,40 +13599,7 @@ sub new($) {
|
|||
}, ref($class) || $class;
|
||||
}
|
||||
|
||||
sub merge_rpl_parts($) {
|
||||
# '{=' 'perlexpr' '=}' => '{= perlexpr =}'
|
||||
# Input:
|
||||
# @in = the @command as given by the user
|
||||
# Uses:
|
||||
# $Global::parensleft
|
||||
# $Global::parensright
|
||||
# Returns:
|
||||
# @command with parts merged to keep {= and =} as one
|
||||
my @in = @_;
|
||||
my @out;
|
||||
my $l = quotemeta($Global::parensleft);
|
||||
my $r = quotemeta($Global::parensright);
|
||||
|
||||
while(@in) {
|
||||
my $s = shift @in;
|
||||
$_ = $s;
|
||||
# Remove matching (right most) parens
|
||||
while(s/(.*)$l.*?$r/$1/os) {}
|
||||
if(/$l/o) {
|
||||
# Missing right parens
|
||||
while(@in) {
|
||||
$s .= " ".shift @in;
|
||||
$_ = $s;
|
||||
while(s/(.*)$l.*?$r/$1/os) {}
|
||||
if(not /$l/o) {
|
||||
last;
|
||||
}
|
||||
}
|
||||
}
|
||||
push @out, $s;
|
||||
}
|
||||
return @out;
|
||||
}
|
||||
|
||||
sub replacement_counts_and_lengths($$@) {
|
||||
# Count the number of different replacement strings.
|
||||
|
|
|
|||
|
|
@ -447,6 +447,27 @@ To understand positional replacement strings see B<{>I<n>B<}>.
|
|||
See also: B<{=>I<perl expression>B<=}> B<{>I<n>B<}>
|
||||
|
||||
|
||||
=item B{I<rpl>:I<format>} (alpha testing)
|
||||
|
||||
Format replacement string.
|
||||
|
||||
Use I<format> to format I<rpl>. I<format> is a format string used in
|
||||
B<printf>. I<rpl> is a replacement string.
|
||||
|
||||
Examples:
|
||||
|
||||
{#:%04d} - Job number ({#}) with 4 digits prepended with 0
|
||||
{%:%02d} - Job slot ({%}) with 2 digits prepended with 0
|
||||
{:%6s} - Input line ({}) right aligned 6 chars wide
|
||||
{/:%12s} - Basename ({/}) right aligned 12 chars wide
|
||||
{2:%8.2f} - Second input source ({2}) 8 chars wide, 2 decimals
|
||||
|
||||
Format strings also works on replacement strings defined via B<--rpl>
|
||||
that start with '{'.
|
||||
|
||||
See also: B<{}> B<{>I<n>B<}> B<--rpl>
|
||||
|
||||
|
||||
=item B<:::> I<arguments>
|
||||
|
||||
Use arguments on the command line as input source.
|
||||
|
|
|
|||
|
|
@ -788,8 +788,10 @@ mixes. Combined with B<paexec_reorder> output order can be the same as
|
|||
input order. In certain situations B<paexec> will eat the last newline
|
||||
of standard output.
|
||||
|
||||
There seems to be no way to make 4 jobs run on a remote server with 4
|
||||
cores and 16 jobs on a remote server with 16 cores.
|
||||
There seems to be no way to have the number og jobs depend on the
|
||||
number of CPU threads in a mixed server setup: E.g run 4 jobs on a
|
||||
remote server with 4 cores and 16 jobs on a remote server with 16
|
||||
cores.
|
||||
|
||||
|
||||
=head3 EXAMPLES FROM man paexec
|
||||
|
|
@ -893,7 +895,7 @@ B<parallel>.
|
|||
awk 'BEGIN {print toupper(ARGV[1])}' < tasks
|
||||
|
||||
13$ paexec -Z240 -x -t ssh -n 'server1 badhostname server2' \
|
||||
-c "awk 'BEGIN {print toupper(ARGV[1])}' " < tasks
|
||||
-c "awk 'BEGIN {print toupper(ARGV[1])}' " < tasks
|
||||
|
||||
13$ parallel --filter-hosts -S 'server1,badhostname,server2' \
|
||||
"awk 'BEGIN {print toupper(ARGV[1])}' " < tasks
|
||||
|
|
@ -1025,7 +1027,7 @@ B<parallel>.
|
|||
time -p paexec -c ~/bin/calc -n +2 -xg | grep -v success
|
||||
|
||||
20$ printf 'small1\nsmall2\nsmall3\nsmall4\nsmall5\nhuge\n' |
|
||||
time -p parallel -j2 ~/bin/calc | grep -v success
|
||||
time -p parallel -j2 ~/bin/calc | grep -v success
|
||||
|
||||
21$ printf 'small1\nsmall2\nsmall3\nsmall4\nweight: huge 6\n' |
|
||||
time -p paexec -c ~/bin/calc -n +2 -x -W1 | grep -v success
|
||||
|
|
@ -1504,10 +1506,67 @@ user uses Rust parallel it will overwrite this file.
|
|||
If /tmp/parallel runs full during the run, Rust parallel does not
|
||||
report this, but finishes with success - thereby risking data loss.
|
||||
|
||||
|
||||
https://github.com/mmstick/parallel
|
||||
(Last checked: 2016-08)
|
||||
|
||||
|
||||
=head2 DIFFERENCES BETWEEN parallelion AND GNU Parallel
|
||||
|
||||
Summary (see legend above):
|
||||
|
||||
=over
|
||||
|
||||
=item - (I2) - I4 - - -
|
||||
|
||||
=item M1 - M3 - - M6
|
||||
|
||||
=item - O2 O3 - O5 (O6) - x x
|
||||
|
||||
=item E1 - - (E4) E5 - - E8 ?
|
||||
|
||||
=item - - - - - - - - -
|
||||
|
||||
=item - -
|
||||
|
||||
=back
|
||||
|
||||
I2: I was unable to cannot get B<parallelion> to read from a file.
|
||||
|
||||
O6: There is extra output if a job fails.
|
||||
|
||||
E4: The default number of parallel jobs is the number of cpu threads.
|
||||
|
||||
-- is needed to force args not be parsed as options:
|
||||
|
||||
parallelion 'echo {}' -- Runs without -v
|
||||
|
||||
parallelion 'echo {}' Runs with -v
|
||||
|
||||
The commands are run through B<ion> shell.
|
||||
|
||||
Ctrl-C does not stop processing.
|
||||
|
||||
The B<--log> is similar to syslog - not a table.
|
||||
|
||||
The progressbar is nice.
|
||||
|
||||
B<parallelion> is fast: 0.1 ms/job. Similar to B<parallel-bash>.
|
||||
|
||||
=head3 EXAMPLES FROM parallelion
|
||||
|
||||
1$ parallelion -progress 'echo {}' {1..1000}
|
||||
|
||||
1$ parallel --bar echo {} ::: {1..1000}
|
||||
|
||||
2$ parallelion -progress 'echo {}' $(seq 1 999)
|
||||
|
||||
2$ seq 1 999 | parallel --bar echo
|
||||
|
||||
https://gitlab.redox-os.org/redox-os/parallel
|
||||
(Last checked: 2024-08)
|
||||
|
||||
|
||||
=head2 DIFFERENCES BETWEEN Rush AND GNU Parallel
|
||||
|
||||
B<rush> (https://github.com/shenwei356/rush) is written in Go and
|
||||
|
|
@ -1972,6 +2031,8 @@ https://github.com/gdm85/coshell
|
|||
|
||||
=head2 DIFFERENCES BETWEEN spread AND GNU Parallel
|
||||
|
||||
Summary (see legend above):
|
||||
|
||||
=over
|
||||
|
||||
=item - - - I4 - - I7
|
||||
|
|
@ -3591,7 +3652,7 @@ not, all the long jobs may end up in the same queue, so you may see:
|
|||
time parallel -P4 sleep {}
|
||||
(7 seconds)
|
||||
$ printf "%b\n" 1 1 1 4 1 1 1 4 1 1 1 4 |
|
||||
time ./parallel-bash.bash -p 4 -c sleep {}
|
||||
time parallel-bash -p 4 -c sleep {}
|
||||
(12 seconds)
|
||||
|
||||
Because it uses bash lists, the total number of jobs is limited to
|
||||
|
|
|
|||
|
|
@ -137,7 +137,7 @@ GetOptions(
|
|||
"help" => \$opt::dummy,
|
||||
) || exit(255);
|
||||
$Global::progname = ($0 =~ m:(^|/)([^/]+)$:)[1];
|
||||
$Global::version = 20240722;
|
||||
$Global::version = 20240723;
|
||||
if($opt::version) { version(); exit 0; }
|
||||
# Remove -D and --parallel=N
|
||||
my @s = (grep { ! /^-D$|^--parallel=\S+$/ }
|
||||
|
|
|
|||
2
src/sql
2
src/sql
|
|
@ -670,7 +670,7 @@ $Global::Initfile && unlink $Global::Initfile;
|
|||
exit ($err);
|
||||
|
||||
sub parse_options {
|
||||
$Global::version = 20240722;
|
||||
$Global::version = 20240723;
|
||||
$Global::progname = 'sql';
|
||||
|
||||
# This must be done first as this may exec myself
|
||||
|
|
|
|||
|
|
@ -131,7 +131,7 @@ install_oracle_client() {
|
|||
echo 'export ORACLE_SID=XE' >> ~/.bashrc
|
||||
fi
|
||||
# libaio
|
||||
sudo apt install libaio1t64
|
||||
sudo apt install libaio1 || sudo apt install libaio1t64
|
||||
sudo ln -s libaio.so.1t64 /usr/lib/x86_64-linux-gnu/libaio.so.1
|
||||
perl -MCPAN -e 'install DBD::Oracle'
|
||||
# TODO set up vagrant oracle server
|
||||
|
|
@ -139,6 +139,7 @@ install_oracle_client() {
|
|||
(
|
||||
cd vagrant-projects/OracleDatabase/23.4.0-Free
|
||||
echo export VM_ORACLE_PWD=`goodpasswd` >> ~/.passwords
|
||||
echo export 'ORACLE_PWD=$VM_ORACLE_PWD' >> ~/.passwords
|
||||
. ~/.passwords
|
||||
vagrant up
|
||||
)
|
||||
|
|
|
|||
|
|
@ -8,7 +8,31 @@
|
|||
# Each should be taking 3-10s and be possible to run in parallel
|
||||
# I.e.: No race conditions, no logins
|
||||
|
||||
par_pty() {
|
||||
par_format_string() {
|
||||
echo Format string + {}
|
||||
parallel echo 12.346 98.765 == {:%.3f} ::: 12.34567 ::: 98.76543
|
||||
echo Format string + {.}
|
||||
parallel echo 12.000 98.000 == {.:%.3f} ::: 12.34567 ::: 98.76543
|
||||
echo Format string + {2}
|
||||
parallel echo 98.765 == {2:%.3f} ::: 12.34567 ::: 98.76543
|
||||
echo Format string + {2.}
|
||||
parallel echo 98.000 == {2.:%.3f} ::: 12.34567 ::: 98.76543
|
||||
echo Format string + {2.}{}
|
||||
parallel echo 98.00012.34567 98.76543 == {2.:%.3f}{} ::: 12.34567 ::: 98.76543
|
||||
echo Dynamic replacement strings
|
||||
echo {dyn} + format
|
||||
parallel --rpl '{/(\d)/(.*)} s/$$1/$$2/g;' echo 12.44 98.77 == {/3/44:%8.2f} ::: 12.34567 ::: 98.76543
|
||||
echo {Positional dyn} + format
|
||||
parallel --rpl '{/(\d)/(.*)} s/$$1/$$2/g;' echo 12.44 98.77 98.765 == {/3/44:%8.2f} {2:%.3f} ::: 12.34567 ::: 98.76543
|
||||
echo {dyn__postfix}
|
||||
parallel --rpl '{/(\d)/(.*)__} s/$$1/$$2/g;' echo 12.444567 98.765444 == {/3/44__} ::: 12.34567 ::: 98.76543
|
||||
echo {dyn__postfix} + format
|
||||
parallel --rpl '{/(\d)/(.*)__} s/$$1/$$2/g;' echo 00012.44 == {/3/44__:%08.2f} ::: 12.34567
|
||||
echo dyn without {}
|
||||
parallel --rpl '/(\d)/(.*)} s/$$1/$$2/g;' echo 12.444567 98.765444 == /3/44} ::: 12.34567 ::: 98.76543
|
||||
}
|
||||
|
||||
par_pty() {
|
||||
parallel 'echo {} > {}' ::: 1 2 3
|
||||
echo 1 > files
|
||||
echo 'xargs Expect: 3 1'
|
||||
|
|
|
|||
|
|
@ -4,6 +4,8 @@
|
|||
#
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
# On a 64G machine this uses 24G RAM + 5 min
|
||||
|
||||
make stopvm >/dev/null 2>/dev/null
|
||||
TMPDIR=${TMPDIR:-/tmp}
|
||||
mkdir -p $TMPDIR
|
||||
|
|
|
|||
|
|
@ -781,6 +781,27 @@ par_eta
par_eta ETA: 0s Left: 0 AVG: 0.00s local:0/0/0%/0.0s [K
|
|||
par_exitval_signal ### Test --joblog with exitval and Test --joblog with signal -- timing dependent
|
||||
par_exitval_signal exitval=128+6 OK
|
||||
par_exitval_signal signal OK
|
||||
par_format_string Format string + {}
|
||||
par_format_string 12.346 98.765 == 12.346 98.765
|
||||
par_format_string Format string + {.}
|
||||
par_format_string 12.000 98.000 == 12.000 98.000
|
||||
par_format_string Format string + {2}
|
||||
par_format_string 98.765 == 98.765
|
||||
par_format_string Format string + {2.}
|
||||
par_format_string 98.000 == 98.000
|
||||
par_format_string Format string + {2.}{}
|
||||
par_format_string 98.00012.34567 98.76543 == 98.00012.34567 98.76543
|
||||
par_format_string Dynamic replacement strings
|
||||
par_format_string {dyn} + format
|
||||
par_format_string 12.44 98.77 == 12.44 98.77
|
||||
par_format_string {Positional dyn} + format
|
||||
par_format_string 12.44 98.77 98.765 == 12.44 98.77 98.765
|
||||
par_format_string {dyn__postfix}
|
||||
par_format_string 12.444567 98.765444 == 12.444567 98.765444
|
||||
par_format_string {dyn__postfix} + format
|
||||
par_format_string 00012.44 == 00012.44
|
||||
par_format_string dyn without {}
|
||||
par_format_string 12.444567 98.765444 == 12.444567 98.765444
|
||||
par_jobslot_repl bug #46232: {%} with --bar/--eta/--shuf or --halt xx% broken
|
||||
par_jobslot_repl 1
|
||||
par_jobslot_repl 2
|
||||
|
|
|
|||
Loading…
Reference in a new issue