parallel: Format replacement string.

This commit is contained in:
Ole Tange 2024-07-27 23:35:55 +02:00
parent b4b3da4b0f
commit 67cfb6b79f
20 changed files with 416 additions and 163 deletions

View file

@ -156,8 +156,8 @@ am__define_uniq_tagged_files = \
if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \ if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \
done | $(am__uniquify_input)` done | $(am__uniquify_input)`
DIST_SUBDIRS = $(SUBDIRS) DIST_SUBDIRS = $(SUBDIRS)
am__DIST_COMMON = $(srcdir)/Makefile.in $(srcdir)/config.h.in COPYING \ am__DIST_COMMON = $(srcdir)/Makefile.in $(srcdir)/config.h.in NEWS \
NEWS README TODO install-sh missing README TODO install-sh missing
DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST) DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST)
distdir = $(PACKAGE)-$(VERSION) distdir = $(PACKAGE)-$(VERSION)
top_distdir = $(distdir) top_distdir = $(distdir)

View file

@ -4,6 +4,12 @@
Quote of the month: Quote of the month:
Recently executed a flawless live data migration of ~2.4pb using GNU parallel for scale and bash scripts.
-- @mechanicker@twitter Dhruva
by extreme do you mean extremely slow? you could do it at least 100x faster using awk + grep + gnu parallel
-- @ObssessedDev@twitter
gnu parallel is amazing gnu parallel is amazing
-- Tarun Agarwal @axrawal -- Tarun Agarwal @axrawal

View file

@ -268,34 +268,30 @@ from:tange@gnu.org
to:parallel@gnu.org, bug-parallel@gnu.org to:parallel@gnu.org, bug-parallel@gnu.org
stable-bcc: Jesse Alama <jessealama@fastmail.fm> stable-bcc: Jesse Alama <jessealama@fastmail.fm>
Subject: GNU Parallel 20240722 ('Assange') released [stable] Subject: GNU Parallel 20240822 ('Southport/<<>>') released <<[stable]>>
GNU Parallel 20240722 ('Assange') has been released. It is available for download at: lbry://@GnuParallel:4 GNU Parallel 20240822 ('<<>>') has been released. It is available for download at: lbry://@GnuParallel:4
Quote of the month: Quote of the month:
parallel is frickin great for launching jobs on multiple <<>>
machines. Ansible and Jenkins and others may be good too but I was
able to jump right in with parallel.
-- dwhite21787@reddit
New in this release: New in this release:
* No new features. This is a candidate for a stable release. <<>>
<<* No new features. This is a candidate for a stable release.>>
* Bug fixes and man page updates. * Bug fixes and man page updates.
News about GNU Parallel: News about GNU Parallel:
* Scientific Workflows at Scale using GNU Parallel https://web.cvent.com/event/f318e73c-2230-432a-a044-b75625020543/websitePage:afd80266-008e-414b-9f94-2fd9b4dd1924?session=fe79a785-ec60-414c-8d2b-c29208f53d4c&shareLink=true https://contentbase.com/blog/increase-file-transfer-speed-parallel-rsync/
* Use GNU Parallel to render blender movies distributed by a bunch of nodes https://github.com/tfmoraes/blender_gnu_parallel_render https://bytefreaks.net/2024/07/27
* Lessons Learned from Scaling to Multi-Terabyte Datasets https://v2thegreat.com/2024/06/19/lessons-learned-from-scaling-to-multi-terabyte-datasets/ https://medium.com/box-developer-blog/turbocharging-the-box-cli-with-gnu-parallel-ee44c48811c0
* Efisiensi Maksimal: Cara Paralelisasi Perintah di CLI Linux https://medium.com/@nfrozi/efisiensi-maksimal-cara-paralelisasi-perintah-di-cli-linux-f4fda3afe2a0 <<>>
* Introduction to GNU parallel https://datascience.101workbook.org/06-hpc/06-parallel/01-intro-to-gnu-parallel/#gsc.tab=0
GNU Parallel - For people who live life in the parallel lane. GNU Parallel - For people who live life in the parallel lane.

View file

@ -601,7 +601,7 @@ _parset_main() {
fi fi
if [ "$_parset_NAME" = "--version" ] ; then if [ "$_parset_NAME" = "--version" ] ; then
# shellcheck disable=SC2006 # shellcheck disable=SC2006
echo "parset 20240722 (GNU parallel `parallel --minversion 1`)" echo "parset 20240723 (GNU parallel `parallel --minversion 1`)"
echo "Copyright (C) 2007-2024 Ole Tange, http://ole.tange.dk and Free Software" echo "Copyright (C) 2007-2024 Ole Tange, http://ole.tange.dk and Free Software"
echo "Foundation, Inc." echo "Foundation, Inc."
echo "License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>" echo "License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>"

View file

@ -605,7 +605,7 @@ _parset_main() {
fi fi
if [ "$_parset_NAME" = "--version" ] ; then if [ "$_parset_NAME" = "--version" ] ; then
# shellcheck disable=SC2006 # shellcheck disable=SC2006
echo "parset 20240722 (GNU parallel `parallel --minversion 1`)" echo "parset 20240723 (GNU parallel `parallel --minversion 1`)"
echo "Copyright (C) 2007-2024 Ole Tange, http://ole.tange.dk and Free Software" echo "Copyright (C) 2007-2024 Ole Tange, http://ole.tange.dk and Free Software"
echo "Foundation, Inc." echo "Foundation, Inc."
echo "License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>" echo "License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>"

View file

@ -601,7 +601,7 @@ _parset_main() {
fi fi
if [ "$_parset_NAME" = "--version" ] ; then if [ "$_parset_NAME" = "--version" ] ; then
# shellcheck disable=SC2006 # shellcheck disable=SC2006
echo "parset 20240722 (GNU parallel `parallel --minversion 1`)" echo "parset 20240723 (GNU parallel `parallel --minversion 1`)"
echo "Copyright (C) 2007-2024 Ole Tange, http://ole.tange.dk and Free Software" echo "Copyright (C) 2007-2024 Ole Tange, http://ole.tange.dk and Free Software"
echo "Foundation, Inc." echo "Foundation, Inc."
echo "License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>" echo "License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>"

View file

@ -601,7 +601,7 @@ _parset_main() {
fi fi
if [ "$_parset_NAME" = "--version" ] ; then if [ "$_parset_NAME" = "--version" ] ; then
# shellcheck disable=SC2006 # shellcheck disable=SC2006
echo "parset 20240722 (GNU parallel `parallel --minversion 1`)" echo "parset 20240723 (GNU parallel `parallel --minversion 1`)"
echo "Copyright (C) 2007-2024 Ole Tange, http://ole.tange.dk and Free Software" echo "Copyright (C) 2007-2024 Ole Tange, http://ole.tange.dk and Free Software"
echo "Foundation, Inc." echo "Foundation, Inc."
echo "License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>" echo "License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>"

View file

@ -570,7 +570,7 @@ _parset_main() {
fi fi
if [ "$_parset_NAME" = "--version" ] ; then if [ "$_parset_NAME" = "--version" ] ; then
# shellcheck disable=SC2006 # shellcheck disable=SC2006
echo "parset 20240722 (GNU parallel `parallel --minversion 1`)" echo "parset 20240723 (GNU parallel `parallel --minversion 1`)"
echo "Copyright (C) 2007-2024 Ole Tange, http://ole.tange.dk and Free Software" echo "Copyright (C) 2007-2024 Ole Tange, http://ole.tange.dk and Free Software"
echo "Foundation, Inc." echo "Foundation, Inc."
echo "License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>" echo "License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>"

View file

@ -601,7 +601,7 @@ _parset_main() {
fi fi
if [ "$_parset_NAME" = "--version" ] ; then if [ "$_parset_NAME" = "--version" ] ; then
# shellcheck disable=SC2006 # shellcheck disable=SC2006
echo "parset 20240722 (GNU parallel `parallel --minversion 1`)" echo "parset 20240723 (GNU parallel `parallel --minversion 1`)"
echo "Copyright (C) 2007-2024 Ole Tange, http://ole.tange.dk and Free Software" echo "Copyright (C) 2007-2024 Ole Tange, http://ole.tange.dk and Free Software"
echo "Foundation, Inc." echo "Foundation, Inc."
echo "License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>" echo "License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>"

View file

@ -601,7 +601,7 @@ _parset_main() {
fi fi
if [ "$_parset_NAME" = "--version" ] ; then if [ "$_parset_NAME" = "--version" ] ; then
# shellcheck disable=SC2006 # shellcheck disable=SC2006
echo "parset 20240722 (GNU parallel `parallel --minversion 1`)" echo "parset 20240723 (GNU parallel `parallel --minversion 1`)"
echo "Copyright (C) 2007-2024 Ole Tange, http://ole.tange.dk and Free Software" echo "Copyright (C) 2007-2024 Ole Tange, http://ole.tange.dk and Free Software"
echo "Foundation, Inc." echo "Foundation, Inc."
echo "License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>" echo "License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>"

View file

@ -26,7 +26,7 @@
use strict; use strict;
use Getopt::Long; use Getopt::Long;
$Global::progname="niceload"; $Global::progname="niceload";
$Global::version = 20240722; $Global::version = 20240723;
Getopt::Long::Configure("bundling","require_order"); Getopt::Long::Configure("bundling","require_order");
get_options_from_array(\@ARGV) || die_usage(); get_options_from_array(\@ARGV) || die_usage();
if($opt::version) { if($opt::version) {

View file

@ -2793,7 +2793,7 @@ sub check_invalid_option_combinations() {
sub init_globals() { sub init_globals() {
# Defaults: # Defaults:
$Global::version = 20240722; $Global::version = 20240723;
$Global::progname = 'parallel'; $Global::progname = 'parallel';
$::name = "GNU Parallel"; $::name = "GNU Parallel";
$Global::infinity = 2**31; $Global::infinity = 2**31;
@ -13282,6 +13282,251 @@ sub skip($) {
package CommandLineQueue; package CommandLineQueue;
sub new($) { sub new($) {
sub merge_rpl_parts(@) {
# '{=' 'perlexpr' '=}' => '{= perlexpr =}'
# Input:
# @in = the @command as given by the user
# Uses:
# $Global::parensleft
# $Global::parensright
# Returns:
# @command with parts merged to keep {= and =} as one
my @in = @_;
my @out;
my $l = quotemeta($Global::parensleft);
my $r = quotemeta($Global::parensright);
while(@in) {
my $s = shift @in;
$_ = $s;
# Remove matching (right most) parens
while(s/(.*)$l.*?$r/$1/os) {}
if(/$l/o) {
# Missing right parens
while(@in) {
$s .= " ".shift @in;
$_ = $s;
while(s/(.*)$l.*?$r/$1/os) {}
if(not /$l/o) {
last;
}
}
}
push @out, $s;
}
return @out;
}
sub escape_177($) {
# Escape \177 => \177\176
my $s = shift;
$Global::escape_string_present += $s =~ s/\177/\177\176/g;
return $s;
}
sub replace_parens($) {
# Needs to match rightmost left parens (Perl defaults to leftmost)
# to deal with: {={==} and {={==}=}
# Replace {= -> \177< and =} -> \177>
#
# Complex way to do:
# s/{=(.*)=}/\177<$1\177>/g
# which would not work
my $s = shift;
$s =~ s[\Q$Global::parensleft\E # Match {=
# Match . unless the next string is {= or =}
# needed to force matching the shortest {= =}
((?:(?! \Q$Global::parensleft\E|\Q$Global::parensright\E ).)*?)
\Q$Global::parensright\E ] # Match =}
{\177<$1\177>}gxs;
# Now {= perlexpr =} => \177< perlexpr \177>
return $s;
}
sub replace_rpl_def($) {
my $s = shift;
# Replace rpl-definitions with the corresponding perl code
for my $rpl (sort { length $b <=> length $a } keys %Global::rpl) {
# Replace long --rpl's before short ones, as a short may be a
# substring of a long:
# --rpl '% s/a/b/' --rpl '%% s/b/a/'
#
# Replace the shorthand string (--rpl)
# with the {= perl expr =}
#
# Avoid searching for shorthand strings inside existing {= perl expr =}
#
# Replace $$1 in {= perl expr =} with groupings in shorthand string
#
# parallel --rpl '{/(\.\S+)/(\.\S+)} s/$$1/$$2_REPLACE/g;' echo {/.gz/.lz} ::: UU.tar.gz
#
# {/.gz/.lz} =>
# \177< $_pAr_gRp1 = ".gz";$_pAr_gRp2 = ".lz";s/${_pAr_gRp1}/${_pAr_gRp2}_REPLACE/g; \177>
# {/.gz/.lz:%8.2s} =>
# \177< $_pAr_gRp1 = ".gz";$_pAr_gRp2 = ".lz";s/${_pAr_gRp1}/${_pAr_gRp2}_REPLACE/g;; $_ = sprintf("%8.2f",$_); \177>
#
sub replacer_format {
my $rpl = shift;
my $unchanged = shift;
my $position = shift;
my $grp_regexp = shift;
my $grp_string = shift;
my $formatstring = shift;
$grp_string =~ /^${grp_regexp}$/ or
::die_bug("Match failed: '$grp_regexp' on $grp_string");
# Dummy entry to start $grp[n] at 1.
my @grp = (1);
if($] >= 5.010) {
@grp = (1, @{^CAPTURE});
} else {
for(my $i = 1; defined $grp[$#grp]; $i++) {
push @grp, eval '$'.$i;
}
}
my $rv = $Global::rpl{$rpl};
# replace $$1 with $_pAr_gRp1, $$2 with $_pAr_gRp2
# in the code to be executed
$rv =~ s/\$\$ (\d+)/\$_pAr_gRp$1/gx;
# prepend with $_pAr_gRp1 = perlquote($1),
my $set_args = "";
for(my $i = 1;defined $grp[$i]; $i++) {
$set_args .= "\$_pAr_gRp$i = \"" .
::perl_quote_scalar($grp[$i]) . "\";";
}
# :%8.2f => %8.2f
$formatstring =~ s/^://;
my $formatcode = "";
if(length $formatstring > 0) {
$formatcode = ";\$_ = sprintf('$formatstring',\$_);";
}
::debug("rpl","match: $rpl ¤ $unchanged ¤ $position ¤ ".
"$grp_regexp ¤ $formatstring\n");
return($unchanged . "\177<" . $position . $set_args .
$rv . $formatcode. "\177>");
}
sub replacer_no_format {
replacer_format(@_);
}
sub replacer {
replacer_format(@_);
}
if($rpl =~ /^\{/) {
my ($prefix,$grp_regexp,$postfix) =
# Ignore { and }
$rpl =~ /^ \{ # {
( [^(]* ) # Prefix (no '{' ) - e.g. %%
( \(.*\) )? # Group capture regexp - e.g (.*)
( [^)]* ) # Postfix (no '}' ) - e.g. end
\} $ # }
/xs;
my $format_regexp = ":%.*?";
q {
# Regexp using named captures - kept for documentation
# It is easier to understand than the backward compatible version
# Look for: { position prefix group format postfix }
while($s =~
s{(?<unchanged> (?: ^|\177> ) (?: [^\177]*|[\177][^<>] )*?)
\{
(?<position> -?\d+)?
\Q$prefix\E \s*
(?<grp>$grp_regexp)
\Q$postfix\E
(?<format> $format_regexp)
\}
}
{
replacer_format($rpl, $+{unchanged}, $+{position},
$grp_regexp, $+{grp}, $+{format});
}gsex){};
# Look for: { position prefix group postfix }
while($s =~
s{(?<unchanged> (?: ^|\177> ) (?: [^\177]*|[\177][^<>] )*?)
\{
(?<position> -?\d+)?
\Q$prefix\E \s*
(?<grp>$grp_regexp)
\Q$postfix\E
\}
}
{
replacer_no_format($rpl, $+{unchanged},
$+{position}, $grp_regexp,
$+{grp});
}gsex){}
};
{
# This a rewrite of the above to perl 5.8
# (does not use $+{...} which was introduced in 5.010
# Look for: { position prefix group format postfix }
while($s =~
s{( (?: ^|\177> ) (?: [^\177]*|[\177][^<>] )*?)
\{
(-?\d+)?
\Q$prefix\E \s*
($grp_regexp)
\Q$postfix\E
($format_regexp)
\}
}
{
replacer_format($rpl, $1, $2,
$grp_regexp, $3, $+);
}gsex){}
# Look for: { position prefix group postfix }
while($s =~
s{( (?: ^|\177> ) (?: [^\177]*|[\177][^<>] )*?)
\{
(-?\d+)?
\Q$prefix\E \s*
($grp_regexp)
\Q$postfix\E
\}
}
{
replacer_no_format($rpl, $1, $2,
$grp_regexp, $3);
}gsex){}
}
} else {
my ($prefix,$grp_regexp,$postfix) =
$rpl =~ /^( [^(]* ) # Prefix - e.g. {%%
( \(.*\) )? # Group capture regexp - e.g (.*)
( [^)]* )$ # Postfix - e.g }
/xs;
q {
# Regexp using named captures - kept for documentation
# Look for: prefix group postfix
while($s =~
s{(?<unchanged> (?: ^|\177> ) (?: [^\177]*|[\177][^<>] )*?)
(?<position>)
\Q$prefix\E \s*
(?<grp>$grp_regexp)
\Q$postfix\E
}
{
replacer($rpl, $+{unchanged}, $+{position},
$grp_regexp, $+{grp});
}gsex){};
};
{
# This a rewrite of the above to perl 5.8
# (does not use $+{...} which was introduced in 5.010
# Look for: prefix group postfix
while($s =~
s{( (?: ^|\177> ) (?: [^\177]*|[\177][^<>] )*?)
()
\Q$prefix\E \s*
($grp_regexp)
\Q$postfix\E
}
{ replacer($rpl, $1, $2, $grp_regexp, $3); }gsex){};
}
}
}
return $s;
}
my $class = shift; my $class = shift;
my $commandref = shift; my $commandref = shift;
my $read_from = shift; my $read_from = shift;
@ -13296,6 +13541,7 @@ sub new($) {
my ($replacecount_ref, $len_ref); my ($replacecount_ref, $len_ref);
my @command = @$commandref; my @command = @$commandref;
my $seq = 1; my $seq = 1;
# Replace replacement strings with {= perl expr =} # Replace replacement strings with {= perl expr =}
# '{=' 'perlexpr' '=}' => '{= perlexpr =}' # '{=' 'perlexpr' '=}' => '{= perlexpr =}'
@command = merge_rpl_parts(@command); @command = merge_rpl_parts(@command);
@ -13312,103 +13558,11 @@ sub new($) {
# Skip if undefined # Skip if undefined
defined($_) or next; defined($_) or next;
# Escape \177 => \177\176 # Escape \177 => \177\176
$Global::escape_string_present += s/\177/\177\176/g; $_ = escape_177($_);
# Needs to match rightmost left parens (Perl defaults to leftmost) # {= perl expr =} => \177< perl expr \177>
# to deal with: {={==} and {={==}=} $_ = replace_parens($_);
# Replace {= -> \177< and =} -> \177> # Replace rpl-definitions with the corresponding perl code
# $_ = replace_rpl_def($_);
# Complex way to do:
# s/{=(.*)=}/\177<$1\177>/g
# which would not work
s[\Q$Global::parensleft\E # Match {=
# Match . unless the next string is {= or =}
# needed to force matching the shortest {= =}
((?:(?! \Q$Global::parensleft\E|\Q$Global::parensright\E ).)*?)
\Q$Global::parensright\E ] # Match =}
{\177<$1\177>}gxs;
for my $rpl (sort { length $b <=> length $a } keys %Global::rpl) {
# Replace long --rpl's before short ones, as a short may be a
# substring of a long:
# --rpl '% s/a/b/' --rpl '%% s/b/a/'
#
# Replace the shorthand string (--rpl)
# with the {= perl expr =}
#
# Avoid searching for shorthand strings inside existing {= perl expr =}
#
# Replace $$1 in {= perl expr =} with groupings in shorthand string
#
# --rpl '{/(\.\S+)/(\.\S+)} s/$$1/$$2/g;'
# echo {/.tar/.gz} ::: UU.tar.gz
my ($prefix,$grp_regexp,$postfix) =
$rpl =~ /^( [^(]* ) # Prefix - e.g. {%%
( \(.*\) )? # Group capture regexp - e.g (.*)
( [^)]* )$ # Postfix - e.g }
/xs;
$grp_regexp ||= '';
my $rplval = $Global::rpl{$rpl};
while(s{( (?: ^|\177> ) (?: [^\177]*|[\177][^<>] )*? )
# Don't replace after \177 unless \177>
\Q$prefix\E $grp_regexp \Q$postfix\E}
{
# The start remains the same
my $unchanged = $1;
# Dummy entry to start at 1.
my @grp = (1);
# $2 = first ()-group in $grp_regexp
# Put $2 in $grp[1], Put $3 in $grp[2]
# so first ()-group in $grp_regexp is $grp[1];
for(my $i = 2; defined $grp[$#grp]; $i++) {
push @grp, eval '$'.$i;
}
my $rv = $rplval;
# replace $$1 with $_pAr_gRp1, $$2 with $_pAr_gRp2
# in the code to be executed
$rv =~ s/\$\$ (\d+)/\$_pAr_gRp$1/gx;
# prepend with $_pAr_gRp1 = perlquote($1),
my $set_args = "";
for(my $i = 1;defined $grp[$i]; $i++) {
$set_args .= "\$_pAr_gRp$i = \"" .
::perl_quote_scalar($grp[$i]) . "\";";
}
$unchanged . "\177<" . $set_args . $rv . "\177>"
}gxes) {
}
# Do the same for the positional replacement strings
$posrpl = $rpl;
if($posrpl =~ s/^\{//) {
# Only do this if the shorthand start with {
$prefix=~s/^\{//;
# Don't replace after \177 unless \177>
while(s{( (?: ^|\177> ) (?: [^\177]*|[\177][^<>] )*? )
\{(-?\d+) \s* \Q$prefix\E $grp_regexp \Q$postfix\E}
{
# The start remains the same
my $unchanged = $1;
my $position = $2;
# Dummy entry to start at 1.
my @grp = (1);
# $3 = first ()-group in $grp_regexp
# Put $3 in $grp[1], Put $4 in $grp[2]
# so first ()-group in $grp_regexp is $grp[1];
for(my $i = 3; defined $grp[$#grp]; $i++) {
push @grp, eval '$'.$i;
}
my $rv = $rplval;
# replace $$1 with $_pAr_gRp1, $$2 with $_pAr_gRp2
# in the code to be executed
$rv =~ s/\$\$ (\d+)/\$_pAr_gRp$1/gx;
# prepend with $_pAr_gRp1 = perlquote($1),
my $set_args = "";
for(my $i = 1;defined $grp[$i]; $i++) {
$set_args .= "\$_pAr_gRp$i = \"" .
::perl_quote_scalar($grp[$i]) . "\";";
}
$unchanged . "\177<" . $position . $set_args . $rv . "\177>"
}gxes) {
}
}
}
} }
# Add {} if no replacement strings in @command # Add {} if no replacement strings in @command
($replacecount_ref, $len_ref, @command) = ($replacecount_ref, $len_ref, @command) =
@ -13445,40 +13599,7 @@ sub new($) {
}, ref($class) || $class; }, ref($class) || $class;
} }
sub merge_rpl_parts($) {
# '{=' 'perlexpr' '=}' => '{= perlexpr =}'
# Input:
# @in = the @command as given by the user
# Uses:
# $Global::parensleft
# $Global::parensright
# Returns:
# @command with parts merged to keep {= and =} as one
my @in = @_;
my @out;
my $l = quotemeta($Global::parensleft);
my $r = quotemeta($Global::parensright);
while(@in) {
my $s = shift @in;
$_ = $s;
# Remove matching (right most) parens
while(s/(.*)$l.*?$r/$1/os) {}
if(/$l/o) {
# Missing right parens
while(@in) {
$s .= " ".shift @in;
$_ = $s;
while(s/(.*)$l.*?$r/$1/os) {}
if(not /$l/o) {
last;
}
}
}
push @out, $s;
}
return @out;
}
sub replacement_counts_and_lengths($$@) { sub replacement_counts_and_lengths($$@) {
# Count the number of different replacement strings. # Count the number of different replacement strings.

View file

@ -447,6 +447,27 @@ To understand positional replacement strings see B<{>I<n>B<}>.
See also: B<{=>I<perl expression>B<=}> B<{>I<n>B<}> See also: B<{=>I<perl expression>B<=}> B<{>I<n>B<}>
=item B{I<rpl>:I<format>} (alpha testing)
Format replacement string.
Use I<format> to format I<rpl>. I<format> is a format string used in
B<printf>. I<rpl> is a replacement string.
Examples:
{#:%04d} - Job number ({#}) with 4 digits prepended with 0
{%:%02d} - Job slot ({%}) with 2 digits prepended with 0
{:%6s} - Input line ({}) right aligned 6 chars wide
{/:%12s} - Basename ({/}) right aligned 12 chars wide
{2:%8.2f} - Second input source ({2}) 8 chars wide, 2 decimals
Format strings also works on replacement strings defined via B<--rpl>
that start with '{'.
See also: B<{}> B<{>I<n>B<}> B<--rpl>
=item B<:::> I<arguments> =item B<:::> I<arguments>
Use arguments on the command line as input source. Use arguments on the command line as input source.

View file

@ -788,8 +788,10 @@ mixes. Combined with B<paexec_reorder> output order can be the same as
input order. In certain situations B<paexec> will eat the last newline input order. In certain situations B<paexec> will eat the last newline
of standard output. of standard output.
There seems to be no way to make 4 jobs run on a remote server with 4 There seems to be no way to have the number og jobs depend on the
cores and 16 jobs on a remote server with 16 cores. number of CPU threads in a mixed server setup: E.g run 4 jobs on a
remote server with 4 cores and 16 jobs on a remote server with 16
cores.
=head3 EXAMPLES FROM man paexec =head3 EXAMPLES FROM man paexec
@ -1504,10 +1506,67 @@ user uses Rust parallel it will overwrite this file.
If /tmp/parallel runs full during the run, Rust parallel does not If /tmp/parallel runs full during the run, Rust parallel does not
report this, but finishes with success - thereby risking data loss. report this, but finishes with success - thereby risking data loss.
https://github.com/mmstick/parallel https://github.com/mmstick/parallel
(Last checked: 2016-08) (Last checked: 2016-08)
=head2 DIFFERENCES BETWEEN parallelion AND GNU Parallel
Summary (see legend above):
=over
=item - (I2) - I4 - - -
=item M1 - M3 - - M6
=item - O2 O3 - O5 (O6) - x x
=item E1 - - (E4) E5 - - E8 ?
=item - - - - - - - - -
=item - -
=back
I2: I was unable to cannot get B<parallelion> to read from a file.
O6: There is extra output if a job fails.
E4: The default number of parallel jobs is the number of cpu threads.
-- is needed to force args not be parsed as options:
parallelion 'echo {}' -- Runs without -v
parallelion 'echo {}' Runs with -v
The commands are run through B<ion> shell.
Ctrl-C does not stop processing.
The B<--log> is similar to syslog - not a table.
The progressbar is nice.
B<parallelion> is fast: 0.1 ms/job. Similar to B<parallel-bash>.
=head3 EXAMPLES FROM parallelion
1$ parallelion -progress 'echo {}' {1..1000}
1$ parallel --bar echo {} ::: {1..1000}
2$ parallelion -progress 'echo {}' $(seq 1 999)
2$ seq 1 999 | parallel --bar echo
https://gitlab.redox-os.org/redox-os/parallel
(Last checked: 2024-08)
=head2 DIFFERENCES BETWEEN Rush AND GNU Parallel =head2 DIFFERENCES BETWEEN Rush AND GNU Parallel
B<rush> (https://github.com/shenwei356/rush) is written in Go and B<rush> (https://github.com/shenwei356/rush) is written in Go and
@ -1972,6 +2031,8 @@ https://github.com/gdm85/coshell
=head2 DIFFERENCES BETWEEN spread AND GNU Parallel =head2 DIFFERENCES BETWEEN spread AND GNU Parallel
Summary (see legend above):
=over =over
=item - - - I4 - - I7 =item - - - I4 - - I7
@ -3591,7 +3652,7 @@ not, all the long jobs may end up in the same queue, so you may see:
time parallel -P4 sleep {} time parallel -P4 sleep {}
(7 seconds) (7 seconds)
$ printf "%b\n" 1 1 1 4 1 1 1 4 1 1 1 4 | $ printf "%b\n" 1 1 1 4 1 1 1 4 1 1 1 4 |
time ./parallel-bash.bash -p 4 -c sleep {} time parallel-bash -p 4 -c sleep {}
(12 seconds) (12 seconds)
Because it uses bash lists, the total number of jobs is limited to Because it uses bash lists, the total number of jobs is limited to

View file

@ -137,7 +137,7 @@ GetOptions(
"help" => \$opt::dummy, "help" => \$opt::dummy,
) || exit(255); ) || exit(255);
$Global::progname = ($0 =~ m:(^|/)([^/]+)$:)[1]; $Global::progname = ($0 =~ m:(^|/)([^/]+)$:)[1];
$Global::version = 20240722; $Global::version = 20240723;
if($opt::version) { version(); exit 0; } if($opt::version) { version(); exit 0; }
# Remove -D and --parallel=N # Remove -D and --parallel=N
my @s = (grep { ! /^-D$|^--parallel=\S+$/ } my @s = (grep { ! /^-D$|^--parallel=\S+$/ }

View file

@ -670,7 +670,7 @@ $Global::Initfile && unlink $Global::Initfile;
exit ($err); exit ($err);
sub parse_options { sub parse_options {
$Global::version = 20240722; $Global::version = 20240723;
$Global::progname = 'sql'; $Global::progname = 'sql';
# This must be done first as this may exec myself # This must be done first as this may exec myself

View file

@ -131,7 +131,7 @@ install_oracle_client() {
echo 'export ORACLE_SID=XE' >> ~/.bashrc echo 'export ORACLE_SID=XE' >> ~/.bashrc
fi fi
# libaio # libaio
sudo apt install libaio1t64 sudo apt install libaio1 || sudo apt install libaio1t64
sudo ln -s libaio.so.1t64 /usr/lib/x86_64-linux-gnu/libaio.so.1 sudo ln -s libaio.so.1t64 /usr/lib/x86_64-linux-gnu/libaio.so.1
perl -MCPAN -e 'install DBD::Oracle' perl -MCPAN -e 'install DBD::Oracle'
# TODO set up vagrant oracle server # TODO set up vagrant oracle server
@ -139,6 +139,7 @@ install_oracle_client() {
( (
cd vagrant-projects/OracleDatabase/23.4.0-Free cd vagrant-projects/OracleDatabase/23.4.0-Free
echo export VM_ORACLE_PWD=`goodpasswd` >> ~/.passwords echo export VM_ORACLE_PWD=`goodpasswd` >> ~/.passwords
echo export 'ORACLE_PWD=$VM_ORACLE_PWD' >> ~/.passwords
. ~/.passwords . ~/.passwords
vagrant up vagrant up
) )

View file

@ -8,6 +8,30 @@
# Each should be taking 3-10s and be possible to run in parallel # Each should be taking 3-10s and be possible to run in parallel
# I.e.: No race conditions, no logins # I.e.: No race conditions, no logins
par_format_string() {
echo Format string + {}
parallel echo 12.346 98.765 == {:%.3f} ::: 12.34567 ::: 98.76543
echo Format string + {.}
parallel echo 12.000 98.000 == {.:%.3f} ::: 12.34567 ::: 98.76543
echo Format string + {2}
parallel echo 98.765 == {2:%.3f} ::: 12.34567 ::: 98.76543
echo Format string + {2.}
parallel echo 98.000 == {2.:%.3f} ::: 12.34567 ::: 98.76543
echo Format string + {2.}{}
parallel echo 98.00012.34567 98.76543 == {2.:%.3f}{} ::: 12.34567 ::: 98.76543
echo Dynamic replacement strings
echo {dyn} + format
parallel --rpl '{/(\d)/(.*)} s/$$1/$$2/g;' echo 12.44 98.77 == {/3/44:%8.2f} ::: 12.34567 ::: 98.76543
echo {Positional dyn} + format
parallel --rpl '{/(\d)/(.*)} s/$$1/$$2/g;' echo 12.44 98.77 98.765 == {/3/44:%8.2f} {2:%.3f} ::: 12.34567 ::: 98.76543
echo {dyn__postfix}
parallel --rpl '{/(\d)/(.*)__} s/$$1/$$2/g;' echo 12.444567 98.765444 == {/3/44__} ::: 12.34567 ::: 98.76543
echo {dyn__postfix} + format
parallel --rpl '{/(\d)/(.*)__} s/$$1/$$2/g;' echo 00012.44 == {/3/44__:%08.2f} ::: 12.34567
echo dyn without {}
parallel --rpl '/(\d)/(.*)} s/$$1/$$2/g;' echo 12.444567 98.765444 == /3/44} ::: 12.34567 ::: 98.76543
}
par_pty() { par_pty() {
parallel 'echo {} > {}' ::: 1 2 3 parallel 'echo {} > {}' ::: 1 2 3
echo 1 > files echo 1 > files

View file

@ -4,6 +4,8 @@
# #
# SPDX-License-Identifier: GPL-3.0-or-later # SPDX-License-Identifier: GPL-3.0-or-later
# On a 64G machine this uses 24G RAM + 5 min
make stopvm >/dev/null 2>/dev/null make stopvm >/dev/null 2>/dev/null
TMPDIR=${TMPDIR:-/tmp} TMPDIR=${TMPDIR:-/tmp}
mkdir -p $TMPDIR mkdir -p $TMPDIR

View file

@ -781,6 +781,27 @@ par_eta par_eta ETA: 0s Left: 0 AVG: 0.00s local:0/0/0%/0.0s 
par_exitval_signal ### Test --joblog with exitval and Test --joblog with signal -- timing dependent par_exitval_signal ### Test --joblog with exitval and Test --joblog with signal -- timing dependent
par_exitval_signal exitval=128+6 OK par_exitval_signal exitval=128+6 OK
par_exitval_signal signal OK par_exitval_signal signal OK
par_format_string Format string + {}
par_format_string 12.346 98.765 == 12.346 98.765
par_format_string Format string + {.}
par_format_string 12.000 98.000 == 12.000 98.000
par_format_string Format string + {2}
par_format_string 98.765 == 98.765
par_format_string Format string + {2.}
par_format_string 98.000 == 98.000
par_format_string Format string + {2.}{}
par_format_string 98.00012.34567 98.76543 == 98.00012.34567 98.76543
par_format_string Dynamic replacement strings
par_format_string {dyn} + format
par_format_string 12.44 98.77 == 12.44 98.77
par_format_string {Positional dyn} + format
par_format_string 12.44 98.77 98.765 == 12.44 98.77 98.765
par_format_string {dyn__postfix}
par_format_string 12.444567 98.765444 == 12.444567 98.765444
par_format_string {dyn__postfix} + format
par_format_string 00012.44 == 00012.44
par_format_string dyn without {}
par_format_string 12.444567 98.765444 == 12.444567 98.765444
par_jobslot_repl bug #46232: {%} with --bar/--eta/--shuf or --halt xx% broken par_jobslot_repl bug #46232: {%} with --bar/--eta/--shuf or --halt xx% broken
par_jobslot_repl 1 par_jobslot_repl 1
par_jobslot_repl 2 par_jobslot_repl 2