From 67cfb6b79fd1777353436a0b68b32a3f632b024d Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Sat, 27 Jul 2024 23:35:55 +0200 Subject: [PATCH] parallel: Format replacement string. --- Makefile.in | 4 +- doc/haikus | 6 + doc/release_new_version | 22 +- src/env_parallel.ash | 2 +- src/env_parallel.bash | 2 +- src/env_parallel.dash | 2 +- src/env_parallel.ksh | 2 +- src/env_parallel.mksh | 2 +- src/env_parallel.sh | 2 +- src/env_parallel.zsh | 2 +- src/niceload | 2 +- src/parallel | 383 ++++++++++++------- src/parallel.pod | 21 + src/parallel_alternatives.pod | 71 +++- src/parsort | 2 +- src/sql | 2 +- testsuite/REQUIREMENTS | 3 +- testsuite/tests-to-run/parallel-local-3s.sh | 26 +- testsuite/tests-to-run/parallel-local-mem.sh | 2 + testsuite/wanted-results/parallel-local-3s | 21 + 20 files changed, 416 insertions(+), 163 deletions(-) diff --git a/Makefile.in b/Makefile.in index 0a7f63e5..4739f6a6 100644 --- a/Makefile.in +++ b/Makefile.in @@ -156,8 +156,8 @@ am__define_uniq_tagged_files = \ if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \ done | $(am__uniquify_input)` DIST_SUBDIRS = $(SUBDIRS) -am__DIST_COMMON = $(srcdir)/Makefile.in $(srcdir)/config.h.in COPYING \ - NEWS README TODO install-sh missing +am__DIST_COMMON = $(srcdir)/Makefile.in $(srcdir)/config.h.in NEWS \ + README TODO install-sh missing DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST) distdir = $(PACKAGE)-$(VERSION) top_distdir = $(distdir) diff --git a/doc/haikus b/doc/haikus index 9c5d9b2d..bc298eaa 100644 --- a/doc/haikus +++ b/doc/haikus @@ -4,6 +4,12 @@ Quote of the month: + Recently executed a flawless live data migration of ~2.4pb using GNU parallel for scale and bash scripts. + -- @mechanicker@twitter Dhruva + + by extreme do you mean extremely slow? you could do it at least 100x faster using awk + grep + gnu parallel + -- @ObssessedDev@twitter + gnu parallel is amazing -- Tarun Agarwal @axrawal diff --git a/doc/release_new_version b/doc/release_new_version index 461f946a..871dd9c4 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -268,34 +268,30 @@ from:tange@gnu.org to:parallel@gnu.org, bug-parallel@gnu.org stable-bcc: Jesse Alama -Subject: GNU Parallel 20240722 ('Assange') released [stable] +Subject: GNU Parallel 20240822 ('Southport/<<>>') released <<[stable]>> -GNU Parallel 20240722 ('Assange') has been released. It is available for download at: lbry://@GnuParallel:4 +GNU Parallel 20240822 ('<<>>') has been released. It is available for download at: lbry://@GnuParallel:4 Quote of the month: - parallel is frickin great for launching jobs on multiple - machines. Ansible and Jenkins and others may be good too but I was - able to jump right in with parallel. - -- dwhite21787@reddit + <<>> New in this release: -* No new features. This is a candidate for a stable release. +<<>> +<<* No new features. This is a candidate for a stable release.>> * Bug fixes and man page updates. News about GNU Parallel: -* Scientific Workflows at Scale using GNU Parallel https://web.cvent.com/event/f318e73c-2230-432a-a044-b75625020543/websitePage:afd80266-008e-414b-9f94-2fd9b4dd1924?session=fe79a785-ec60-414c-8d2b-c29208f53d4c&shareLink=true +https://contentbase.com/blog/increase-file-transfer-speed-parallel-rsync/ -* Use GNU Parallel to render blender movies distributed by a bunch of nodes https://github.com/tfmoraes/blender_gnu_parallel_render +https://bytefreaks.net/2024/07/27 -* Lessons Learned from Scaling to Multi-Terabyte Datasets https://v2thegreat.com/2024/06/19/lessons-learned-from-scaling-to-multi-terabyte-datasets/ +https://medium.com/box-developer-blog/turbocharging-the-box-cli-with-gnu-parallel-ee44c48811c0 -* Efisiensi Maksimal: Cara Paralelisasi Perintah di CLI Linux https://medium.com/@nfrozi/efisiensi-maksimal-cara-paralelisasi-perintah-di-cli-linux-f4fda3afe2a0 - -* Introduction to GNU parallel https://datascience.101workbook.org/06-hpc/06-parallel/01-intro-to-gnu-parallel/#gsc.tab=0 +<<>> GNU Parallel - For people who live life in the parallel lane. diff --git a/src/env_parallel.ash b/src/env_parallel.ash index eb850f17..f060a607 100755 --- a/src/env_parallel.ash +++ b/src/env_parallel.ash @@ -601,7 +601,7 @@ _parset_main() { fi if [ "$_parset_NAME" = "--version" ] ; then # shellcheck disable=SC2006 - echo "parset 20240722 (GNU parallel `parallel --minversion 1`)" + echo "parset 20240723 (GNU parallel `parallel --minversion 1`)" echo "Copyright (C) 2007-2024 Ole Tange, http://ole.tange.dk and Free Software" echo "Foundation, Inc." echo "License GPLv3+: GNU GPL version 3 or later " diff --git a/src/env_parallel.bash b/src/env_parallel.bash index 57c8c32f..40bbbffe 100755 --- a/src/env_parallel.bash +++ b/src/env_parallel.bash @@ -605,7 +605,7 @@ _parset_main() { fi if [ "$_parset_NAME" = "--version" ] ; then # shellcheck disable=SC2006 - echo "parset 20240722 (GNU parallel `parallel --minversion 1`)" + echo "parset 20240723 (GNU parallel `parallel --minversion 1`)" echo "Copyright (C) 2007-2024 Ole Tange, http://ole.tange.dk and Free Software" echo "Foundation, Inc." echo "License GPLv3+: GNU GPL version 3 or later " diff --git a/src/env_parallel.dash b/src/env_parallel.dash index fc9cae22..c819e7c6 100755 --- a/src/env_parallel.dash +++ b/src/env_parallel.dash @@ -601,7 +601,7 @@ _parset_main() { fi if [ "$_parset_NAME" = "--version" ] ; then # shellcheck disable=SC2006 - echo "parset 20240722 (GNU parallel `parallel --minversion 1`)" + echo "parset 20240723 (GNU parallel `parallel --minversion 1`)" echo "Copyright (C) 2007-2024 Ole Tange, http://ole.tange.dk and Free Software" echo "Foundation, Inc." echo "License GPLv3+: GNU GPL version 3 or later " diff --git a/src/env_parallel.ksh b/src/env_parallel.ksh index e9192214..4c51ba9f 100755 --- a/src/env_parallel.ksh +++ b/src/env_parallel.ksh @@ -601,7 +601,7 @@ _parset_main() { fi if [ "$_parset_NAME" = "--version" ] ; then # shellcheck disable=SC2006 - echo "parset 20240722 (GNU parallel `parallel --minversion 1`)" + echo "parset 20240723 (GNU parallel `parallel --minversion 1`)" echo "Copyright (C) 2007-2024 Ole Tange, http://ole.tange.dk and Free Software" echo "Foundation, Inc." echo "License GPLv3+: GNU GPL version 3 or later " diff --git a/src/env_parallel.mksh b/src/env_parallel.mksh index c2b9f228..b102c505 100644 --- a/src/env_parallel.mksh +++ b/src/env_parallel.mksh @@ -570,7 +570,7 @@ _parset_main() { fi if [ "$_parset_NAME" = "--version" ] ; then # shellcheck disable=SC2006 - echo "parset 20240722 (GNU parallel `parallel --minversion 1`)" + echo "parset 20240723 (GNU parallel `parallel --minversion 1`)" echo "Copyright (C) 2007-2024 Ole Tange, http://ole.tange.dk and Free Software" echo "Foundation, Inc." echo "License GPLv3+: GNU GPL version 3 or later " diff --git a/src/env_parallel.sh b/src/env_parallel.sh index e04d4f23..d2ea36d9 100755 --- a/src/env_parallel.sh +++ b/src/env_parallel.sh @@ -601,7 +601,7 @@ _parset_main() { fi if [ "$_parset_NAME" = "--version" ] ; then # shellcheck disable=SC2006 - echo "parset 20240722 (GNU parallel `parallel --minversion 1`)" + echo "parset 20240723 (GNU parallel `parallel --minversion 1`)" echo "Copyright (C) 2007-2024 Ole Tange, http://ole.tange.dk and Free Software" echo "Foundation, Inc." echo "License GPLv3+: GNU GPL version 3 or later " diff --git a/src/env_parallel.zsh b/src/env_parallel.zsh index 0e4e4537..6e0d811b 100755 --- a/src/env_parallel.zsh +++ b/src/env_parallel.zsh @@ -601,7 +601,7 @@ _parset_main() { fi if [ "$_parset_NAME" = "--version" ] ; then # shellcheck disable=SC2006 - echo "parset 20240722 (GNU parallel `parallel --minversion 1`)" + echo "parset 20240723 (GNU parallel `parallel --minversion 1`)" echo "Copyright (C) 2007-2024 Ole Tange, http://ole.tange.dk and Free Software" echo "Foundation, Inc." echo "License GPLv3+: GNU GPL version 3 or later " diff --git a/src/niceload b/src/niceload index 9503ba40..9348a762 100755 --- a/src/niceload +++ b/src/niceload @@ -26,7 +26,7 @@ use strict; use Getopt::Long; $Global::progname="niceload"; -$Global::version = 20240722; +$Global::version = 20240723; Getopt::Long::Configure("bundling","require_order"); get_options_from_array(\@ARGV) || die_usage(); if($opt::version) { diff --git a/src/parallel b/src/parallel index 84a04fc3..21a38bba 100755 --- a/src/parallel +++ b/src/parallel @@ -2793,7 +2793,7 @@ sub check_invalid_option_combinations() { sub init_globals() { # Defaults: - $Global::version = 20240722; + $Global::version = 20240723; $Global::progname = 'parallel'; $::name = "GNU Parallel"; $Global::infinity = 2**31; @@ -13282,6 +13282,251 @@ sub skip($) { package CommandLineQueue; sub new($) { + sub merge_rpl_parts(@) { + # '{=' 'perlexpr' '=}' => '{= perlexpr =}' + # Input: + # @in = the @command as given by the user + # Uses: + # $Global::parensleft + # $Global::parensright + # Returns: + # @command with parts merged to keep {= and =} as one + my @in = @_; + my @out; + my $l = quotemeta($Global::parensleft); + my $r = quotemeta($Global::parensright); + + while(@in) { + my $s = shift @in; + $_ = $s; + # Remove matching (right most) parens + while(s/(.*)$l.*?$r/$1/os) {} + if(/$l/o) { + # Missing right parens + while(@in) { + $s .= " ".shift @in; + $_ = $s; + while(s/(.*)$l.*?$r/$1/os) {} + if(not /$l/o) { + last; + } + } + } + push @out, $s; + } + return @out; + } + + sub escape_177($) { + # Escape \177 => \177\176 + my $s = shift; + $Global::escape_string_present += $s =~ s/\177/\177\176/g; + return $s; + } + + sub replace_parens($) { + # Needs to match rightmost left parens (Perl defaults to leftmost) + # to deal with: {={==} and {={==}=} + # Replace {= -> \177< and =} -> \177> + # + # Complex way to do: + # s/{=(.*)=}/\177<$1\177>/g + # which would not work + my $s = shift; + $s =~ s[\Q$Global::parensleft\E # Match {= + # Match . unless the next string is {= or =} + # needed to force matching the shortest {= =} + ((?:(?! \Q$Global::parensleft\E|\Q$Global::parensright\E ).)*?) + \Q$Global::parensright\E ] # Match =} + {\177<$1\177>}gxs; + # Now {= perlexpr =} => \177< perlexpr \177> + return $s; + } + + sub replace_rpl_def($) { + my $s = shift; + # Replace rpl-definitions with the corresponding perl code + for my $rpl (sort { length $b <=> length $a } keys %Global::rpl) { + # Replace long --rpl's before short ones, as a short may be a + # substring of a long: + # --rpl '% s/a/b/' --rpl '%% s/b/a/' + # + # Replace the shorthand string (--rpl) + # with the {= perl expr =} + # + # Avoid searching for shorthand strings inside existing {= perl expr =} + # + # Replace $$1 in {= perl expr =} with groupings in shorthand string + # + # parallel --rpl '{/(\.\S+)/(\.\S+)} s/$$1/$$2_REPLACE/g;' echo {/.gz/.lz} ::: UU.tar.gz + # + # {/.gz/.lz} => + # \177< $_pAr_gRp1 = ".gz";$_pAr_gRp2 = ".lz";s/${_pAr_gRp1}/${_pAr_gRp2}_REPLACE/g; \177> + # {/.gz/.lz:%8.2s} => + # \177< $_pAr_gRp1 = ".gz";$_pAr_gRp2 = ".lz";s/${_pAr_gRp1}/${_pAr_gRp2}_REPLACE/g;; $_ = sprintf("%8.2f",$_); \177> + # + sub replacer_format { + my $rpl = shift; + my $unchanged = shift; + my $position = shift; + my $grp_regexp = shift; + my $grp_string = shift; + my $formatstring = shift; + + $grp_string =~ /^${grp_regexp}$/ or + ::die_bug("Match failed: '$grp_regexp' on $grp_string"); + # Dummy entry to start $grp[n] at 1. + my @grp = (1); + if($] >= 5.010) { + @grp = (1, @{^CAPTURE}); + } else { + for(my $i = 1; defined $grp[$#grp]; $i++) { + push @grp, eval '$'.$i; + } + } + my $rv = $Global::rpl{$rpl}; + # replace $$1 with $_pAr_gRp1, $$2 with $_pAr_gRp2 + # in the code to be executed + $rv =~ s/\$\$ (\d+)/\$_pAr_gRp$1/gx; + # prepend with $_pAr_gRp1 = perlquote($1), + my $set_args = ""; + for(my $i = 1;defined $grp[$i]; $i++) { + $set_args .= "\$_pAr_gRp$i = \"" . + ::perl_quote_scalar($grp[$i]) . "\";"; + } + # :%8.2f => %8.2f + $formatstring =~ s/^://; + my $formatcode = ""; + if(length $formatstring > 0) { + $formatcode = ";\$_ = sprintf('$formatstring',\$_);"; + } + ::debug("rpl","match: $rpl ¤ $unchanged ¤ $position ¤ ". + "$grp_regexp ¤ $formatstring\n"); + return($unchanged . "\177<" . $position . $set_args . + $rv . $formatcode. "\177>"); + } + sub replacer_no_format { + replacer_format(@_); + } + sub replacer { + replacer_format(@_); + } + if($rpl =~ /^\{/) { + my ($prefix,$grp_regexp,$postfix) = + # Ignore { and } + $rpl =~ /^ \{ # { + ( [^(]* ) # Prefix (no '{' ) - e.g. %% + ( \(.*\) )? # Group capture regexp - e.g (.*) + ( [^)]* ) # Postfix (no '}' ) - e.g. end + \} $ # } + /xs; + my $format_regexp = ":%.*?"; + q { + # Regexp using named captures - kept for documentation + # It is easier to understand than the backward compatible version + # Look for: { position prefix group format postfix } + while($s =~ + s{(? (?: ^|\177> ) (?: [^\177]*|[\177][^<>] )*?) + \{ + (? -?\d+)? + \Q$prefix\E \s* + (?$grp_regexp) + \Q$postfix\E + (? $format_regexp) + \} + } + { + replacer_format($rpl, $+{unchanged}, $+{position}, + $grp_regexp, $+{grp}, $+{format}); + }gsex){}; + # Look for: { position prefix group postfix } + while($s =~ + s{(? (?: ^|\177> ) (?: [^\177]*|[\177][^<>] )*?) + \{ + (? -?\d+)? + \Q$prefix\E \s* + (?$grp_regexp) + \Q$postfix\E + \} + } + { + replacer_no_format($rpl, $+{unchanged}, + $+{position}, $grp_regexp, + $+{grp}); + }gsex){} + }; + { + # This a rewrite of the above to perl 5.8 + # (does not use $+{...} which was introduced in 5.010 + # Look for: { position prefix group format postfix } + while($s =~ + s{( (?: ^|\177> ) (?: [^\177]*|[\177][^<>] )*?) + \{ + (-?\d+)? + \Q$prefix\E \s* + ($grp_regexp) + \Q$postfix\E + ($format_regexp) + \} + } + { + replacer_format($rpl, $1, $2, + $grp_regexp, $3, $+); + }gsex){} + # Look for: { position prefix group postfix } + while($s =~ + s{( (?: ^|\177> ) (?: [^\177]*|[\177][^<>] )*?) + \{ + (-?\d+)? + \Q$prefix\E \s* + ($grp_regexp) + \Q$postfix\E + \} + } + { + replacer_no_format($rpl, $1, $2, + $grp_regexp, $3); + }gsex){} + } + } else { + my ($prefix,$grp_regexp,$postfix) = + $rpl =~ /^( [^(]* ) # Prefix - e.g. {%% + ( \(.*\) )? # Group capture regexp - e.g (.*) + ( [^)]* )$ # Postfix - e.g } + /xs; + q { + # Regexp using named captures - kept for documentation + # Look for: prefix group postfix + while($s =~ + s{(? (?: ^|\177> ) (?: [^\177]*|[\177][^<>] )*?) + (?) + \Q$prefix\E \s* + (?$grp_regexp) + \Q$postfix\E + } + { + replacer($rpl, $+{unchanged}, $+{position}, + $grp_regexp, $+{grp}); + }gsex){}; + }; + { + # This a rewrite of the above to perl 5.8 + # (does not use $+{...} which was introduced in 5.010 + # Look for: prefix group postfix + while($s =~ + s{( (?: ^|\177> ) (?: [^\177]*|[\177][^<>] )*?) + () + \Q$prefix\E \s* + ($grp_regexp) + \Q$postfix\E + } + { replacer($rpl, $1, $2, $grp_regexp, $3); }gsex){}; + } + } + } + return $s; + } + my $class = shift; my $commandref = shift; my $read_from = shift; @@ -13296,6 +13541,7 @@ sub new($) { my ($replacecount_ref, $len_ref); my @command = @$commandref; my $seq = 1; + # Replace replacement strings with {= perl expr =} # '{=' 'perlexpr' '=}' => '{= perlexpr =}' @command = merge_rpl_parts(@command); @@ -13312,103 +13558,11 @@ sub new($) { # Skip if undefined defined($_) or next; # Escape \177 => \177\176 - $Global::escape_string_present += s/\177/\177\176/g; - # Needs to match rightmost left parens (Perl defaults to leftmost) - # to deal with: {={==} and {={==}=} - # Replace {= -> \177< and =} -> \177> - # - # Complex way to do: - # s/{=(.*)=}/\177<$1\177>/g - # which would not work - s[\Q$Global::parensleft\E # Match {= - # Match . unless the next string is {= or =} - # needed to force matching the shortest {= =} - ((?:(?! \Q$Global::parensleft\E|\Q$Global::parensright\E ).)*?) - \Q$Global::parensright\E ] # Match =} - {\177<$1\177>}gxs; - for my $rpl (sort { length $b <=> length $a } keys %Global::rpl) { - # Replace long --rpl's before short ones, as a short may be a - # substring of a long: - # --rpl '% s/a/b/' --rpl '%% s/b/a/' - # - # Replace the shorthand string (--rpl) - # with the {= perl expr =} - # - # Avoid searching for shorthand strings inside existing {= perl expr =} - # - # Replace $$1 in {= perl expr =} with groupings in shorthand string - # - # --rpl '{/(\.\S+)/(\.\S+)} s/$$1/$$2/g;' - # echo {/.tar/.gz} ::: UU.tar.gz - my ($prefix,$grp_regexp,$postfix) = - $rpl =~ /^( [^(]* ) # Prefix - e.g. {%% - ( \(.*\) )? # Group capture regexp - e.g (.*) - ( [^)]* )$ # Postfix - e.g } - /xs; - $grp_regexp ||= ''; - my $rplval = $Global::rpl{$rpl}; - while(s{( (?: ^|\177> ) (?: [^\177]*|[\177][^<>] )*? ) - # Don't replace after \177 unless \177> - \Q$prefix\E $grp_regexp \Q$postfix\E} - { - # The start remains the same - my $unchanged = $1; - # Dummy entry to start at 1. - my @grp = (1); - # $2 = first ()-group in $grp_regexp - # Put $2 in $grp[1], Put $3 in $grp[2] - # so first ()-group in $grp_regexp is $grp[1]; - for(my $i = 2; defined $grp[$#grp]; $i++) { - push @grp, eval '$'.$i; - } - my $rv = $rplval; - # replace $$1 with $_pAr_gRp1, $$2 with $_pAr_gRp2 - # in the code to be executed - $rv =~ s/\$\$ (\d+)/\$_pAr_gRp$1/gx; - # prepend with $_pAr_gRp1 = perlquote($1), - my $set_args = ""; - for(my $i = 1;defined $grp[$i]; $i++) { - $set_args .= "\$_pAr_gRp$i = \"" . - ::perl_quote_scalar($grp[$i]) . "\";"; - } - $unchanged . "\177<" . $set_args . $rv . "\177>" - }gxes) { - } - # Do the same for the positional replacement strings - $posrpl = $rpl; - if($posrpl =~ s/^\{//) { - # Only do this if the shorthand start with { - $prefix=~s/^\{//; - # Don't replace after \177 unless \177> - while(s{( (?: ^|\177> ) (?: [^\177]*|[\177][^<>] )*? ) - \{(-?\d+) \s* \Q$prefix\E $grp_regexp \Q$postfix\E} - { - # The start remains the same - my $unchanged = $1; - my $position = $2; - # Dummy entry to start at 1. - my @grp = (1); - # $3 = first ()-group in $grp_regexp - # Put $3 in $grp[1], Put $4 in $grp[2] - # so first ()-group in $grp_regexp is $grp[1]; - for(my $i = 3; defined $grp[$#grp]; $i++) { - push @grp, eval '$'.$i; - } - my $rv = $rplval; - # replace $$1 with $_pAr_gRp1, $$2 with $_pAr_gRp2 - # in the code to be executed - $rv =~ s/\$\$ (\d+)/\$_pAr_gRp$1/gx; - # prepend with $_pAr_gRp1 = perlquote($1), - my $set_args = ""; - for(my $i = 1;defined $grp[$i]; $i++) { - $set_args .= "\$_pAr_gRp$i = \"" . - ::perl_quote_scalar($grp[$i]) . "\";"; - } - $unchanged . "\177<" . $position . $set_args . $rv . "\177>" - }gxes) { - } - } - } + $_ = escape_177($_); + # {= perl expr =} => \177< perl expr \177> + $_ = replace_parens($_); + # Replace rpl-definitions with the corresponding perl code + $_ = replace_rpl_def($_); } # Add {} if no replacement strings in @command ($replacecount_ref, $len_ref, @command) = @@ -13445,40 +13599,7 @@ sub new($) { }, ref($class) || $class; } -sub merge_rpl_parts($) { - # '{=' 'perlexpr' '=}' => '{= perlexpr =}' - # Input: - # @in = the @command as given by the user - # Uses: - # $Global::parensleft - # $Global::parensright - # Returns: - # @command with parts merged to keep {= and =} as one - my @in = @_; - my @out; - my $l = quotemeta($Global::parensleft); - my $r = quotemeta($Global::parensright); - while(@in) { - my $s = shift @in; - $_ = $s; - # Remove matching (right most) parens - while(s/(.*)$l.*?$r/$1/os) {} - if(/$l/o) { - # Missing right parens - while(@in) { - $s .= " ".shift @in; - $_ = $s; - while(s/(.*)$l.*?$r/$1/os) {} - if(not /$l/o) { - last; - } - } - } - push @out, $s; - } - return @out; -} sub replacement_counts_and_lengths($$@) { # Count the number of different replacement strings. diff --git a/src/parallel.pod b/src/parallel.pod index e78601c5..0c8290e1 100644 --- a/src/parallel.pod +++ b/src/parallel.pod @@ -447,6 +447,27 @@ To understand positional replacement strings see B<{>IB<}>. See also: B<{=>IB<=}> B<{>IB<}> +=item B{I:I} (alpha testing) + +Format replacement string. + +Use I to format I. I is a format string used in +B. I is a replacement string. + +Examples: + + {#:%04d} - Job number ({#}) with 4 digits prepended with 0 + {%:%02d} - Job slot ({%}) with 2 digits prepended with 0 + {:%6s} - Input line ({}) right aligned 6 chars wide + {/:%12s} - Basename ({/}) right aligned 12 chars wide + {2:%8.2f} - Second input source ({2}) 8 chars wide, 2 decimals + +Format strings also works on replacement strings defined via B<--rpl> +that start with '{'. + +See also: B<{}> B<{>IB<}> B<--rpl> + + =item B<:::> I Use arguments on the command line as input source. diff --git a/src/parallel_alternatives.pod b/src/parallel_alternatives.pod index 82c51a33..fc567313 100644 --- a/src/parallel_alternatives.pod +++ b/src/parallel_alternatives.pod @@ -788,8 +788,10 @@ mixes. Combined with B output order can be the same as input order. In certain situations B will eat the last newline of standard output. -There seems to be no way to make 4 jobs run on a remote server with 4 -cores and 16 jobs on a remote server with 16 cores. +There seems to be no way to have the number og jobs depend on the +number of CPU threads in a mixed server setup: E.g run 4 jobs on a +remote server with 4 cores and 16 jobs on a remote server with 16 +cores. =head3 EXAMPLES FROM man paexec @@ -893,7 +895,7 @@ B. awk 'BEGIN {print toupper(ARGV[1])}' < tasks 13$ paexec -Z240 -x -t ssh -n 'server1 badhostname server2' \ - -c "awk 'BEGIN {print toupper(ARGV[1])}' " < tasks + -c "awk 'BEGIN {print toupper(ARGV[1])}' " < tasks 13$ parallel --filter-hosts -S 'server1,badhostname,server2' \ "awk 'BEGIN {print toupper(ARGV[1])}' " < tasks @@ -1025,7 +1027,7 @@ B. time -p paexec -c ~/bin/calc -n +2 -xg | grep -v success 20$ printf 'small1\nsmall2\nsmall3\nsmall4\nsmall5\nhuge\n' | - time -p parallel -j2 ~/bin/calc | grep -v success + time -p parallel -j2 ~/bin/calc | grep -v success 21$ printf 'small1\nsmall2\nsmall3\nsmall4\nweight: huge 6\n' | time -p paexec -c ~/bin/calc -n +2 -x -W1 | grep -v success @@ -1504,10 +1506,67 @@ user uses Rust parallel it will overwrite this file. If /tmp/parallel runs full during the run, Rust parallel does not report this, but finishes with success - thereby risking data loss. + https://github.com/mmstick/parallel (Last checked: 2016-08) +=head2 DIFFERENCES BETWEEN parallelion AND GNU Parallel + +Summary (see legend above): + +=over + +=item - (I2) - I4 - - - + +=item M1 - M3 - - M6 + +=item - O2 O3 - O5 (O6) - x x + +=item E1 - - (E4) E5 - - E8 ? + +=item - - - - - - - - - + +=item - - + +=back + +I2: I was unable to cannot get B to read from a file. + +O6: There is extra output if a job fails. + +E4: The default number of parallel jobs is the number of cpu threads. + +-- is needed to force args not be parsed as options: + + parallelion 'echo {}' -- Runs without -v + + parallelion 'echo {}' Runs with -v + +The commands are run through B shell. + +Ctrl-C does not stop processing. + +The B<--log> is similar to syslog - not a table. + +The progressbar is nice. + +B is fast: 0.1 ms/job. Similar to B. + +=head3 EXAMPLES FROM parallelion + + 1$ parallelion -progress 'echo {}' {1..1000} + + 1$ parallel --bar echo {} ::: {1..1000} + + 2$ parallelion -progress 'echo {}' $(seq 1 999) + + 2$ seq 1 999 | parallel --bar echo + +https://gitlab.redox-os.org/redox-os/parallel +(Last checked: 2024-08) + + =head2 DIFFERENCES BETWEEN Rush AND GNU Parallel B (https://github.com/shenwei356/rush) is written in Go and @@ -1972,6 +2031,8 @@ https://github.com/gdm85/coshell =head2 DIFFERENCES BETWEEN spread AND GNU Parallel +Summary (see legend above): + =over =item - - - I4 - - I7 @@ -3591,7 +3652,7 @@ not, all the long jobs may end up in the same queue, so you may see: time parallel -P4 sleep {} (7 seconds) $ printf "%b\n" 1 1 1 4 1 1 1 4 1 1 1 4 | - time ./parallel-bash.bash -p 4 -c sleep {} + time parallel-bash -p 4 -c sleep {} (12 seconds) Because it uses bash lists, the total number of jobs is limited to diff --git a/src/parsort b/src/parsort index 0f80e4ab..a4204142 100755 --- a/src/parsort +++ b/src/parsort @@ -137,7 +137,7 @@ GetOptions( "help" => \$opt::dummy, ) || exit(255); $Global::progname = ($0 =~ m:(^|/)([^/]+)$:)[1]; -$Global::version = 20240722; +$Global::version = 20240723; if($opt::version) { version(); exit 0; } # Remove -D and --parallel=N my @s = (grep { ! /^-D$|^--parallel=\S+$/ } diff --git a/src/sql b/src/sql index 3a79c25f..0b8570bd 100755 --- a/src/sql +++ b/src/sql @@ -670,7 +670,7 @@ $Global::Initfile && unlink $Global::Initfile; exit ($err); sub parse_options { - $Global::version = 20240722; + $Global::version = 20240723; $Global::progname = 'sql'; # This must be done first as this may exec myself diff --git a/testsuite/REQUIREMENTS b/testsuite/REQUIREMENTS index c7e82601..4b06048d 100644 --- a/testsuite/REQUIREMENTS +++ b/testsuite/REQUIREMENTS @@ -131,7 +131,7 @@ install_oracle_client() { echo 'export ORACLE_SID=XE' >> ~/.bashrc fi # libaio - sudo apt install libaio1t64 + sudo apt install libaio1 || sudo apt install libaio1t64 sudo ln -s libaio.so.1t64 /usr/lib/x86_64-linux-gnu/libaio.so.1 perl -MCPAN -e 'install DBD::Oracle' # TODO set up vagrant oracle server @@ -139,6 +139,7 @@ install_oracle_client() { ( cd vagrant-projects/OracleDatabase/23.4.0-Free echo export VM_ORACLE_PWD=`goodpasswd` >> ~/.passwords + echo export 'ORACLE_PWD=$VM_ORACLE_PWD' >> ~/.passwords . ~/.passwords vagrant up ) diff --git a/testsuite/tests-to-run/parallel-local-3s.sh b/testsuite/tests-to-run/parallel-local-3s.sh index af1232bd..f0255afc 100755 --- a/testsuite/tests-to-run/parallel-local-3s.sh +++ b/testsuite/tests-to-run/parallel-local-3s.sh @@ -8,7 +8,31 @@ # Each should be taking 3-10s and be possible to run in parallel # I.e.: No race conditions, no logins -par_pty() { +par_format_string() { + echo Format string + {} + parallel echo 12.346 98.765 == {:%.3f} ::: 12.34567 ::: 98.76543 + echo Format string + {.} + parallel echo 12.000 98.000 == {.:%.3f} ::: 12.34567 ::: 98.76543 + echo Format string + {2} + parallel echo 98.765 == {2:%.3f} ::: 12.34567 ::: 98.76543 + echo Format string + {2.} + parallel echo 98.000 == {2.:%.3f} ::: 12.34567 ::: 98.76543 + echo Format string + {2.}{} + parallel echo 98.00012.34567 98.76543 == {2.:%.3f}{} ::: 12.34567 ::: 98.76543 + echo Dynamic replacement strings + echo {dyn} + format + parallel --rpl '{/(\d)/(.*)} s/$$1/$$2/g;' echo 12.44 98.77 == {/3/44:%8.2f} ::: 12.34567 ::: 98.76543 + echo {Positional dyn} + format + parallel --rpl '{/(\d)/(.*)} s/$$1/$$2/g;' echo 12.44 98.77 98.765 == {/3/44:%8.2f} {2:%.3f} ::: 12.34567 ::: 98.76543 + echo {dyn__postfix} + parallel --rpl '{/(\d)/(.*)__} s/$$1/$$2/g;' echo 12.444567 98.765444 == {/3/44__} ::: 12.34567 ::: 98.76543 + echo {dyn__postfix} + format + parallel --rpl '{/(\d)/(.*)__} s/$$1/$$2/g;' echo 00012.44 == {/3/44__:%08.2f} ::: 12.34567 + echo dyn without {} + parallel --rpl '/(\d)/(.*)} s/$$1/$$2/g;' echo 12.444567 98.765444 == /3/44} ::: 12.34567 ::: 98.76543 +} + + par_pty() { parallel 'echo {} > {}' ::: 1 2 3 echo 1 > files echo 'xargs Expect: 3 1' diff --git a/testsuite/tests-to-run/parallel-local-mem.sh b/testsuite/tests-to-run/parallel-local-mem.sh index 6e37ed37..3bae61db 100755 --- a/testsuite/tests-to-run/parallel-local-mem.sh +++ b/testsuite/tests-to-run/parallel-local-mem.sh @@ -4,6 +4,8 @@ # # SPDX-License-Identifier: GPL-3.0-or-later +# On a 64G machine this uses 24G RAM + 5 min + make stopvm >/dev/null 2>/dev/null TMPDIR=${TMPDIR:-/tmp} mkdir -p $TMPDIR diff --git a/testsuite/wanted-results/parallel-local-3s b/testsuite/wanted-results/parallel-local-3s index 385840db..e03c4b5d 100644 --- a/testsuite/wanted-results/parallel-local-3s +++ b/testsuite/wanted-results/parallel-local-3s @@ -781,6 +781,27 @@ par_eta par_eta ETA: 0s Left: 0 AVG: 0.00s local:0/0/0%/0.0s  par_exitval_signal ### Test --joblog with exitval and Test --joblog with signal -- timing dependent par_exitval_signal exitval=128+6 OK par_exitval_signal signal OK +par_format_string Format string + {} +par_format_string 12.346 98.765 == 12.346 98.765 +par_format_string Format string + {.} +par_format_string 12.000 98.000 == 12.000 98.000 +par_format_string Format string + {2} +par_format_string 98.765 == 98.765 +par_format_string Format string + {2.} +par_format_string 98.000 == 98.000 +par_format_string Format string + {2.}{} +par_format_string 98.00012.34567 98.76543 == 98.00012.34567 98.76543 +par_format_string Dynamic replacement strings +par_format_string {dyn} + format +par_format_string 12.44 98.77 == 12.44 98.77 +par_format_string {Positional dyn} + format +par_format_string 12.44 98.77 98.765 == 12.44 98.77 98.765 +par_format_string {dyn__postfix} +par_format_string 12.444567 98.765444 == 12.444567 98.765444 +par_format_string {dyn__postfix} + format +par_format_string 00012.44 == 00012.44 +par_format_string dyn without {} +par_format_string 12.444567 98.765444 == 12.444567 98.765444 par_jobslot_repl bug #46232: {%} with --bar/--eta/--shuf or --halt xx% broken par_jobslot_repl 1 par_jobslot_repl 2