Fixed bug #50655: Positional dynamic replacement strings do not work.

This commit is contained in:
Ole Tange 2017-03-28 21:30:06 +02:00
parent 86615dac7e
commit e135fb6fd3
7 changed files with 303 additions and 115 deletions

View file

@ -9487,13 +9487,6 @@ sub new {
#
# Replace $$1 in {= perl expr =} with groupings in short hand string
#
# ppar --rpl '{rm_suffix(\S+)} s/$$1$//;' echo {rm_suffix.tar.gz} ::: UU.tar.gz
# ppar --rpl '{%(\S+)} s/$$1$//;' echo {%.tar.gz} ::: UU.tar.gz
#
# ppar --rpl '{rm_prefix(\S+)} s/^$$1//;' echo {rm_prefixUU.} ::: UU.tar.gz
# ppar --rpl '{#(\S+)} s/^$$1//;' echo {rm_prefixUU.} ::: UU.tar.gz
#
# ppar --rpl '{replace(\.\S+)/(\.\S+)} s/$$1/$$2/g;' echo {replace.tar/.gz} ::: UU.tar.gz
# ppar --rpl '{/(\.\S+)/(\.\S+)} s/$$1/$$2/g;' echo {/.tar/.gz} ::: UU.tar.gz
my ($prefix,$grp_regexp,$postfix) =
$rpl =~ /^( [^(]* ) # Prefix - e.g. {%%
@ -9507,7 +9500,7 @@ sub new {
{
# The start remains the same
my $unchanged = $1;
# Dummy entry to start at 2.
# Dummy entry to start at 1.
my @grp = (1);
# $2 = first ()-group in $grp_regexp
# Put $2 in $grp[1], Put $3 in $grp[2]
@ -9518,7 +9511,7 @@ sub new {
my $rv = $rplval;
# replace $$1 with $_pAr_gRp1, $$2 with $_pAr_gRp2
# in the code to be executed
$rv =~ s/\$\$(\d+)/\$_pAr_gRp$1/g;
$rv =~ s/\$\$ (\d+)/\$_pAr_gRp$1/gx;
# prepend with $_pAr_gRp1 = perlquote($1),
my $set_args = "";
for(my $i = 1;defined $grp[$i]; $i++) {
@ -9529,13 +9522,36 @@ sub new {
}gxe) {
}
# Do the same for the positional replacement strings
# A bit harder as we have to put in the position number
$posrpl = $rpl;
if($posrpl =~ s/^\{//) {
# Only do this if the shorthand start with {
while(s{((^|\257>)[^\257]*?) # Don't replace after \257 unless \257>
\{(-?\d+)\Q$posrpl\E}
{$1\257<$3 $Global::rpl{$rpl}\257>}xg) {
$prefix=~s/^\{//;
while(s{( (?: ^|\257> ) [^\257]*? ) # Don't replace after \257 unless \257>
\{(-?\d+) \s* \Q$prefix\E $grp_regexp \Q$postfix\E}
{
# The start remains the same
my $unchanged = $1;
my $position = $2;
# Dummy entry to start at 1.
my @grp = (1);
# $3 = first ()-group in $grp_regexp
# Put $3 in $grp[1], Put $4 in $grp[2]
# so first ()-group in $grp_regexp is $grp[1];
for(my $i = 3; defined $grp[$#grp]; $i++) {
push @grp, eval '$'.$i;
}
my $rv = $rplval;
# replace $$1 with $_pAr_gRp1, $$2 with $_pAr_gRp2
# in the code to be executed
$rv =~ s/\$\$(\d+)/\$_pAr_gRp$1/gx;
# prepend with $_pAr_gRp1 = perlquote($1),
my $set_args = "";
for(my $i = 1;defined $grp[$i]; $i++) {
$set_args .= "\$_pAr_gRp$i = \"" .
::perl_quote_scalar($grp[$i]) . "\";";
}
$unchanged . "\257<" . $position . $set_args . $rv . "\257>"
}gxe) {
}
}
}

View file

@ -2741,7 +2741,7 @@ Put all converted in the same directory:
parallel lame {} -o mydir/{/.}.mp3
=head1 EXAMPLE: Removing two file extensions when processing files
=head1 EXAMPLE: Removing strings from the argument
If you have directory with tar.gz files and want these extracted in
the corresponding dir (e.g foo.tar.gz will be extracted in the dir
@ -2749,6 +2749,19 @@ foo) you can do:
parallel --plus 'mkdir {..}; tar -C {..} -xf {}' ::: *.tar.gz
If you want to remove a different ending, you can use {%string}:
parallel --plus echo {%_beta} ::: mycode_beta keep_beta_here
You can also remove a starting string with {#string}
parallel --plus echo {#beta_} ::: beta_mycode keep_beta_here
To remove a string anywhere you can use regular expressions with
{/regexp/replacement} and leave the replacement empty:
parallel --plus echo {/beta_/} ::: beta_mycode remove_beta_here
=head1 EXAMPLE: Download 24 images for each of the past 30 days
@ -3561,7 +3574,8 @@ you can run:
parallel -a table_file.tsv --colsep '\t' cmd -o {2} -i {1}
Note: The default for GNU B<parallel> is to remove the spaces around the columns. To keep the spaces:
Note: The default for GNU B<parallel> is to remove the spaces around
the columns. To keep the spaces:
parallel -a table_file.tsv --trim n --colsep '\t' cmd -o {2} -i {1}

View file

@ -576,6 +576,32 @@ replacement string, too:
Output: Same as above.
If the shorthand contains matching parenthesis the replacement string
becomes a dynamic replacement string and the string in the parenthesis
can be accessed as $$1. If there are multiple matching parenthesis,
the matched strings can be accessed using $$2, $$3 and so on.
You can think of this as giving arguments to the replacement
string. Here we give the argument B<.tar.gz> to the replacement string
B<{%I<string>}> which removes I<string>:
parallel --rpl '{%(.+?)} s/$$1$//;' echo {%.tar.gz}.zip ::: foo.tar.gz
Output:
foo.zip
Here we give the two arguments B<tar.gz> and B<zip> to the replacement
string B<{/I<string1>/I<string2>}> which replaces I<string1> with
I<string2>:
parallel --rpl '{/(.+?)/(.*?)} s/$$1/$$2/;' echo {/tar.gz/zip} ::: foo.tar.gz
Output:
foo.zip
GNU B<parallel>'s 7 replacement strings are implemented as this:
--rpl '{} '
@ -714,6 +740,76 @@ Output:
Job 4 of 5
Job 5 of 5
=head4 Dynamic replacement strings
B<--plus> also defines B<{:-I<string>}>, B<{:I<number>}>,
B<{:I<number1>:I<number2>}>, B<{#I<string>}>, B<{%I<string>}>,
B<{/I<string1>/I<string2>}>, B<{^I<string>}>, B<{^^I<string>}>,
B<{,I<string>}>, and B<{,,I<string>}>. They are inspired from B<Bash>:
unset myvar
echo ${myvar:-myval}
parallel --plus echo {:-myval} ::: "$myvar"
myvar=abcAaAdef
echo ${myvar:2}
parallel --plus echo {:2} ::: "$myvar"
echo ${myvar:2:3}
parallel --plus echo {:2:3} ::: "$myvar"
echo ${myvar#bc}
parallel --plus echo {#bc} ::: "$myvar"
echo ${myvar#abc}
parallel --plus echo {#abc} ::: "$myvar"
echo ${myvar%de}
parallel --plus echo {%de} ::: "$myvar"
echo ${myvar%def}
parallel --plus echo {%def} ::: "$myvar"
echo ${myvar/def/ghi}
parallel --plus echo {/def/ghi} ::: "$myvar"
echo ${myvar^a}
parallel --plus echo {^a} ::: "$myvar"
echo ${myvar^^a}
parallel --plus echo {^^a} ::: "$myvar"
myvar=AbcAaAdef
echo ${myvar,A}
parallel --plus echo '{,A}' ::: "$myvar"
echo ${myvar,,A}
parallel --plus echo '{,,A}' ::: "$myvar"
Output:
myval
myval
cAaAdef
cAaAdef
cAa
cAa
abcAaAdef
abcAaAdef
AaAdef
AaAdef
abcAaAdef
abcAaAdef
abcAaA
abcAaA
abcAaAghi
abcAaAghi
AbcAaAdef
AbcAaAdef
AbcAAAdef
AbcAAAdef
abcAaAdef
abcAaAdef
abcaaadef
abcaaadef
=head2 More than one argument
With B<--xargs> GNU B<parallel> will fit as many arguments as possible on a
@ -1857,8 +1953,8 @@ Output:
in my_func baz
GNU B<parallel> can copy all defined variables and functions to the
remote system. It just needs to record which ones to ignore in
GNU B<parallel> can copy all user defined variables and functions to
the remote system. It just needs to record which ones to ignore in
B<~/.parallel/ignored_vars>. Do that by running this once:
parallel --record-env
@ -1868,8 +1964,8 @@ Output:
[list of variables to ignore - including $PATH and $HOME]
Now all new variables and functions defined will be copied when using
B<--env _>:
Now all other variables and functions defined will be copied when
using B<--env _>.
# The function is only copied if using Bash
my_func2() {
@ -1886,6 +1982,20 @@ Output:
foo
in my_func2 foo bar
If you use B<env_parallel> the variables, functions, and aliases do
not even need to be exported to be copied:
NOT='not exported var'
alias myecho=echo
not_ex() {
myecho in not_exported_func $NOT $1
}
env_parallel --env _ -S $SERVER1 'echo $NOT; not_ex' ::: bar
Output:
not exported var
in not_exported_func not exported var bar
=head2 Showing what is actually run

View file

@ -683,62 +683,6 @@ par_tagstring_pipe() {
seq 3000 | parallel -j4 --pipe -N1000 -k --tagstring {%} LANG=C wc
}
par_plus_dyn_repl() {
echo "Dynamic replacement strings defined by --plus"
unset a
echo ${a:-myval}
parallel --rpl '{:-(.+)} $_ ||= $$1' echo {:-myval} ::: "$a"
parallel --plus echo {:-myval} ::: "$a"
a=abcAaAdef
echo ${a:2}
parallel --rpl '{:(\d+)} substr($_,0,$$1) = ""' echo {:2} ::: "$a"
parallel --plus echo {:2} ::: "$a"
echo ${a:2:3}
parallel --rpl '{:(\d+?):(\d+?)} $_ = substr($_,$$1,$$2);' echo {:2:3} ::: "$a"
parallel --plus echo {:2:3} ::: "$a"
echo ${#a}
parallel --rpl '{#} $_ = length $_;' echo {#} ::: "$a"
# {#} used for job number
parallel --plus echo {#} ::: "$a"
echo ${a#bc}
parallel --rpl '{#(.+?)} s/^$$1//;' echo {#bc} ::: "$a"
parallel --plus echo {#bc} ::: "$a"
echo ${a#abc}
parallel --rpl '{#(.+?)} s/^$$1//;' echo {#abc} ::: "$a"
parallel --plus echo {#abc} ::: "$a"
echo ${a%de}
parallel --rpl '{%(.+?)} s/$$1$//;' echo {%de} ::: "$a"
parallel --plus echo {%de} ::: "$a"
echo ${a%def}
parallel --rpl '{%(.+?)} s/$$1$//;' echo {%def} ::: "$a"
parallel --plus echo {%def} ::: "$a"
echo ${a/def/ghi}
parallel --rpl '{/(.+?)/(.+?)} s/$$1/$$2/;' echo {/def/ghi} ::: "$a"
parallel --plus echo {/def/ghi} ::: "$a"
echo ${a^a}
parallel --rpl '{^(.+?)} s/($$1)/uc($1)/e;' echo {^a} ::: "$a"
parallel --plus echo {^a} ::: "$a"
echo ${a^^a}
parallel --rpl '{^^(.+?)} s/($$1)/uc($1)/eg;' echo {^^a} ::: "$a"
parallel --plus echo {^^a} ::: "$a"
a=AbcAaAdef
echo ${a,A}
parallel --rpl '{,(.+?)} s/($$1)/lc($1)/e;' echo '{,A}' ::: "$a"
parallel --plus echo '{,A}' ::: "$a"
echo ${a,,A}
parallel --rpl '{,,(.+?)} s/($$1)/lc($1)/eg;' echo '{,,A}' ::: "$a"
parallel --plus echo '{,,A}' ::: "$a"
}
export -f $(compgen -A function | grep par_)
compgen -A function | grep par_ | sort |
parallel -j6 --tag -k --joblog +/tmp/jl-`basename $0` '{} 2>&1'

View file

@ -250,6 +250,86 @@ par_pipepart_tee() {
rm $tmp
}
par_plus_dyn_repl() {
echo "Dynamic replacement strings defined by --plus"
unset myvar
echo ${myvar:-myval}
parallel --rpl '{:-(.+)} $_ ||= $$1' echo {:-myval} ::: "$myvar"
parallel --plus echo {:-myval} ::: "$myvar"
parallel --plus echo {2:-myval} ::: "wrong" ::: "$myvar" ::: "wrong"
parallel --plus echo {-2:-myval} ::: "wrong" ::: "$myvar" ::: "wrong"
myvar=abcAaAdef
echo ${myvar:2}
parallel --rpl '{:(\d+)} substr($_,0,$$1) = ""' echo {:2} ::: "$myvar"
parallel --plus echo {:2} ::: "$myvar"
parallel --plus echo {2:2} ::: "wrong" ::: "$myvar" ::: "wrong"
parallel --plus echo {-2:2} ::: "wrong" ::: "$myvar" ::: "wrong"
echo ${myvar:2:3}
parallel --rpl '{:(\d+?):(\d+?)} $_ = substr($_,$$1,$$2);' echo {:2:3} ::: "$myvar"
parallel --plus echo {:2:3} ::: "$myvar"
parallel --plus echo {2:2:3} ::: "wrong" ::: "$myvar" ::: "wrong"
parallel --plus echo {-2:2:3} ::: "wrong" ::: "$myvar" ::: "wrong"
echo ${#myvar}
parallel --rpl '{#} $_ = length $_;' echo {#} ::: "$myvar"
# {#} used for job number
parallel --plus echo {#} ::: "$myvar"
echo ${myvar#bc}
parallel --rpl '{#(.+?)} s/^$$1//;' echo {#bc} ::: "$myvar"
parallel --plus echo {#bc} ::: "$myvar"
parallel --plus echo {2#bc} ::: "wrong" ::: "$myvar" ::: "wrong"
parallel --plus echo {-2#bc} ::: "wrong" ::: "$myvar" ::: "wrong"
echo ${myvar#abc}
parallel --rpl '{#(.+?)} s/^$$1//;' echo {#abc} ::: "$myvar"
parallel --plus echo {#abc} ::: "$myvar"
parallel --plus echo {2#abc} ::: "wrong" ::: "$myvar" ::: "wrong"
parallel --plus echo {-2#abc} ::: "wrong" ::: "$myvar" ::: "wrong"
echo ${myvar%de}
parallel --rpl '{%(.+?)} s/$$1$//;' echo {%de} ::: "$myvar"
parallel --plus echo {%de} ::: "$myvar"
parallel --plus echo {2%de} ::: "wrong" ::: "$myvar" ::: "wrong"
parallel --plus echo {-2%de} ::: "wrong" ::: "$myvar" ::: "wrong"
echo ${myvar%def}
parallel --rpl '{%(.+?)} s/$$1$//;' echo {%def} ::: "$myvar"
parallel --plus echo {%def} ::: "$myvar"
parallel --plus echo {2%def} ::: "wrong" ::: "$myvar" ::: "wrong"
parallel --plus echo {-2%def} ::: "wrong" ::: "$myvar" ::: "wrong"
echo ${myvar/def/ghi}
parallel --rpl '{/(.+?)/(.+?)} s/$$1/$$2/;' echo {/def/ghi} ::: "$myvar"
parallel --plus echo {/def/ghi} ::: "$myvar"
parallel --plus echo {2/def/ghi} ::: "wrong" ::: "$myvar" ::: "wrong"
parallel --plus echo {-2/def/ghi} ::: "wrong" ::: "$myvar" ::: "wrong"
echo ${myvar^a}
parallel --rpl '{^(.+?)} s/^($$1)/uc($1)/e;' echo {^a} ::: "$myvar"
parallel --plus echo {^a} ::: "$myvar"
parallel --plus echo {2^a} ::: "wrong" ::: "$myvar" ::: "wrong"
parallel --plus echo {-2^a} ::: "wrong" ::: "$myvar" ::: "wrong"
echo ${myvar^^a}
parallel --rpl '{^^(.+?)} s/($$1)/uc($1)/eg;' echo {^^a} ::: "$myvar"
parallel --plus echo {^^a} ::: "$myvar"
parallel --plus echo {2^^a} ::: "wrong" ::: "$myvar" ::: "wrong"
parallel --plus echo {-2^^a} ::: "wrong" ::: "$myvar" ::: "wrong"
myvar=AbcAaAdef
echo ${myvar,A}
parallel --rpl '{,(.+?)} s/^($$1)/lc($1)/e;' echo '{,A}' ::: "$myvar"
parallel --plus echo '{,A}' ::: "$myvar"
parallel --plus echo '{2,A}' ::: "wrong" ::: "$myvar" ::: "wrong"
parallel --plus echo '{-2,A}' ::: "wrong" ::: "$myvar" ::: "wrong"
echo ${myvar,,A}
parallel --rpl '{,,(.+?)} s/($$1)/lc($1)/eg;' echo '{,,A}' ::: "$myvar"
parallel --plus echo '{,,A}' ::: "$myvar"
parallel --plus echo '{2,,A}' ::: "wrong" ::: "$myvar" ::: "wrong"
parallel --plus echo '{-2,,A}' ::: "wrong" ::: "$myvar" ::: "wrong"
}
export -f $(compgen -A function | grep par_)
compgen -A function | grep par_ | sort |
parallel --joblog /tmp/jl-`basename $0` -j10 --tag -k '{} 2>&1'

View file

@ -1623,46 +1623,6 @@ par_file_ending_in_newline gzip /tmp/parallel_f2'
par_file_ending_in_newline '
par_pipepart_block_bigger_2G ### Test that --pipepart can have blocks > 2GB
par_pipepart_block_bigger_2G 1 1 4
par_plus_dyn_repl Dynamic replacement strings defined by --plus
par_plus_dyn_repl myval
par_plus_dyn_repl myval
par_plus_dyn_repl myval
par_plus_dyn_repl cAaAdef
par_plus_dyn_repl cAaAdef
par_plus_dyn_repl cAaAdef
par_plus_dyn_repl cAa
par_plus_dyn_repl cAa
par_plus_dyn_repl cAa
par_plus_dyn_repl 9
par_plus_dyn_repl 9
par_plus_dyn_repl 1
par_plus_dyn_repl abcAaAdef
par_plus_dyn_repl abcAaAdef
par_plus_dyn_repl abcAaAdef
par_plus_dyn_repl AaAdef
par_plus_dyn_repl AaAdef
par_plus_dyn_repl AaAdef
par_plus_dyn_repl abcAaAdef
par_plus_dyn_repl abcAaAdef
par_plus_dyn_repl abcAaAdef
par_plus_dyn_repl abcAaA
par_plus_dyn_repl abcAaA
par_plus_dyn_repl abcAaA
par_plus_dyn_repl abcAaAghi
par_plus_dyn_repl abcAaAghi
par_plus_dyn_repl abcAaAghi
par_plus_dyn_repl AbcAaAdef
par_plus_dyn_repl AbcAaAdef
par_plus_dyn_repl AbcAaAdef
par_plus_dyn_repl AbcAAAdef
par_plus_dyn_repl AbcAAAdef
par_plus_dyn_repl AbcAAAdef
par_plus_dyn_repl abcAaAdef
par_plus_dyn_repl abcAaAdef
par_plus_dyn_repl abcAaAdef
par_plus_dyn_repl abcaaadef
par_plus_dyn_repl abcaaadef
par_plus_dyn_repl abcaaadef
par_python_children ### bug #49970: Python child process dies if --env is used
par_retries_replacement_string 11
par_retries_replacement_string 22

View file

@ -500,6 +500,70 @@ par_pipepart_spawn 1:local / 8 / 999
par_pipepart_tee bug #45479: --pipe/--pipepart --tee
par_pipepart_tee --pipepart --tee
par_pipepart_tee 3221225472
par_plus_dyn_repl Dynamic replacement strings defined by --plus
par_plus_dyn_repl myval
par_plus_dyn_repl myval
par_plus_dyn_repl myval
par_plus_dyn_repl myval
par_plus_dyn_repl myval
par_plus_dyn_repl cAaAdef
par_plus_dyn_repl cAaAdef
par_plus_dyn_repl cAaAdef
par_plus_dyn_repl cAaAdef
par_plus_dyn_repl cAaAdef
par_plus_dyn_repl cAa
par_plus_dyn_repl cAa
par_plus_dyn_repl cAa
par_plus_dyn_repl cAa
par_plus_dyn_repl cAa
par_plus_dyn_repl 9
par_plus_dyn_repl 9
par_plus_dyn_repl 1
par_plus_dyn_repl abcAaAdef
par_plus_dyn_repl abcAaAdef
par_plus_dyn_repl abcAaAdef
par_plus_dyn_repl abcAaAdef
par_plus_dyn_repl abcAaAdef
par_plus_dyn_repl AaAdef
par_plus_dyn_repl AaAdef
par_plus_dyn_repl AaAdef
par_plus_dyn_repl AaAdef
par_plus_dyn_repl AaAdef
par_plus_dyn_repl abcAaAdef
par_plus_dyn_repl abcAaAdef
par_plus_dyn_repl abcAaAdef
par_plus_dyn_repl abcAaAdef
par_plus_dyn_repl abcAaAdef
par_plus_dyn_repl abcAaA
par_plus_dyn_repl abcAaA
par_plus_dyn_repl abcAaA
par_plus_dyn_repl abcAaA
par_plus_dyn_repl abcAaA
par_plus_dyn_repl abcAaAghi
par_plus_dyn_repl abcAaAghi
par_plus_dyn_repl abcAaAghi
par_plus_dyn_repl abcAaAghi
par_plus_dyn_repl abcAaAghi
par_plus_dyn_repl AbcAaAdef
par_plus_dyn_repl AbcAaAdef
par_plus_dyn_repl AbcAaAdef
par_plus_dyn_repl AbcAaAdef
par_plus_dyn_repl AbcAaAdef
par_plus_dyn_repl AbcAAAdef
par_plus_dyn_repl AbcAAAdef
par_plus_dyn_repl AbcAAAdef
par_plus_dyn_repl AbcAAAdef
par_plus_dyn_repl AbcAAAdef
par_plus_dyn_repl abcAaAdef
par_plus_dyn_repl abcAaAdef
par_plus_dyn_repl abcAaAdef
par_plus_dyn_repl abcAaAdef
par_plus_dyn_repl abcAaAdef
par_plus_dyn_repl abcaaadef
par_plus_dyn_repl abcaaadef
par_plus_dyn_repl abcaaadef
par_plus_dyn_repl abcaaadef
par_plus_dyn_repl abcaaadef
par_print_before_halt_on_error ### What is printed before the jobs are killed
par_print_before_halt_on_error -2 exit code 0
par_print_before_halt_on_error -2 0 0