diff --git a/src/parallel b/src/parallel index f8fe9165..a3ff4ff4 100755 --- a/src/parallel +++ b/src/parallel @@ -9487,13 +9487,6 @@ sub new { # # Replace $$1 in {= perl expr =} with groupings in short hand string # - # ppar --rpl '{rm_suffix(\S+)} s/$$1$//;' echo {rm_suffix.tar.gz} ::: UU.tar.gz - # ppar --rpl '{%(\S+)} s/$$1$//;' echo {%.tar.gz} ::: UU.tar.gz - # - # ppar --rpl '{rm_prefix(\S+)} s/^$$1//;' echo {rm_prefixUU.} ::: UU.tar.gz - # ppar --rpl '{#(\S+)} s/^$$1//;' echo {rm_prefixUU.} ::: UU.tar.gz - # - # ppar --rpl '{replace(\.\S+)/(\.\S+)} s/$$1/$$2/g;' echo {replace.tar/.gz} ::: UU.tar.gz # ppar --rpl '{/(\.\S+)/(\.\S+)} s/$$1/$$2/g;' echo {/.tar/.gz} ::: UU.tar.gz my ($prefix,$grp_regexp,$postfix) = $rpl =~ /^( [^(]* ) # Prefix - e.g. {%% @@ -9507,7 +9500,7 @@ sub new { { # The start remains the same my $unchanged = $1; - # Dummy entry to start at 2. + # Dummy entry to start at 1. my @grp = (1); # $2 = first ()-group in $grp_regexp # Put $2 in $grp[1], Put $3 in $grp[2] @@ -9518,7 +9511,7 @@ sub new { my $rv = $rplval; # replace $$1 with $_pAr_gRp1, $$2 with $_pAr_gRp2 # in the code to be executed - $rv =~ s/\$\$(\d+)/\$_pAr_gRp$1/g; + $rv =~ s/\$\$ (\d+)/\$_pAr_gRp$1/gx; # prepend with $_pAr_gRp1 = perlquote($1), my $set_args = ""; for(my $i = 1;defined $grp[$i]; $i++) { @@ -9529,13 +9522,36 @@ sub new { }gxe) { } # Do the same for the positional replacement strings - # A bit harder as we have to put in the position number $posrpl = $rpl; if($posrpl =~ s/^\{//) { # Only do this if the shorthand start with { - while(s{((^|\257>)[^\257]*?) # Don't replace after \257 unless \257> - \{(-?\d+)\Q$posrpl\E} - {$1\257<$3 $Global::rpl{$rpl}\257>}xg) { + $prefix=~s/^\{//; + while(s{( (?: ^|\257> ) [^\257]*? ) # Don't replace after \257 unless \257> + \{(-?\d+) \s* \Q$prefix\E $grp_regexp \Q$postfix\E} + { + # The start remains the same + my $unchanged = $1; + my $position = $2; + # Dummy entry to start at 1. + my @grp = (1); + # $3 = first ()-group in $grp_regexp + # Put $3 in $grp[1], Put $4 in $grp[2] + # so first ()-group in $grp_regexp is $grp[1]; + for(my $i = 3; defined $grp[$#grp]; $i++) { + push @grp, eval '$'.$i; + } + my $rv = $rplval; + # replace $$1 with $_pAr_gRp1, $$2 with $_pAr_gRp2 + # in the code to be executed + $rv =~ s/\$\$(\d+)/\$_pAr_gRp$1/gx; + # prepend with $_pAr_gRp1 = perlquote($1), + my $set_args = ""; + for(my $i = 1;defined $grp[$i]; $i++) { + $set_args .= "\$_pAr_gRp$i = \"" . + ::perl_quote_scalar($grp[$i]) . "\";"; + } + $unchanged . "\257<" . $position . $set_args . $rv . "\257>" + }gxe) { } } } diff --git a/src/parallel.pod b/src/parallel.pod index 44135993..58d04ce9 100644 --- a/src/parallel.pod +++ b/src/parallel.pod @@ -2741,7 +2741,7 @@ Put all converted in the same directory: parallel lame {} -o mydir/{/.}.mp3 -=head1 EXAMPLE: Removing two file extensions when processing files +=head1 EXAMPLE: Removing strings from the argument If you have directory with tar.gz files and want these extracted in the corresponding dir (e.g foo.tar.gz will be extracted in the dir @@ -2749,6 +2749,19 @@ foo) you can do: parallel --plus 'mkdir {..}; tar -C {..} -xf {}' ::: *.tar.gz +If you want to remove a different ending, you can use {%string}: + + parallel --plus echo {%_beta} ::: mycode_beta keep_beta_here + +You can also remove a starting string with {#string} + + parallel --plus echo {#beta_} ::: beta_mycode keep_beta_here + +To remove a string anywhere you can use regular expressions with +{/regexp/replacement} and leave the replacement empty: + + parallel --plus echo {/beta_/} ::: beta_mycode remove_beta_here + =head1 EXAMPLE: Download 24 images for each of the past 30 days @@ -3561,7 +3574,8 @@ you can run: parallel -a table_file.tsv --colsep '\t' cmd -o {2} -i {1} -Note: The default for GNU B is to remove the spaces around the columns. To keep the spaces: +Note: The default for GNU B is to remove the spaces around +the columns. To keep the spaces: parallel -a table_file.tsv --trim n --colsep '\t' cmd -o {2} -i {1} diff --git a/src/parallel_tutorial.pod b/src/parallel_tutorial.pod index dd388611..e1637326 100644 --- a/src/parallel_tutorial.pod +++ b/src/parallel_tutorial.pod @@ -576,6 +576,32 @@ replacement string, too: Output: Same as above. +If the shorthand contains matching parenthesis the replacement string +becomes a dynamic replacement string and the string in the parenthesis +can be accessed as $$1. If there are multiple matching parenthesis, +the matched strings can be accessed using $$2, $$3 and so on. + +You can think of this as giving arguments to the replacement +string. Here we give the argument B<.tar.gz> to the replacement string +B<{%I}> which removes I: + + parallel --rpl '{%(.+?)} s/$$1$//;' echo {%.tar.gz}.zip ::: foo.tar.gz + +Output: + + foo.zip + +Here we give the two arguments B and B to the replacement +string B<{/I/I}> which replaces I with +I: + + parallel --rpl '{/(.+?)/(.*?)} s/$$1/$$2/;' echo {/tar.gz/zip} ::: foo.tar.gz + +Output: + + foo.zip + + GNU B's 7 replacement strings are implemented as this: --rpl '{} ' @@ -714,6 +740,76 @@ Output: Job 4 of 5 Job 5 of 5 +=head4 Dynamic replacement strings + +B<--plus> also defines B<{:-I}>, B<{:I}>, +B<{:I:I}>, B<{#I}>, B<{%I}>, +B<{/I/I}>, B<{^I}>, B<{^^I}>, +B<{,I}>, and B<{,,I}>. They are inspired from B: + + unset myvar + echo ${myvar:-myval} + parallel --plus echo {:-myval} ::: "$myvar" + + myvar=abcAaAdef + echo ${myvar:2} + parallel --plus echo {:2} ::: "$myvar" + + echo ${myvar:2:3} + parallel --plus echo {:2:3} ::: "$myvar" + + echo ${myvar#bc} + parallel --plus echo {#bc} ::: "$myvar" + echo ${myvar#abc} + parallel --plus echo {#abc} ::: "$myvar" + + echo ${myvar%de} + parallel --plus echo {%de} ::: "$myvar" + echo ${myvar%def} + parallel --plus echo {%def} ::: "$myvar" + + echo ${myvar/def/ghi} + parallel --plus echo {/def/ghi} ::: "$myvar" + + echo ${myvar^a} + parallel --plus echo {^a} ::: "$myvar" + echo ${myvar^^a} + parallel --plus echo {^^a} ::: "$myvar" + + myvar=AbcAaAdef + echo ${myvar,A} + parallel --plus echo '{,A}' ::: "$myvar" + echo ${myvar,,A} + parallel --plus echo '{,,A}' ::: "$myvar" + +Output: + + myval + myval + cAaAdef + cAaAdef + cAa + cAa + abcAaAdef + abcAaAdef + AaAdef + AaAdef + abcAaAdef + abcAaAdef + abcAaA + abcAaA + abcAaAghi + abcAaAghi + AbcAaAdef + AbcAaAdef + AbcAAAdef + AbcAAAdef + abcAaAdef + abcAaAdef + abcaaadef + abcaaadef + + =head2 More than one argument With B<--xargs> GNU B will fit as many arguments as possible on a @@ -1857,8 +1953,8 @@ Output: in my_func baz -GNU B can copy all defined variables and functions to the -remote system. It just needs to record which ones to ignore in +GNU B can copy all user defined variables and functions to +the remote system. It just needs to record which ones to ignore in B<~/.parallel/ignored_vars>. Do that by running this once: parallel --record-env @@ -1868,8 +1964,8 @@ Output: [list of variables to ignore - including $PATH and $HOME] -Now all new variables and functions defined will be copied when using -B<--env _>: +Now all other variables and functions defined will be copied when +using B<--env _>. # The function is only copied if using Bash my_func2() { @@ -1886,6 +1982,20 @@ Output: foo in my_func2 foo bar +If you use B the variables, functions, and aliases do +not even need to be exported to be copied: + + NOT='not exported var' + alias myecho=echo + not_ex() { + myecho in not_exported_func $NOT $1 + } + env_parallel --env _ -S $SERVER1 'echo $NOT; not_ex' ::: bar + +Output: + + not exported var + in not_exported_func not exported var bar =head2 Showing what is actually run diff --git a/testsuite/tests-to-run/parallel-local-0.3s.sh b/testsuite/tests-to-run/parallel-local-0.3s.sh index c1f31ef2..c4b8cb17 100644 --- a/testsuite/tests-to-run/parallel-local-0.3s.sh +++ b/testsuite/tests-to-run/parallel-local-0.3s.sh @@ -683,62 +683,6 @@ par_tagstring_pipe() { seq 3000 | parallel -j4 --pipe -N1000 -k --tagstring {%} LANG=C wc } -par_plus_dyn_repl() { - echo "Dynamic replacement strings defined by --plus" - - unset a - echo ${a:-myval} - parallel --rpl '{:-(.+)} $_ ||= $$1' echo {:-myval} ::: "$a" - parallel --plus echo {:-myval} ::: "$a" - - a=abcAaAdef - echo ${a:2} - parallel --rpl '{:(\d+)} substr($_,0,$$1) = ""' echo {:2} ::: "$a" - parallel --plus echo {:2} ::: "$a" - - echo ${a:2:3} - parallel --rpl '{:(\d+?):(\d+?)} $_ = substr($_,$$1,$$2);' echo {:2:3} ::: "$a" - parallel --plus echo {:2:3} ::: "$a" - - echo ${#a} - parallel --rpl '{#} $_ = length $_;' echo {#} ::: "$a" - # {#} used for job number - parallel --plus echo {#} ::: "$a" - - echo ${a#bc} - parallel --rpl '{#(.+?)} s/^$$1//;' echo {#bc} ::: "$a" - parallel --plus echo {#bc} ::: "$a" - echo ${a#abc} - parallel --rpl '{#(.+?)} s/^$$1//;' echo {#abc} ::: "$a" - parallel --plus echo {#abc} ::: "$a" - - echo ${a%de} - parallel --rpl '{%(.+?)} s/$$1$//;' echo {%de} ::: "$a" - parallel --plus echo {%de} ::: "$a" - echo ${a%def} - parallel --rpl '{%(.+?)} s/$$1$//;' echo {%def} ::: "$a" - parallel --plus echo {%def} ::: "$a" - - echo ${a/def/ghi} - parallel --rpl '{/(.+?)/(.+?)} s/$$1/$$2/;' echo {/def/ghi} ::: "$a" - parallel --plus echo {/def/ghi} ::: "$a" - - echo ${a^a} - parallel --rpl '{^(.+?)} s/($$1)/uc($1)/e;' echo {^a} ::: "$a" - parallel --plus echo {^a} ::: "$a" - echo ${a^^a} - parallel --rpl '{^^(.+?)} s/($$1)/uc($1)/eg;' echo {^^a} ::: "$a" - parallel --plus echo {^^a} ::: "$a" - - a=AbcAaAdef - echo ${a,A} - parallel --rpl '{,(.+?)} s/($$1)/lc($1)/e;' echo '{,A}' ::: "$a" - parallel --plus echo '{,A}' ::: "$a" - echo ${a,,A} - parallel --rpl '{,,(.+?)} s/($$1)/lc($1)/eg;' echo '{,,A}' ::: "$a" - parallel --plus echo '{,,A}' ::: "$a" -} - export -f $(compgen -A function | grep par_) compgen -A function | grep par_ | sort | parallel -j6 --tag -k --joblog +/tmp/jl-`basename $0` '{} 2>&1' diff --git a/testsuite/tests-to-run/parallel-local-10s.sh b/testsuite/tests-to-run/parallel-local-10s.sh index 2bf6db6e..28eb95b8 100644 --- a/testsuite/tests-to-run/parallel-local-10s.sh +++ b/testsuite/tests-to-run/parallel-local-10s.sh @@ -250,6 +250,86 @@ par_pipepart_tee() { rm $tmp } +par_plus_dyn_repl() { + echo "Dynamic replacement strings defined by --plus" + + unset myvar + echo ${myvar:-myval} + parallel --rpl '{:-(.+)} $_ ||= $$1' echo {:-myval} ::: "$myvar" + parallel --plus echo {:-myval} ::: "$myvar" + parallel --plus echo {2:-myval} ::: "wrong" ::: "$myvar" ::: "wrong" + parallel --plus echo {-2:-myval} ::: "wrong" ::: "$myvar" ::: "wrong" + + myvar=abcAaAdef + echo ${myvar:2} + parallel --rpl '{:(\d+)} substr($_,0,$$1) = ""' echo {:2} ::: "$myvar" + parallel --plus echo {:2} ::: "$myvar" + parallel --plus echo {2:2} ::: "wrong" ::: "$myvar" ::: "wrong" + parallel --plus echo {-2:2} ::: "wrong" ::: "$myvar" ::: "wrong" + + echo ${myvar:2:3} + parallel --rpl '{:(\d+?):(\d+?)} $_ = substr($_,$$1,$$2);' echo {:2:3} ::: "$myvar" + parallel --plus echo {:2:3} ::: "$myvar" + parallel --plus echo {2:2:3} ::: "wrong" ::: "$myvar" ::: "wrong" + parallel --plus echo {-2:2:3} ::: "wrong" ::: "$myvar" ::: "wrong" + + echo ${#myvar} + parallel --rpl '{#} $_ = length $_;' echo {#} ::: "$myvar" + # {#} used for job number + parallel --plus echo {#} ::: "$myvar" + + echo ${myvar#bc} + parallel --rpl '{#(.+?)} s/^$$1//;' echo {#bc} ::: "$myvar" + parallel --plus echo {#bc} ::: "$myvar" + parallel --plus echo {2#bc} ::: "wrong" ::: "$myvar" ::: "wrong" + parallel --plus echo {-2#bc} ::: "wrong" ::: "$myvar" ::: "wrong" + echo ${myvar#abc} + parallel --rpl '{#(.+?)} s/^$$1//;' echo {#abc} ::: "$myvar" + parallel --plus echo {#abc} ::: "$myvar" + parallel --plus echo {2#abc} ::: "wrong" ::: "$myvar" ::: "wrong" + parallel --plus echo {-2#abc} ::: "wrong" ::: "$myvar" ::: "wrong" + + echo ${myvar%de} + parallel --rpl '{%(.+?)} s/$$1$//;' echo {%de} ::: "$myvar" + parallel --plus echo {%de} ::: "$myvar" + parallel --plus echo {2%de} ::: "wrong" ::: "$myvar" ::: "wrong" + parallel --plus echo {-2%de} ::: "wrong" ::: "$myvar" ::: "wrong" + echo ${myvar%def} + parallel --rpl '{%(.+?)} s/$$1$//;' echo {%def} ::: "$myvar" + parallel --plus echo {%def} ::: "$myvar" + parallel --plus echo {2%def} ::: "wrong" ::: "$myvar" ::: "wrong" + parallel --plus echo {-2%def} ::: "wrong" ::: "$myvar" ::: "wrong" + + echo ${myvar/def/ghi} + parallel --rpl '{/(.+?)/(.+?)} s/$$1/$$2/;' echo {/def/ghi} ::: "$myvar" + parallel --plus echo {/def/ghi} ::: "$myvar" + parallel --plus echo {2/def/ghi} ::: "wrong" ::: "$myvar" ::: "wrong" + parallel --plus echo {-2/def/ghi} ::: "wrong" ::: "$myvar" ::: "wrong" + + echo ${myvar^a} + parallel --rpl '{^(.+?)} s/^($$1)/uc($1)/e;' echo {^a} ::: "$myvar" + parallel --plus echo {^a} ::: "$myvar" + parallel --plus echo {2^a} ::: "wrong" ::: "$myvar" ::: "wrong" + parallel --plus echo {-2^a} ::: "wrong" ::: "$myvar" ::: "wrong" + echo ${myvar^^a} + parallel --rpl '{^^(.+?)} s/($$1)/uc($1)/eg;' echo {^^a} ::: "$myvar" + parallel --plus echo {^^a} ::: "$myvar" + parallel --plus echo {2^^a} ::: "wrong" ::: "$myvar" ::: "wrong" + parallel --plus echo {-2^^a} ::: "wrong" ::: "$myvar" ::: "wrong" + + myvar=AbcAaAdef + echo ${myvar,A} + parallel --rpl '{,(.+?)} s/^($$1)/lc($1)/e;' echo '{,A}' ::: "$myvar" + parallel --plus echo '{,A}' ::: "$myvar" + parallel --plus echo '{2,A}' ::: "wrong" ::: "$myvar" ::: "wrong" + parallel --plus echo '{-2,A}' ::: "wrong" ::: "$myvar" ::: "wrong" + echo ${myvar,,A} + parallel --rpl '{,,(.+?)} s/($$1)/lc($1)/eg;' echo '{,,A}' ::: "$myvar" + parallel --plus echo '{,,A}' ::: "$myvar" + parallel --plus echo '{2,,A}' ::: "wrong" ::: "$myvar" ::: "wrong" + parallel --plus echo '{-2,,A}' ::: "wrong" ::: "$myvar" ::: "wrong" +} + export -f $(compgen -A function | grep par_) compgen -A function | grep par_ | sort | parallel --joblog /tmp/jl-`basename $0` -j10 --tag -k '{} 2>&1' diff --git a/testsuite/wanted-results/parallel-local-0.3s b/testsuite/wanted-results/parallel-local-0.3s index c63c648f..833894ad 100644 --- a/testsuite/wanted-results/parallel-local-0.3s +++ b/testsuite/wanted-results/parallel-local-0.3s @@ -1623,46 +1623,6 @@ par_file_ending_in_newline gzip /tmp/parallel_f2' par_file_ending_in_newline ' par_pipepart_block_bigger_2G ### Test that --pipepart can have blocks > 2GB par_pipepart_block_bigger_2G 1 1 4 -par_plus_dyn_repl Dynamic replacement strings defined by --plus -par_plus_dyn_repl myval -par_plus_dyn_repl myval -par_plus_dyn_repl myval -par_plus_dyn_repl cAaAdef -par_plus_dyn_repl cAaAdef -par_plus_dyn_repl cAaAdef -par_plus_dyn_repl cAa -par_plus_dyn_repl cAa -par_plus_dyn_repl cAa -par_plus_dyn_repl 9 -par_plus_dyn_repl 9 -par_plus_dyn_repl 1 -par_plus_dyn_repl abcAaAdef -par_plus_dyn_repl abcAaAdef -par_plus_dyn_repl abcAaAdef -par_plus_dyn_repl AaAdef -par_plus_dyn_repl AaAdef -par_plus_dyn_repl AaAdef -par_plus_dyn_repl abcAaAdef -par_plus_dyn_repl abcAaAdef -par_plus_dyn_repl abcAaAdef -par_plus_dyn_repl abcAaA -par_plus_dyn_repl abcAaA -par_plus_dyn_repl abcAaA -par_plus_dyn_repl abcAaAghi -par_plus_dyn_repl abcAaAghi -par_plus_dyn_repl abcAaAghi -par_plus_dyn_repl AbcAaAdef -par_plus_dyn_repl AbcAaAdef -par_plus_dyn_repl AbcAaAdef -par_plus_dyn_repl AbcAAAdef -par_plus_dyn_repl AbcAAAdef -par_plus_dyn_repl AbcAAAdef -par_plus_dyn_repl abcAaAdef -par_plus_dyn_repl abcAaAdef -par_plus_dyn_repl abcAaAdef -par_plus_dyn_repl abcaaadef -par_plus_dyn_repl abcaaadef -par_plus_dyn_repl abcaaadef par_python_children ### bug #49970: Python child process dies if --env is used par_retries_replacement_string 11 par_retries_replacement_string 22 diff --git a/testsuite/wanted-results/parallel-local-10s b/testsuite/wanted-results/parallel-local-10s index 8c66594d..2b5f5847 100644 --- a/testsuite/wanted-results/parallel-local-10s +++ b/testsuite/wanted-results/parallel-local-10s @@ -500,6 +500,70 @@ par_pipepart_spawn 1:local / 8 / 999 par_pipepart_tee bug #45479: --pipe/--pipepart --tee par_pipepart_tee --pipepart --tee par_pipepart_tee 3221225472 +par_plus_dyn_repl Dynamic replacement strings defined by --plus +par_plus_dyn_repl myval +par_plus_dyn_repl myval +par_plus_dyn_repl myval +par_plus_dyn_repl myval +par_plus_dyn_repl myval +par_plus_dyn_repl cAaAdef +par_plus_dyn_repl cAaAdef +par_plus_dyn_repl cAaAdef +par_plus_dyn_repl cAaAdef +par_plus_dyn_repl cAaAdef +par_plus_dyn_repl cAa +par_plus_dyn_repl cAa +par_plus_dyn_repl cAa +par_plus_dyn_repl cAa +par_plus_dyn_repl cAa +par_plus_dyn_repl 9 +par_plus_dyn_repl 9 +par_plus_dyn_repl 1 +par_plus_dyn_repl abcAaAdef +par_plus_dyn_repl abcAaAdef +par_plus_dyn_repl abcAaAdef +par_plus_dyn_repl abcAaAdef +par_plus_dyn_repl abcAaAdef +par_plus_dyn_repl AaAdef +par_plus_dyn_repl AaAdef +par_plus_dyn_repl AaAdef +par_plus_dyn_repl AaAdef +par_plus_dyn_repl AaAdef +par_plus_dyn_repl abcAaAdef +par_plus_dyn_repl abcAaAdef +par_plus_dyn_repl abcAaAdef +par_plus_dyn_repl abcAaAdef +par_plus_dyn_repl abcAaAdef +par_plus_dyn_repl abcAaA +par_plus_dyn_repl abcAaA +par_plus_dyn_repl abcAaA +par_plus_dyn_repl abcAaA +par_plus_dyn_repl abcAaA +par_plus_dyn_repl abcAaAghi +par_plus_dyn_repl abcAaAghi +par_plus_dyn_repl abcAaAghi +par_plus_dyn_repl abcAaAghi +par_plus_dyn_repl abcAaAghi +par_plus_dyn_repl AbcAaAdef +par_plus_dyn_repl AbcAaAdef +par_plus_dyn_repl AbcAaAdef +par_plus_dyn_repl AbcAaAdef +par_plus_dyn_repl AbcAaAdef +par_plus_dyn_repl AbcAAAdef +par_plus_dyn_repl AbcAAAdef +par_plus_dyn_repl AbcAAAdef +par_plus_dyn_repl AbcAAAdef +par_plus_dyn_repl AbcAAAdef +par_plus_dyn_repl abcAaAdef +par_plus_dyn_repl abcAaAdef +par_plus_dyn_repl abcAaAdef +par_plus_dyn_repl abcAaAdef +par_plus_dyn_repl abcAaAdef +par_plus_dyn_repl abcaaadef +par_plus_dyn_repl abcaaadef +par_plus_dyn_repl abcaaadef +par_plus_dyn_repl abcaaadef +par_plus_dyn_repl abcaaadef par_print_before_halt_on_error ### What is printed before the jobs are killed par_print_before_halt_on_error -2 exit code 0 par_print_before_halt_on_error -2 0 0