tange/parallel
1
0
Fork 0
mirror of https://git.savannah.gnu.org/git/parallel.git synced 2024-10-18 12:59:39 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

parallel: Implemented --header 0 {filename} replacement string.

Browse source
This commit is contained in:
Ole Tange 2022-07-29 15:10:35 +02:00
parent 14249876fc
commit 71c11e7fd2
6 changed files with 167 additions and 85 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

25
src/parallel
View file

@ -38,7 +38,7 @@ use File::Basename;
sub set_input_source_header($$) {
my ($command_ref,$input_source_fh_ref) = @_;
if($opt::header and not $opt::pipe) {
if(defined $opt::header and not $opt::pipe) {
# split with colsep or \t
# $header force $colsep = \t if undef?
my $delimiter = defined $opt::colsep ? $opt::colsep : "\t";
@ -48,6 +48,7 @@ sub set_input_source_header($$) {
# regexp for =}
my $right = "\Q$Global::parensright\E";
my $r = $Global::parensright;
if($opt::header ne "0") {
my $id = 1;
for my $fh (@$input_source_fh_ref) {
my $line = <$fh>;
@ -72,6 +73,28 @@ sub set_input_source_header($$) {
$id++;
}
}
}
# Make it possible to do:
# parallel --header 0 echo {file2} {file1} :::: file1 file2
my $id = 1;
for my $s (@opt::a) {
# ::: are put into files and given a filehandle
# ignore these and only keep the filenames.
fileno $s and next;
for(@$command_ref, @Global::ret_files,
@Global::transfer_files, $opt::tagstring,
$opt::workdir, $opt::results, $opt::retries,
@Global::template_contents, @Global::template_names,
@opt::filter) {
# Skip if undefined
$_ or next;
s:\{$s(|/|//|\.|/\.)\}:\{$id$1\}:g;
# {=header1 ... =} => {=1 ... =}
s:$left $s (.*?) $right:$l$id$1$r:gx;
}
$Global::input_source_header{$id} = $s;
$id++;
}
} else {
my $id = 1;
for my $fh (@$input_source_fh_ref) {

8
src/parallel.pod
View file

@ -1265,7 +1265,7 @@ soon,fail=1-99%
=back
=item B<--header> I<regexp>
=item B<--header> I<regexp> (alpha testing)
Use regexp as header.
@ -1282,7 +1282,11 @@ B<--header :> is an alias for B<--header '.*\n'>.
If I<regexp> is a number, it is a fixed number of lines.
See also: B<--colsep> B<--pipe> B<--pipe-part>
B<--header 0> is special: It will make replacement strings for files
given with B<--arg-file> or B<::::>. It will make B<{foo/bar}> for the
file B<foo/bar>.
See also: B<--colsep> B<--pipe> B<--pipe-part> B<--arg-file>
=item B<--hostgroups>

26
testsuite/tests-to-run/parallel-local-0.3s.sh
View file

@ -63,32 +63,6 @@ par_ctagstring() {
parallel --ctagstring 'I{1}\tB{2}' echo ::: 1 ::: a | wc -c
}
par_plus() {
echo '### --plus'
echo '(It is OK to start with extra / or end with extra .)'
parallel -k --plus echo {} = {+/}/{/} = {.}.{+.} = {+/}/{/.}.{+.} = \
{..}.{+..} = {+/}/{/..}.{+..} = {...}.{+...} = \
{+/}/{/...}.{+...} \
::: a a.b a.b.c a.b.c.d a/1 a.b/1.2 a.b.c/1.2.3 a.b.c.d/1.2.3.4
echo '### Test {%...} {%%...} {#...} {##...}'
a=z.z.z.foo
echo ${a#z*z.}
parallel --plus echo {#z.*z.} ::: z.z.z.foo
echo ${a##z*z.}
parallel --plus echo {##z.*z.} ::: z.z.z.foo
a=foo.z.z.z
echo ${a%.z.z}
parallel --plus echo {%.z.z} ::: foo.z.z.z
echo ${a%%.z*z}
parallel --plus echo {%%.z.*z} ::: foo.z.z.z
parallel -k --plus echo {uniq} ::: A B C ::: A B C ::: A B C
parallel -k --plus echo {1uniq}+{2uniq}+{3uniq} ::: A B C ::: A B C ::: A B C
parallel -k --plus echo {choose_k} ::: A B C D ::: A B C D ::: A B C D
}
par_env_parallel_pipefail() {
cat <<'EOF' | bash
echo "### test env_parallel with pipefail + inherit_errexit"

57
testsuite/tests-to-run/parallel-local-1s.sh
View file

@ -8,6 +8,63 @@
# Each should be taking 1-3s and be possible to run in parallel
# I.e.: No race conditions, no logins
par_plus() {
echo '### --plus'
echo '(It is OK to start with extra / or end with extra .)'
parallel -k --plus echo {} = {+/}/{/} = {.}.{+.} = {+/}/{/.}.{+.} = \
{..}.{+..} = {+/}/{/..}.{+..} = {...}.{+...} = \
{+/}/{/...}.{+...} \
::: a a.b a.b.c a.b.c.d a/1 a.b/1.2 a.b.c/1.2.3 a.b.c.d/1.2.3.4
echo '### Test {%...} {%%...} {#...} {##...}'
a=z.z.z.foo
echo ${a#z*z.}
parallel --plus echo {#z.*z.} ::: z.z.z.foo
echo ${a##z*z.}
parallel --plus echo {##z.*z.} ::: z.z.z.foo
a=foo.z.z.z
echo ${a%.z.z}
parallel --plus echo {%.z.z} ::: foo.z.z.z
echo ${a%%.z*z}
parallel --plus echo {%%.z.*z} ::: foo.z.z.z
parallel -k --plus echo {uniq} ::: A B C ::: A B C ::: A B C
parallel -k --plus echo {1uniq}+{2uniq}+{3uniq} ::: A B C ::: A B C ::: A B C
parallel -k --plus echo {choose_k} ::: A B C D ::: A B C D ::: A B C D
}
par_file_rpl() {
echo '### file as replacement string'
tmp="$(mktemp)"
(
echo contest1
echo contest2
echo File name "$tmp"
) > "$tmp"
(
# {filename}
parallel -k --header 0 echo {"$tmp"} :::: "$tmp"
# Conflict: both {filename} and {/regexp/rpl}
parallel -k --plus echo {"$tmp"} :::: "$tmp"
parallel -k --header 0 --plus echo {"$tmp"} :::: "$tmp"
tmpd="$(mktemp -d)"
cd "$tmpd"
# Conflict: both {filename} and {n}
seq 1 > 1
seq 2 > 2
seq 3 > 3
parallel -k echo {1} :::: 3 2 1
parallel -k --header 0 echo {1} :::: 3 2 1
# Conflict: both {filename} and {=expr=}
seq 3 > =chop=
parallel -k echo {=chop=} ::: =chop=
parallel -k --header 0 echo {=chop=} ::: =chop=
rm -rf "$tmpd"
) | perl -pe 's/tmp\.\w+/tmp.XXXXXX/g'
rm "$tmp"
}
par_commandline_with_newline() {
echo 'bug #51299: --retry-failed with command with newline'
echo 'The format must remain the same'

35
testsuite/wanted-results/parallel-local-0.3s
View file

@ -765,41 +765,6 @@ par_pipepart_triple_colon ### bug #62311: --pipepart + ::: fail
par_pipepart_triple_colon 1 1 2
par_pipepart_triple_colon 1 1 2
par_pipepart_triple_colon 1 1 2
par_plus ### --plus
par_plus (It is OK to start with extra / or end with extra .)
par_plus a = /a = a. = /a. = a. = /a. = a. = /a.
par_plus a.b = /a.b = a.b = /a.b = a.b. = /a.b. = a.b. = /a.b.
par_plus a.b.c = /a.b.c = a.b.c = /a.b.c = a.b.c = /a.b.c = a.b.c. = /a.b.c.
par_plus a.b.c.d = /a.b.c.d = a.b.c.d = /a.b.c.d = a.b.c.d = /a.b.c.d = a.b.c.d = /a.b.c.d
par_plus a/1 = a/1 = a/1. = a/1. = a/1. = a/1. = a/1. = a/1.
par_plus a.b/1.2 = a.b/1.2 = a.b/1.2 = a.b/1.2 = a.b/1.2. = a.b/1.2. = a.b/1.2. = a.b/1.2.
par_plus a.b.c/1.2.3 = a.b.c/1.2.3 = a.b.c/1.2.3 = a.b.c/1.2.3 = a.b.c/1.2.3 = a.b.c/1.2.3 = a.b.c/1.2.3. = a.b.c/1.2.3.
par_plus a.b.c.d/1.2.3.4 = a.b.c.d/1.2.3.4 = a.b.c.d/1.2.3.4 = a.b.c.d/1.2.3.4 = a.b.c.d/1.2.3.4 = a.b.c.d/1.2.3.4 = a.b.c.d/1.2.3.4 = a.b.c.d/1.2.3.4
par_plus ### Test {%...} {%%...} {#...} {##...}
par_plus z.foo
par_plus z.foo
par_plus foo
par_plus foo
par_plus foo.z
par_plus foo.z
par_plus foo
par_plus foo
par_plus A B C
par_plus A C B
par_plus B A C
par_plus B C A
par_plus C A B
par_plus C B A
par_plus A+B+C
par_plus A+C+B
par_plus B+A+C
par_plus B+C+A
par_plus C+A+B
par_plus C+B+A
par_plus A B C
par_plus A B D
par_plus A C D
par_plus B C D
par_profile ### Test -J profile, -J /dir/profile, -J ./profile
par_profile local local
par_profile abs abs

59
testsuite/wanted-results/parallel-local-1s
View file

@ -123,6 +123,30 @@ par_empty_string_command_line ole
par_empty_string_command_line bar
par_eof_on_command_line_input_source ### Test of eof string on :::
par_eof_on_command_line_input_source foo
par_file_rpl ### file as replacement string
par_file_rpl contest1
par_file_rpl contest2
par_file_rpl File name /tmp/parallel-local-1s-tmpdir/tmp.XXXXXX
par_file_rpl contest1
par_file_rpl contest2
par_file_rpl File name /parallel-local-1s-tmpdir/tmp.XXXXXX/parallel-local-1s-tmpdir/tmp.XXXXXX
par_file_rpl contest1
par_file_rpl contest2
par_file_rpl File name /tmp/parallel-local-1s-tmpdir/tmp.XXXXXX
par_file_rpl 1
par_file_rpl 1
par_file_rpl 2
par_file_rpl 2
par_file_rpl 3
par_file_rpl 3
par_file_rpl 1
par_file_rpl 1
par_file_rpl 1
par_file_rpl 1
par_file_rpl 1
par_file_rpl 1
par_file_rpl =chop
par_file_rpl =chop
par_header ### Test --header with -N
par_header Start
par_header h1
@ -408,6 +432,41 @@ par_pipepart_block 17-20
par_pipepart_block 18-20
par_pipepart_block 19-20
par_pipepart_block 20-20
par_plus ### --plus
par_plus (It is OK to start with extra / or end with extra .)
par_plus a = /a = a. = /a. = a. = /a. = a. = /a.
par_plus a.b = /a.b = a.b = /a.b = a.b. = /a.b. = a.b. = /a.b.
par_plus a.b.c = /a.b.c = a.b.c = /a.b.c = a.b.c = /a.b.c = a.b.c. = /a.b.c.
par_plus a.b.c.d = /a.b.c.d = a.b.c.d = /a.b.c.d = a.b.c.d = /a.b.c.d = a.b.c.d = /a.b.c.d
par_plus a/1 = a/1 = a/1. = a/1. = a/1. = a/1. = a/1. = a/1.
par_plus a.b/1.2 = a.b/1.2 = a.b/1.2 = a.b/1.2 = a.b/1.2. = a.b/1.2. = a.b/1.2. = a.b/1.2.
par_plus a.b.c/1.2.3 = a.b.c/1.2.3 = a.b.c/1.2.3 = a.b.c/1.2.3 = a.b.c/1.2.3 = a.b.c/1.2.3 = a.b.c/1.2.3. = a.b.c/1.2.3.
par_plus a.b.c.d/1.2.3.4 = a.b.c.d/1.2.3.4 = a.b.c.d/1.2.3.4 = a.b.c.d/1.2.3.4 = a.b.c.d/1.2.3.4 = a.b.c.d/1.2.3.4 = a.b.c.d/1.2.3.4 = a.b.c.d/1.2.3.4
par_plus ### Test {%...} {%%...} {#...} {##...}
par_plus z.foo
par_plus z.foo
par_plus foo
par_plus foo
par_plus foo.z
par_plus foo.z
par_plus foo
par_plus foo
par_plus A B C
par_plus A C B
par_plus B A C
par_plus B C A
par_plus C A B
par_plus C B A
par_plus A+B+C
par_plus A+C+B
par_plus B+A+C
par_plus B+C+A
par_plus C+A+B
par_plus C+B+A
par_plus A B C
par_plus A B D
par_plus A C D
par_plus B C D
par_profiles_with_space ### bug #42902: profiles containing arguments with space
par_profiles_with_space /bin/bash=/bin/bash
par_profiles_with_space echo '/bin/bash=/bin/bash'