diff --git a/src/parallel b/src/parallel index 9da42998..5d08e683 100755 --- a/src/parallel +++ b/src/parallel @@ -38,7 +38,7 @@ use File::Basename; sub set_input_source_header($$) { my ($command_ref,$input_source_fh_ref) = @_; - if($opt::header and not $opt::pipe) { + if(defined $opt::header and not $opt::pipe) { # split with colsep or \t # $header force $colsep = \t if undef? my $delimiter = defined $opt::colsep ? $opt::colsep : "\t"; @@ -48,30 +48,53 @@ sub set_input_source_header($$) { # regexp for =} my $right = "\Q$Global::parensright\E"; my $r = $Global::parensright; - my $id = 1; - for my $fh (@$input_source_fh_ref) { - my $line = <$fh>; - chomp($line); - $line =~ s/\r$//; - ::debug("init", "Delimiter: '$delimiter'"); - for my $s (split /$delimiter/o, $line) { - ::debug("init", "Colname: '$s'"); - # Replace {colname} with {2} - for(@$command_ref, @Global::ret_files, - @Global::transfer_files, $opt::tagstring, - $opt::workdir, $opt::results, $opt::retries, - @Global::template_contents, @Global::template_names, - @opt::filter) { - # Skip if undefined - $_ or next; - s:\{$s(|/|//|\.|/\.)\}:\{$id$1\}:g; - # {=header1 ... =} => {=1 ... =} - s:$left $s (.*?) $right:$l$id$1$r:gx; + if($opt::header ne "0") { + my $id = 1; + for my $fh (@$input_source_fh_ref) { + my $line = <$fh>; + chomp($line); + $line =~ s/\r$//; + ::debug("init", "Delimiter: '$delimiter'"); + for my $s (split /$delimiter/o, $line) { + ::debug("init", "Colname: '$s'"); + # Replace {colname} with {2} + for(@$command_ref, @Global::ret_files, + @Global::transfer_files, $opt::tagstring, + $opt::workdir, $opt::results, $opt::retries, + @Global::template_contents, @Global::template_names, + @opt::filter) { + # Skip if undefined + $_ or next; + s:\{$s(|/|//|\.|/\.)\}:\{$id$1\}:g; + # {=header1 ... =} => {=1 ... =} + s:$left $s (.*?) $right:$l$id$1$r:gx; + } + $Global::input_source_header{$id} = $s; + $id++; } - $Global::input_source_header{$id} = $s; - $id++; } } + # Make it possible to do: + # parallel --header 0 echo {file2} {file1} :::: file1 file2 + my $id = 1; + for my $s (@opt::a) { + # ::: are put into files and given a filehandle + # ignore these and only keep the filenames. + fileno $s and next; + for(@$command_ref, @Global::ret_files, + @Global::transfer_files, $opt::tagstring, + $opt::workdir, $opt::results, $opt::retries, + @Global::template_contents, @Global::template_names, + @opt::filter) { + # Skip if undefined + $_ or next; + s:\{$s(|/|//|\.|/\.)\}:\{$id$1\}:g; + # {=header1 ... =} => {=1 ... =} + s:$left $s (.*?) $right:$l$id$1$r:gx; + } + $Global::input_source_header{$id} = $s; + $id++; + } } else { my $id = 1; for my $fh (@$input_source_fh_ref) { diff --git a/src/parallel.pod b/src/parallel.pod index 6b31c7a7..78e3793e 100644 --- a/src/parallel.pod +++ b/src/parallel.pod @@ -1265,7 +1265,7 @@ soon,fail=1-99% =back -=item B<--header> I +=item B<--header> I (alpha testing) Use regexp as header. @@ -1282,7 +1282,11 @@ B<--header :> is an alias for B<--header '.*\n'>. If I is a number, it is a fixed number of lines. -See also: B<--colsep> B<--pipe> B<--pipe-part> +B<--header 0> is special: It will make replacement strings for files +given with B<--arg-file> or B<::::>. It will make B<{foo/bar}> for the +file B. + +See also: B<--colsep> B<--pipe> B<--pipe-part> B<--arg-file> =item B<--hostgroups> diff --git a/testsuite/tests-to-run/parallel-local-0.3s.sh b/testsuite/tests-to-run/parallel-local-0.3s.sh index fb7fbec2..50e07807 100644 --- a/testsuite/tests-to-run/parallel-local-0.3s.sh +++ b/testsuite/tests-to-run/parallel-local-0.3s.sh @@ -63,32 +63,6 @@ par_ctagstring() { parallel --ctagstring 'I{1}\tB{2}' echo ::: 1 ::: a | wc -c } -par_plus() { - echo '### --plus' - echo '(It is OK to start with extra / or end with extra .)' - parallel -k --plus echo {} = {+/}/{/} = {.}.{+.} = {+/}/{/.}.{+.} = \ - {..}.{+..} = {+/}/{/..}.{+..} = {...}.{+...} = \ - {+/}/{/...}.{+...} \ - ::: a a.b a.b.c a.b.c.d a/1 a.b/1.2 a.b.c/1.2.3 a.b.c.d/1.2.3.4 - - echo '### Test {%...} {%%...} {#...} {##...}' - a=z.z.z.foo - echo ${a#z*z.} - parallel --plus echo {#z.*z.} ::: z.z.z.foo - echo ${a##z*z.} - parallel --plus echo {##z.*z.} ::: z.z.z.foo - - a=foo.z.z.z - echo ${a%.z.z} - parallel --plus echo {%.z.z} ::: foo.z.z.z - echo ${a%%.z*z} - parallel --plus echo {%%.z.*z} ::: foo.z.z.z - - parallel -k --plus echo {uniq} ::: A B C ::: A B C ::: A B C - parallel -k --plus echo {1uniq}+{2uniq}+{3uniq} ::: A B C ::: A B C ::: A B C - parallel -k --plus echo {choose_k} ::: A B C D ::: A B C D ::: A B C D -} - par_env_parallel_pipefail() { cat <<'EOF' | bash echo "### test env_parallel with pipefail + inherit_errexit" diff --git a/testsuite/tests-to-run/parallel-local-1s.sh b/testsuite/tests-to-run/parallel-local-1s.sh index 0bf3362a..c4e61738 100644 --- a/testsuite/tests-to-run/parallel-local-1s.sh +++ b/testsuite/tests-to-run/parallel-local-1s.sh @@ -8,6 +8,63 @@ # Each should be taking 1-3s and be possible to run in parallel # I.e.: No race conditions, no logins +par_plus() { + echo '### --plus' + echo '(It is OK to start with extra / or end with extra .)' + parallel -k --plus echo {} = {+/}/{/} = {.}.{+.} = {+/}/{/.}.{+.} = \ + {..}.{+..} = {+/}/{/..}.{+..} = {...}.{+...} = \ + {+/}/{/...}.{+...} \ + ::: a a.b a.b.c a.b.c.d a/1 a.b/1.2 a.b.c/1.2.3 a.b.c.d/1.2.3.4 + + echo '### Test {%...} {%%...} {#...} {##...}' + a=z.z.z.foo + echo ${a#z*z.} + parallel --plus echo {#z.*z.} ::: z.z.z.foo + echo ${a##z*z.} + parallel --plus echo {##z.*z.} ::: z.z.z.foo + + a=foo.z.z.z + echo ${a%.z.z} + parallel --plus echo {%.z.z} ::: foo.z.z.z + echo ${a%%.z*z} + parallel --plus echo {%%.z.*z} ::: foo.z.z.z + + parallel -k --plus echo {uniq} ::: A B C ::: A B C ::: A B C + parallel -k --plus echo {1uniq}+{2uniq}+{3uniq} ::: A B C ::: A B C ::: A B C + parallel -k --plus echo {choose_k} ::: A B C D ::: A B C D ::: A B C D +} + +par_file_rpl() { + echo '### file as replacement string' + tmp="$(mktemp)" + ( + echo contest1 + echo contest2 + echo File name "$tmp" + ) > "$tmp" + ( + # {filename} + parallel -k --header 0 echo {"$tmp"} :::: "$tmp" + # Conflict: both {filename} and {/regexp/rpl} + parallel -k --plus echo {"$tmp"} :::: "$tmp" + parallel -k --header 0 --plus echo {"$tmp"} :::: "$tmp" + tmpd="$(mktemp -d)" + cd "$tmpd" + # Conflict: both {filename} and {n} + seq 1 > 1 + seq 2 > 2 + seq 3 > 3 + parallel -k echo {1} :::: 3 2 1 + parallel -k --header 0 echo {1} :::: 3 2 1 + # Conflict: both {filename} and {=expr=} + seq 3 > =chop= + parallel -k echo {=chop=} ::: =chop= + parallel -k --header 0 echo {=chop=} ::: =chop= + rm -rf "$tmpd" + ) | perl -pe 's/tmp\.\w+/tmp.XXXXXX/g' + rm "$tmp" +} + par_commandline_with_newline() { echo 'bug #51299: --retry-failed with command with newline' echo 'The format must remain the same' diff --git a/testsuite/wanted-results/parallel-local-0.3s b/testsuite/wanted-results/parallel-local-0.3s index 6429c0d4..72eab31e 100644 --- a/testsuite/wanted-results/parallel-local-0.3s +++ b/testsuite/wanted-results/parallel-local-0.3s @@ -765,41 +765,6 @@ par_pipepart_triple_colon ### bug #62311: --pipepart + ::: fail par_pipepart_triple_colon 1 1 2 par_pipepart_triple_colon 1 1 2 par_pipepart_triple_colon 1 1 2 -par_plus ### --plus -par_plus (It is OK to start with extra / or end with extra .) -par_plus a = /a = a. = /a. = a. = /a. = a. = /a. -par_plus a.b = /a.b = a.b = /a.b = a.b. = /a.b. = a.b. = /a.b. -par_plus a.b.c = /a.b.c = a.b.c = /a.b.c = a.b.c = /a.b.c = a.b.c. = /a.b.c. -par_plus a.b.c.d = /a.b.c.d = a.b.c.d = /a.b.c.d = a.b.c.d = /a.b.c.d = a.b.c.d = /a.b.c.d -par_plus a/1 = a/1 = a/1. = a/1. = a/1. = a/1. = a/1. = a/1. -par_plus a.b/1.2 = a.b/1.2 = a.b/1.2 = a.b/1.2 = a.b/1.2. = a.b/1.2. = a.b/1.2. = a.b/1.2. -par_plus a.b.c/1.2.3 = a.b.c/1.2.3 = a.b.c/1.2.3 = a.b.c/1.2.3 = a.b.c/1.2.3 = a.b.c/1.2.3 = a.b.c/1.2.3. = a.b.c/1.2.3. -par_plus a.b.c.d/1.2.3.4 = a.b.c.d/1.2.3.4 = a.b.c.d/1.2.3.4 = a.b.c.d/1.2.3.4 = a.b.c.d/1.2.3.4 = a.b.c.d/1.2.3.4 = a.b.c.d/1.2.3.4 = a.b.c.d/1.2.3.4 -par_plus ### Test {%...} {%%...} {#...} {##...} -par_plus z.foo -par_plus z.foo -par_plus foo -par_plus foo -par_plus foo.z -par_plus foo.z -par_plus foo -par_plus foo -par_plus A B C -par_plus A C B -par_plus B A C -par_plus B C A -par_plus C A B -par_plus C B A -par_plus A+B+C -par_plus A+C+B -par_plus B+A+C -par_plus B+C+A -par_plus C+A+B -par_plus C+B+A -par_plus A B C -par_plus A B D -par_plus A C D -par_plus B C D par_profile ### Test -J profile, -J /dir/profile, -J ./profile par_profile local local par_profile abs abs diff --git a/testsuite/wanted-results/parallel-local-1s b/testsuite/wanted-results/parallel-local-1s index 968613b1..3caad3c0 100644 --- a/testsuite/wanted-results/parallel-local-1s +++ b/testsuite/wanted-results/parallel-local-1s @@ -123,6 +123,30 @@ par_empty_string_command_line ole par_empty_string_command_line bar par_eof_on_command_line_input_source ### Test of eof string on ::: par_eof_on_command_line_input_source foo +par_file_rpl ### file as replacement string +par_file_rpl contest1 +par_file_rpl contest2 +par_file_rpl File name /tmp/parallel-local-1s-tmpdir/tmp.XXXXXX +par_file_rpl contest1 +par_file_rpl contest2 +par_file_rpl File name /parallel-local-1s-tmpdir/tmp.XXXXXX/parallel-local-1s-tmpdir/tmp.XXXXXX +par_file_rpl contest1 +par_file_rpl contest2 +par_file_rpl File name /tmp/parallel-local-1s-tmpdir/tmp.XXXXXX +par_file_rpl 1 +par_file_rpl 1 +par_file_rpl 2 +par_file_rpl 2 +par_file_rpl 3 +par_file_rpl 3 +par_file_rpl 1 +par_file_rpl 1 +par_file_rpl 1 +par_file_rpl 1 +par_file_rpl 1 +par_file_rpl 1 +par_file_rpl =chop +par_file_rpl =chop par_header ### Test --header with -N par_header Start par_header h1 @@ -408,6 +432,41 @@ par_pipepart_block 17-20 par_pipepart_block 18-20 par_pipepart_block 19-20 par_pipepart_block 20-20 +par_plus ### --plus +par_plus (It is OK to start with extra / or end with extra .) +par_plus a = /a = a. = /a. = a. = /a. = a. = /a. +par_plus a.b = /a.b = a.b = /a.b = a.b. = /a.b. = a.b. = /a.b. +par_plus a.b.c = /a.b.c = a.b.c = /a.b.c = a.b.c = /a.b.c = a.b.c. = /a.b.c. +par_plus a.b.c.d = /a.b.c.d = a.b.c.d = /a.b.c.d = a.b.c.d = /a.b.c.d = a.b.c.d = /a.b.c.d +par_plus a/1 = a/1 = a/1. = a/1. = a/1. = a/1. = a/1. = a/1. +par_plus a.b/1.2 = a.b/1.2 = a.b/1.2 = a.b/1.2 = a.b/1.2. = a.b/1.2. = a.b/1.2. = a.b/1.2. +par_plus a.b.c/1.2.3 = a.b.c/1.2.3 = a.b.c/1.2.3 = a.b.c/1.2.3 = a.b.c/1.2.3 = a.b.c/1.2.3 = a.b.c/1.2.3. = a.b.c/1.2.3. +par_plus a.b.c.d/1.2.3.4 = a.b.c.d/1.2.3.4 = a.b.c.d/1.2.3.4 = a.b.c.d/1.2.3.4 = a.b.c.d/1.2.3.4 = a.b.c.d/1.2.3.4 = a.b.c.d/1.2.3.4 = a.b.c.d/1.2.3.4 +par_plus ### Test {%...} {%%...} {#...} {##...} +par_plus z.foo +par_plus z.foo +par_plus foo +par_plus foo +par_plus foo.z +par_plus foo.z +par_plus foo +par_plus foo +par_plus A B C +par_plus A C B +par_plus B A C +par_plus B C A +par_plus C A B +par_plus C B A +par_plus A+B+C +par_plus A+C+B +par_plus B+A+C +par_plus B+C+A +par_plus C+A+B +par_plus C+B+A +par_plus A B C +par_plus A B D +par_plus A C D +par_plus B C D par_profiles_with_space ### bug #42902: profiles containing arguments with space par_profiles_with_space /bin/bash=/bin/bash par_profiles_with_space echo '/bin/bash=/bin/bash'