mirror of
https://git.savannah.gnu.org/git/parallel.git
synced 2024-11-22 05:57:54 +00:00
parallel: Bug fix: If replace string contains regexp special char it should be escaped.
Most xargs options implemented with unittest - though still undocumented
This commit is contained in:
Ole Tange
parent
f5cca23204
commit
0214645ed9
76
parallel
76
parallel
|
|
@ -133,7 +133,8 @@ output from different commands to be mixed. Can be reversed with B<-g>.
|
|||
|
||||
=item B<-v>
|
||||
|
||||
Verbose. Print the job to be run on STDOUT. Can be reversed with B<-s>.
|
||||
Verbose. Print the job to be run on STDOUT. Can be reversed with
|
||||
B<--silent>.
|
||||
|
||||
=item B<--xargs>
|
||||
=item B<-m>
|
||||
|
|
@ -512,12 +513,12 @@ parallel --wait /tmp/comfile will wait until no more locks on the file
|
|||
|
||||
Copyright (C) 2007-10-18 Ole Tange, http://ole.tange.dk
|
||||
|
||||
Copyright (C) 2008-2009 Ole Tange, http://ole.tange.dk
|
||||
Copyright (C) 2008-2010 Ole Tange, http://ole.tange.dk
|
||||
|
||||
|
||||
=head1 LICENSE
|
||||
|
||||
Copyright (C) 2007-2009 Free Software Foundation, Inc.
|
||||
Copyright (C) 2007-2010 Free Software Foundation, Inc.
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
|
|
@ -535,7 +536,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|||
|
||||
=head1 DEPENDENCIES
|
||||
|
||||
B<parallel> uses Perl, and the Perl modules Getopt::Std, IPC::Open3,
|
||||
B<parallel> uses Perl, and the Perl modules Getopt::Long, IPC::Open3,
|
||||
Symbol, IO::File, POSIX, and File::Temp.
|
||||
|
||||
|
||||
|
|
@ -571,36 +572,35 @@ GetOptions("debug|D" => \$::opt_D,
|
|||
"quote|q" => \$::opt_q,
|
||||
"I=s" => \$::opt_I,
|
||||
"jobs|j=s" => \$::opt_P,
|
||||
# xargs-compatability - implemented - unittest missing
|
||||
# xargs-compatibility - implemented, man, unittest
|
||||
"max-procs|P=s" => \$::opt_P,
|
||||
"delimiter|d=s" => \$::opt_d,
|
||||
# xargs-compatibility - implemented, unittest - man missing
|
||||
"max-chars|s=i" => \$::opt_s,
|
||||
"arg-file|a=s" => \$::opt_a,
|
||||
"delimiter|d=s" => \$::opt_d,
|
||||
"no-run-if-empty|r" => \$::opt_r,
|
||||
## echo " " | parallel -r echo
|
||||
## echo " " | parallel echo
|
||||
"replace|i:s" => \$::opt_i,
|
||||
"E=s" => \$::opt_E,
|
||||
"eof|e:s" => \$::opt_E,
|
||||
"max-args|n=i" => \$::opt_n,
|
||||
## (echo a b;echo c;echo d) | parallel -k -n1 -X echo
|
||||
## (echo a b;echo c;echo d) | parallel -k -n2 -X echo
|
||||
"verbose|t" => \$::opt_t,
|
||||
"help|h" => \$::opt_h,
|
||||
"version" => \$::opt_version,
|
||||
## xargs-compatibility - implemented - unittest missing - man missing
|
||||
"interactive|p" => \$::opt_p,
|
||||
## How to unittest? tty skal emuleres
|
||||
"verbose|t" => \$::opt_t,
|
||||
|
||||
# xargs-compatability - unimplemented
|
||||
"help|h" => \$::opt_help,
|
||||
"L=i" => \$::opt_L,
|
||||
"max-lines|l:i" => \$::opt_l,
|
||||
# (echo a b;echo c) | xargs -l1 echo
|
||||
# (echo a b' ';echo c) | xargs -l1 echo
|
||||
"version" => \$::opt_version,
|
||||
## (echo a b;echo c) | xargs -l1 echo
|
||||
## (echo a b' ';echo c) | xargs -l1 echo
|
||||
"show-limits" => \$::opt_show_limits,
|
||||
"exit|x" => \$::opt_x,
|
||||
) || die_usage();
|
||||
|
||||
# Defaults:
|
||||
$Global::version = 20100203;
|
||||
$Global::debug = 0;
|
||||
$Global::processes_to_run = 10;
|
||||
$command = undef;
|
||||
|
|
@ -627,7 +627,7 @@ if(defined $::opt_u) { $Global::grouped = 0; }
|
|||
if(defined $::opt_c) { $Global::input_is_filename = 0; }
|
||||
if(defined $::opt_f) { $Global::input_is_filename = 1; }
|
||||
if(defined $::opt_0) { $/ = "\0"; }
|
||||
if(defined $::opt_d) { $/ = $::opt_d; }
|
||||
if(defined $::opt_d) { my $e="sprintf \"$::opt_d\""; $/ = eval $e; }
|
||||
if(defined $::opt_p) { $Global::interactive = $::opt_p; }
|
||||
if(defined $::opt_q) { $Global::quoting = 1; }
|
||||
if(defined $::opt_r) { $Global::ignore_empty = 1; }
|
||||
|
|
@ -636,8 +636,10 @@ if(defined $::opt_I) { $Global::replacestring = $::opt_I; }
|
|||
if(defined $::opt_i and $::opt_i) { $Global::replacestring = $::opt_i; }
|
||||
if(defined $::opt_E and $::opt_E) { $Global::end_of_file_string = $::opt_E; }
|
||||
if(defined $::opt_n and $::opt_n) { $Global::max_number_of_args = $::opt_n; }
|
||||
if(defined $::opt_h) { die_usage(); }
|
||||
if(defined $::opt_version) { version(); exit(0); }
|
||||
|
||||
if(defined $::opt_a) {
|
||||
if(defined $::opt_a) {
|
||||
if(not open(ARGFILE,"<".$::opt_a)) {
|
||||
print STDERR "parallel: Cannot open input file `$::opt_a': No such file or directory\n";
|
||||
exit(-1);
|
||||
|
|
@ -677,18 +679,18 @@ sub generate_command_line {
|
|||
my ($length_of_command_no_args);
|
||||
if($Global::xargs or $Global::Xargs) {
|
||||
# Count number of {}'s on the command line
|
||||
$number_of_substitution = ($command =~ s/$Global::replacestring/$Global::replacestring/go);
|
||||
$number_of_substitution = ($command =~ s/\Q$Global::replacestring\E/$Global::replacestring/go);
|
||||
$number_of_substitution ||= 1;
|
||||
}
|
||||
if($Global::xargs) {
|
||||
my $c = $command;
|
||||
# remove all {}s
|
||||
$c =~ s/$Global::replacestring//go;
|
||||
$c =~ s/\Q$Global::replacestring\E//go;
|
||||
$length_of_command_no_args = length($c);
|
||||
}
|
||||
if($Global::Xargs) {
|
||||
my $c = $command;
|
||||
while($c =~ s/(\S*$Global::replacestring\S*)//o) {
|
||||
while($c =~ s/(\S*\Q$Global::replacestring\E\S*)//o) {
|
||||
# Length of context minus the {}
|
||||
$length_of_context += length($1) - 2;
|
||||
}
|
||||
|
|
@ -713,7 +715,7 @@ sub generate_command_line {
|
|||
if($quoted_args[0]) {
|
||||
last;
|
||||
} else {
|
||||
die ("Command line too long at $next_arg");
|
||||
die ("Command line too long at: $next_arg");
|
||||
}
|
||||
}
|
||||
if($Global::max_number_of_args and $number_of_args >= $Global::max_number_of_args) {
|
||||
|
|
@ -723,17 +725,17 @@ sub generate_command_line {
|
|||
}
|
||||
if(@quoted_args) {
|
||||
$job_line = $command;
|
||||
if(defined $job_line and $job_line =~/$Global::replacestring/o) {
|
||||
if(defined $job_line and $job_line =~/\Q$Global::replacestring\E/o) {
|
||||
# substitute {} with args
|
||||
if($Global::Xargs) {
|
||||
# Context sensitive replace (foo{}bar with fooargsbar)
|
||||
while($job_line =~/$Global::replacestring/o) {
|
||||
$job_line =~ /(\S*$Global::replacestring\S*)/ or die ("This should never happen");
|
||||
while($job_line =~/\Q$Global::replacestring\E/o) {
|
||||
$job_line =~ /(\S*\Q$Global::replacestring\E\S*)/ or die ("This should never happen");
|
||||
my $wordarg = $1;
|
||||
my @all_word_arg;
|
||||
for my $arg (@quoted_args) {
|
||||
my $substituted = $wordarg;
|
||||
$substituted=~s/$Global::replacestring/$arg/go;
|
||||
$substituted=~s/\Q$Global::replacestring\E/$arg/go;
|
||||
push @all_word_arg, $substituted;
|
||||
}
|
||||
my $all_word_arg = join(" ",@all_word_arg);
|
||||
|
|
@ -743,7 +745,7 @@ sub generate_command_line {
|
|||
} else {
|
||||
# Normal replace {} with args
|
||||
my $arg=join(" ",@quoted_args);
|
||||
$job_line =~ s/$Global::replacestring/$arg/go;
|
||||
$job_line =~ s/\Q$Global::replacestring\E/$arg/go;
|
||||
}
|
||||
} else {
|
||||
# append args
|
||||
|
|
@ -948,11 +950,11 @@ sub user_requested_processes {
|
|||
my $opt_P = shift;
|
||||
if(defined $opt_P) {
|
||||
if($opt_P =~ /^\+(\d+)$/) {
|
||||
# E.g. -j +2
|
||||
# E.g. -P +2
|
||||
my $j = $1;
|
||||
$processes = $j + no_of_cpus();
|
||||
} elsif ($opt_P =~ /^-(\d+)$/) {
|
||||
# E.g. -j -2
|
||||
# E.g. -P -2
|
||||
my $j = $1;
|
||||
$processes = no_of_cpus() - $j;
|
||||
} elsif ($opt_P =~ /^(\d+)\%$/) {
|
||||
|
|
@ -961,7 +963,7 @@ sub user_requested_processes {
|
|||
} elsif ($opt_P =~ /^(\d+)$/) {
|
||||
$processes = $1;
|
||||
if($processes == 0) {
|
||||
# -j 0 = infinity (or at least close)
|
||||
# -P 0 = infinity (or at least close)
|
||||
$processes = 2**31;
|
||||
}
|
||||
} else {
|
||||
|
|
@ -1303,6 +1305,19 @@ sub usage {
|
|||
print "parallel [-0cdfgkqsuvxX] [-j num] [command [arguments]] < list_of_arguments\n";
|
||||
}
|
||||
|
||||
sub version {
|
||||
print join("\n",
|
||||
"parallel $Global::version",
|
||||
"Copyright (C) 2007-2010 Free Software Foundation, Inc.",
|
||||
"License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>",
|
||||
"This is free software: you are free to change and redistribute it.",
|
||||
"There is NO WARRANTY, to the extent permitted by law.",
|
||||
"",
|
||||
"Written by Ole Tange <http://ole.tange.dk>.\n"
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
#
|
||||
# Debugging
|
||||
#
|
||||
|
|
@ -1371,5 +1386,4 @@ $main::opt_X = $main::opt_x = $main::opt_k = $main::opt_d =
|
|||
$main::opt_P = $main::opt_i = $main::opt_p = $main::opt_a =
|
||||
$main::opt_version = $main::opt_L = $main::opt_l =
|
||||
$main::opt_show_limits = $main::opt_n = $main::opt_e = $main::opt_t =
|
||||
$main::opt_E = $main::opt_r = $main::opt_help = $Global::xargs =
|
||||
$Global::keeporder = 0;
|
||||
$main::opt_E = $main::opt_r = $Global::xargs = $Global::keeporder = 0;
|
||||
|
|
|
|||
11
parallel.1
11
parallel.1
|
|
@ -124,7 +124,7 @@
|
|||
.\" ========================================================================
|
||||
.\"
|
||||
.IX Title "PARALLEL 1"
|
||||
.TH PARALLEL 1 "2010-02-03" "perl v5.10.1" "User Contributed Perl Documentation"
|
||||
.TH PARALLEL 1 "2010-02-04" "perl v5.10.1" "User Contributed Perl Documentation"
|
||||
.\" For nroff, turn off justification. Always turn off hyphenation; it makes
|
||||
.\" way too many mistakes in technical documents.
|
||||
.if n .ad l
|
||||
|
|
@ -221,7 +221,8 @@ Ungroup output. Output is printed as soon as possible. This may cause
|
|||
output from different commands to be mixed. Can be reversed with \fB\-g\fR.
|
||||
.IP "\fB\-v\fR" 9
|
||||
.IX Item "-v"
|
||||
Verbose. Print the job to be run on \s-1STDOUT\s0. Can be reversed with \fB\-s\fR.
|
||||
Verbose. Print the job to be run on \s-1STDOUT\s0. Can be reversed with
|
||||
\&\fB\-\-silent\fR.
|
||||
.IP "\fB\-\-xargs\fR =item \fB\-m\fR" 9
|
||||
.IX Item "--xargs =item -m"
|
||||
Multiple. Insert as many arguments as the command line length permits. If
|
||||
|
|
@ -571,10 +572,10 @@ parallel \-\-wait /tmp/comfile will wait until no more locks on the file
|
|||
.IX Header "AUTHOR"
|
||||
Copyright (C) 2007\-10\-18 Ole Tange, http://ole.tange.dk
|
||||
.PP
|
||||
Copyright (C) 2008\-2009 Ole Tange, http://ole.tange.dk
|
||||
Copyright (C) 2008\-2010 Ole Tange, http://ole.tange.dk
|
||||
.SH "LICENSE"
|
||||
.IX Header "LICENSE"
|
||||
Copyright (C) 2007\-2009 Free Software Foundation, Inc.
|
||||
Copyright (C) 2007\-2010 Free Software Foundation, Inc.
|
||||
.PP
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the \s-1GNU\s0 General Public License as published by
|
||||
|
|
@ -590,7 +591,7 @@ You should have received a copy of the \s-1GNU\s0 General Public License
|
|||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
.SH "DEPENDENCIES"
|
||||
.IX Header "DEPENDENCIES"
|
||||
\&\fBparallel\fR uses Perl, and the Perl modules Getopt::Std, IPC::Open3,
|
||||
\&\fBparallel\fR uses Perl, and the Perl modules Getopt::Long, IPC::Open3,
|
||||
Symbol, IO::File, \s-1POSIX\s0, and File::Temp.
|
||||
.SH "SEE ALSO"
|
||||
.IX Header "SEE ALSO"
|
||||
|
|
|
|||
100
unittest/actual-results/test15
Normal file
100
unittest/actual-results/test15
Normal file
|
|
@ -0,0 +1,100 @@
|
|||
1
|
||||
2
|
||||
3
|
||||
4
|
||||
5
|
||||
6
|
||||
7
|
||||
8
|
||||
9
|
||||
10
|
||||
1
|
||||
2
|
||||
3
|
||||
4
|
||||
5
|
||||
6
|
||||
7
|
||||
8
|
||||
9
|
||||
10
|
||||
replace
|
||||
replace
|
||||
replace
|
||||
replace
|
||||
replace
|
||||
replace
|
||||
replace
|
||||
include this
|
||||
include this
|
||||
include this
|
||||
include this
|
||||
include this
|
||||
include this
|
||||
line 1
|
||||
line 2
|
||||
line 3
|
||||
line 1 line 1
|
||||
line 2
|
||||
line 1
|
||||
line 2
|
||||
line 3
|
||||
line 1 line 1
|
||||
line 2
|
||||
line 1
|
||||
line 2
|
||||
line 3
|
||||
line 1
|
||||
line 2
|
||||
line 3
|
||||
line 1 line 1
|
||||
line 2
|
||||
line 1 line 1
|
||||
line 2
|
||||
1
|
||||
2
|
||||
3
|
||||
4
|
||||
5
|
||||
6
|
||||
7
|
||||
8
|
||||
9
|
||||
10
|
||||
1
|
||||
2
|
||||
3
|
||||
4
|
||||
5
|
||||
6
|
||||
7
|
||||
8
|
||||
9
|
||||
10
|
||||
This is line 1
|
||||
This is line 2
|
||||
This is line 3
|
||||
|
||||
This is line 1
|
||||
This is line 2
|
||||
This is line 3
|
||||
|
||||
line 1
|
||||
line 2
|
||||
line 3
|
||||
line 1
|
||||
line 2
|
||||
line 3
|
||||
line 1 line 1
|
||||
line 2
|
||||
line 1 line 1
|
||||
line 2
|
||||
2
|
||||
2
|
||||
7
|
||||
echo bar
|
||||
echo car
|
||||
echo far
|
||||
echo bar
|
||||
echo car
|
||||
echo far
|
||||
68
unittest/tests-to-run/test15.sh
Normal file
68
unittest/tests-to-run/test15.sh
Normal file
|
|
@ -0,0 +1,68 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Test xargs compatibility
|
||||
|
||||
|
||||
# Test -a and --arg-file: Read input from file instead of stdin
|
||||
seq 1 10 >/tmp/$$
|
||||
parallel -a /tmp/$$ echo
|
||||
parallel --arg-file /tmp/$$ echo
|
||||
|
||||
# Test -i and --replace: Replace with argument
|
||||
(echo a; echo END; echo b) | parallel -k -i -eEND echo repl{}ce
|
||||
(echo a; echo END; echo b) | parallel -k --replace -eEND echo repl{}ce
|
||||
(echo a; echo END; echo b) | parallel -k -i+ -eEND echo repl+ce
|
||||
(echo a; echo END; echo b) | parallel -k --replace + -eEND echo repl+ce
|
||||
(echo a; echo END; echo b) | parallel -k --replace== -eEND echo repl=ce
|
||||
(echo a; echo END; echo b) | parallel -k --replace=^ -eEND echo repl^ce
|
||||
(echo a; echo END; echo b) | parallel -k -I^ -eEND echo repl^ce
|
||||
|
||||
# Test -E: Artificial end-of-file
|
||||
(echo include this; echo END; echo not this) | parallel -k -E END echo
|
||||
(echo include this; echo END; echo not this) | parallel -k -EEND echo
|
||||
|
||||
# Test -e and --eof: Artificial end-of-file
|
||||
(echo include this; echo END; echo not this) | parallel -k -e END echo
|
||||
(echo include this; echo END; echo not this) | parallel -k -eEND echo
|
||||
(echo include this; echo END; echo not this) | parallel -k --eof=END echo
|
||||
(echo include this; echo END; echo not this) | parallel -k --eof END echo
|
||||
|
||||
# Test -n and --max-args: Max number of args per line (only with -X and -m)
|
||||
(echo line 1;echo line 2;echo line 3) | parallel -k -n1 -m echo
|
||||
(echo line 1;echo line 1;echo line 2) | parallel -k -n2 -m echo
|
||||
(echo line 1;echo line 2;echo line 3) | parallel -k -n1 -X echo
|
||||
(echo line 1;echo line 1;echo line 2) | parallel -k -n2 -X echo
|
||||
(echo line 1;echo line 2;echo line 3) | parallel -k --max-args=1 -X echo
|
||||
(echo line 1;echo line 2;echo line 3) | parallel -k --max-args 1 -X echo
|
||||
(echo line 1;echo line 1;echo line 2) | parallel -k --max-args=2 -X echo
|
||||
(echo line 1;echo line 1;echo line 2) | parallel -k --max-args 2 -X echo
|
||||
|
||||
# Test --max-procs and -P: Number of processes
|
||||
seq 1 10 | parallel -k --max-procs +0 echo
|
||||
seq 1 10 | parallel -k -P 200% echo
|
||||
|
||||
# Test --delimiter and -d: Delimiter instead of newline
|
||||
# Yes there is supposed to be an extra newline for -d N
|
||||
echo line 1Nline 2Nline 3 | parallel -k -d N echo This is
|
||||
echo line 1Nline 2Nline 3 | parallel -k --delimiter N echo This is
|
||||
printf "line 1\0line 2\0line 3" | parallel -d '\0' echo
|
||||
printf "line 1\tline 2\tline 3" | parallel --delimiter '\t' echo
|
||||
|
||||
# Test --max-chars and -s: Max number of chars in a line
|
||||
(echo line 1;echo line 1;echo line 2) | parallel -k --max-chars 25 -X echo
|
||||
(echo line 1;echo line 1;echo line 2) | parallel -k -s 25 -X echo
|
||||
|
||||
# Test --no-run-if-empty and -r: This should give no output
|
||||
echo " " | parallel -r echo
|
||||
echo " " | parallel --no-run-if-empty echo
|
||||
|
||||
# Test --help and -h: Help output (just check we get the same amount of lines)
|
||||
parallel -h | wc -l
|
||||
parallel --help | wc -l
|
||||
|
||||
# Test --version: Version output (just check we get the same amount of lines)
|
||||
parallel --version | wc -l
|
||||
|
||||
# Test --verbose and -t
|
||||
(echo b; echo c; echo f) | parallel -k -t echo {}ar 2>&1 >/dev/null
|
||||
(echo b; echo c; echo f) | parallel -k --verbose echo {}ar 2>&1 >/dev/null
|
||||
100
unittest/wanted-results/test15
Normal file
100
unittest/wanted-results/test15
Normal file
|
|
@ -0,0 +1,100 @@
|
|||
1
|
||||
2
|
||||
3
|
||||
4
|
||||
5
|
||||
6
|
||||
7
|
||||
8
|
||||
9
|
||||
10
|
||||
1
|
||||
2
|
||||
3
|
||||
4
|
||||
5
|
||||
6
|
||||
7
|
||||
8
|
||||
9
|
||||
10
|
||||
replace
|
||||
replace
|
||||
replace
|
||||
replace
|
||||
replace
|
||||
replace
|
||||
replace
|
||||
include this
|
||||
include this
|
||||
include this
|
||||
include this
|
||||
include this
|
||||
include this
|
||||
line 1
|
||||
line 2
|
||||
line 3
|
||||
line 1 line 1
|
||||
line 2
|
||||
line 1
|
||||
line 2
|
||||
line 3
|
||||
line 1 line 1
|
||||
line 2
|
||||
line 1
|
||||
line 2
|
||||
line 3
|
||||
line 1
|
||||
line 2
|
||||
line 3
|
||||
line 1 line 1
|
||||
line 2
|
||||
line 1 line 1
|
||||
line 2
|
||||
1
|
||||
2
|
||||
3
|
||||
4
|
||||
5
|
||||
6
|
||||
7
|
||||
8
|
||||
9
|
||||
10
|
||||
1
|
||||
2
|
||||
3
|
||||
4
|
||||
5
|
||||
6
|
||||
7
|
||||
8
|
||||
9
|
||||
10
|
||||
This is line 1
|
||||
This is line 2
|
||||
This is line 3
|
||||
|
||||
This is line 1
|
||||
This is line 2
|
||||
This is line 3
|
||||
|
||||
line 1
|
||||
line 2
|
||||
line 3
|
||||
line 1
|
||||
line 2
|
||||
line 3
|
||||
line 1 line 1
|
||||
line 2
|
||||
line 1 line 1
|
||||
line 2
|
||||
2
|
||||
2
|
||||
7
|
||||
echo bar
|
||||
echo car
|
||||
echo far
|
||||
echo bar
|
||||
echo car
|
||||
echo far
|
||||
Loading…
Reference in a new issue