parallel: Implemented replacement string -I similar to xargs -I

This commit is contained in:
Ole Tange 2009-11-10 15:14:15 +01:00
parent 5059b5453a
commit fcfc9f6d70
8 changed files with 212 additions and 20 deletions

View file

@ -6,7 +6,7 @@ parallel - build and execute command lines from standard input in parallel
=head1 SYNOPSIS
B<parallel> [-0cfgkqsuvxX] [-j num] [command [arguments]] < list_of_arguments
B<parallel> [-0cfgkqsuvxX] [-I str] [-j num] [command [arguments]] < list_of_arguments
=head1 DESCRIPTION
@ -52,6 +52,10 @@ Group output. Output from each jobs is grouped together and is only
printed when the command is finished. STDERR first followed by STDOUT.
B<-g> is the default. Can be reversed with B<-u>.
=item B<-I> I<string>
Use the replacement string I<string> instead of {}.
=item B<-j> I<N>
Run N jobs in parallel. 0 means as many as possible. Default is 10.
@ -458,7 +462,7 @@ use Getopt::Std;
use strict;
my ($processes,$command);
getopts("0cdfgj:kqsuvxX") || die_usage();
getopts("0cdfgI:j:kqsuvxX") || die_usage();
# Defaults:
$Global::debug = 0;
@ -468,6 +472,7 @@ $Global::verbose = 0;
$Global::grouped = 1;
$Global::keeporder = 0;
$Global::quoting = 0;
$Global::replacestring = '{}';
$Global::input_is_filename = (@ARGV);
$/="\n";
@ -483,6 +488,8 @@ if(defined $::opt_c) { $Global::input_is_filename = 0; }
if(defined $::opt_f) { $Global::input_is_filename = 1; }
if(defined $::opt_0) { $/ = "\0"; }
if(defined $::opt_q) { $Global::quoting = 1; }
if(defined $::opt_I) { $Global::replacestring = $::opt_I; }
if(@ARGV) {
if($Global::quoting) {
$Global::command = join(" ", shell_quote(@ARGV));
@ -515,18 +522,18 @@ sub generate_command_line {
my ($length_of_command_no_args);
if($Global::xargs or $Global::Xargs) {
# Count number of {}'s on the command line
$number_of_substitution = ($command =~ s/{}/{}/g);
$number_of_substitution = ($command =~ s/$Global::replacestring/$Global::replacestring/go);
$number_of_substitution ||= 1;
}
if($Global::xargs) {
my $c = $command;
# remove all {}s
$c =~ s/{}//g;
$c =~ s/$Global::replacestring//go;
$length_of_command_no_args = length($c);
}
if($Global::Xargs) {
my $c = $command;
while($c =~ s/(\S*{}\S*)//) {
while($c =~ s/(\S*$Global::replacestring\S*)//o) {
# Length of context minus the {}
$length_of_context += length($1) - 2;
}
@ -556,17 +563,17 @@ sub generate_command_line {
}
if(@quoted_args) {
$job_line = $command;
if(defined $job_line and $job_line =~/{}/) {
if(defined $job_line and $job_line =~/$Global::replacestring/o) {
# substitute {} with args
if($Global::Xargs) {
# Context sensitive replace
while($job_line =~/{}/) {
$job_line =~ /(\S*{}\S*)/ or die ("This should never happen");
# Context sensitive replace (foo{}bar with fooargsbar)
while($job_line =~/$Global::replacestring/o) {
$job_line =~ /(\S*$Global::replacestring\S*)/ or die ("This should never happen");
my $wordarg = $1;
my @all_word_arg;
for my $arg (@quoted_args) {
my $substituted = $wordarg;
$substituted=~s/{}/$arg/g;
$substituted=~s/$Global::replacestring/$arg/go;
push @all_word_arg, $substituted;
}
my $all_word_arg = join(" ",@all_word_arg);
@ -574,9 +581,9 @@ sub generate_command_line {
$job_line =~ s/$quoted_wordarg/$all_word_arg/;
}
} else {
# Normal replace
# Normal replace {} with args
my $arg=join(" ",@quoted_args);
$job_line =~ s/{}/$arg/g;
$job_line =~ s/$Global::replacestring/$arg/go;
}
} else {
# append args

View file

@ -124,7 +124,7 @@
.\" ========================================================================
.\"
.IX Title "PARALLEL 1"
.TH PARALLEL 1 "2009-10-26" "perl v5.10.1" "User Contributed Perl Documentation"
.TH PARALLEL 1 "2009-11-10" "perl v5.10.1" "User Contributed Perl Documentation"
.\" For nroff, turn off justification. Always turn off hyphenation; it makes
.\" way too many mistakes in technical documents.
.if n .ad l
@ -133,7 +133,7 @@
parallel \- build and execute command lines from standard input in parallel
.SH "SYNOPSIS"
.IX Header "SYNOPSIS"
\&\fBparallel\fR [\-0cfgkqsuvxX] [\-j num] [command [arguments]] < list_of_arguments
\&\fBparallel\fR [\-0cfgkqsuvxX] [\-I str] [\-j num] [command [arguments]] < list_of_arguments
.SH "DESCRIPTION"
.IX Header "DESCRIPTION"
For each line of input \fBparallel\fR will execute \fBcommand\fR with the
@ -170,6 +170,9 @@ quoted so it is not evaluated by the shell. This is the default if
Group output. Output from each jobs is grouped together and is only
printed when the command is finished. \s-1STDERR\s0 first followed by \s-1STDOUT\s0.
\&\fB\-g\fR is the default. Can be reversed with \fB\-u\fR.
.IP "\fB\-I\fR \fIstring\fR" 9
.IX Item "-I string"
Use the replacement string \fIstring\fR instead of {}.
.IP "\fB\-j\fR \fIN\fR" 9
.IX Item "-j N"
Run N jobs in parallel. 0 means as many as possible. Default is 10.
@ -478,6 +481,8 @@ Report bugs to <bug\-parallel@tange.dk>.
xargs dropin-replacement.
Implement the missing \-\-features
.PP
\&\-I <string> replacement string
.PP
monitor to see which jobs are currently running
http://code.google.com/p/ppss/
.PP

View file

@ -4,9 +4,9 @@ b35d8e49be8d94899b719c40d3f1f4bb -
3 60000 348894
1foo bar2foo bar3 Afoo barBfoo barC
1foo2foo3 1bar2bar3 AfooBfooC AbarBbarC
1c0c49286e5b5b18437e51b438ea5475 -
51736abdee4738369ce04b354d40c887 -
6 119994 697800
1c0c49286e5b5b18437e51b438ea5475 -
51736abdee4738369ce04b354d40c887 -
Chars per line: 116300
'a'
'a'

View file

@ -0,0 +1,79 @@
1 1
2 1
2 2
3 1
3 2
3 3
4 1
4 2
4 3
4 4
5 1
5 2
5 3
5 4
5 5
6 1
6 2
6 3
6 4
6 5
6 6
7 1
7 2
7 3
7 4
7 5
7 6
7 7
8 1
8 2
8 3
8 4
8 5
8 6
8 7
8 8
9 1
9 2
9 3
9 4
9 5
9 6
9 7
9 8
9 9
10 1
10 2
10 3
10 4
10 5
10 6
10 7
10 8
10 9
10 10
a1 b1
a2 b1 b2
a3 b1 b2 b3
a4 b1 b2 b3 b4
a5 b1 b2 b3 b4 b5
a6 b1 b2 b3 b4 b5 b6
a7 b1 b2 b3 b4 b5 b6 b7
a8 b1 b2 b3 b4 b5 b6 b7 b8
a9 b1 b2 b3 b4 b5 b6 b7 b8 b9
a10 b1 b2 b3 b4 b5 b6 b7 b8 b9 b10
a1 b1
a2 b1 2
a3 b1 2 3
a4 b1 2 3 4
a5 b1 2 3 4 5
a6 b1 2 3 4 5 6
a7 b1 2 3 4 5 6 7
a8 b1 2 3 4 5 6 7 8
a9 b1 2 3 4 5 6 7 8 9
a10 b1 2 3 4 5 6 7 8 9 10
51736abdee4738369ce04b354d40c887 -
Chars per line: 116300
1471045299517233a8dc29b1c3227f2e -
Chars per line: 102223

View file

@ -8,8 +8,8 @@ seq 1 40 | parallel -j 0 seq 1 10 '| parallel -j 3 echo' | sort |md5sum
seq 1 60000 | parallel -x echo | mop -d 4 "|sort |md5sum" "| wc"
(echo foo;echo bar) | parallel -x echo 1{}2{}3 A{}B{}C
(echo foo;echo bar) | parallel -X echo 1{}2{}3 A{}B{}C
seq 1 60000 | parallel -x echo 1{}2{}3 | mop -d 4 "|sort |md5sum" "| wc"
seq 1 60000 | parallel -x echo 1{}2{}3 | \
seq 1 60000 | parallel -x echo a{}b{}c | mop -d 4 "|sort |md5sum" "| wc"
seq 1 60000 | parallel -x echo a{}b{}c | \
mop -q "|sort |md5sum" :parallel
echo -n "Chars per line: "
CHAR=$(cat ~/.mop/:parallel | wc -c)

View file

@ -0,0 +1,22 @@
#!/bin/bash
# Test -I
seq 1 10 | parallel -k 'seq 1 {} | parallel -k -I :: echo {} ::'
seq 1 10 | parallel -k 'seq 1 {} | parallel -X -k -I :: echo a{} b::'
seq 1 10 | parallel -k 'seq 1 {} | parallel -x -k -I :: echo a{} b::'
seq 1 60000 | parallel -I :: -x echo a::b::c | \
mop -q "|sort |md5sum" :parallel
echo -n "Chars per line: "
CHAR=$(cat ~/.mop/:parallel | wc -c)
LINES=$(cat ~/.mop/:parallel | wc -l)
echo "$CHAR/$LINES" | bc
seq 1 60000 | parallel -I :: -X echo a::b::c | \
mop -q "|sort |md5sum" :parallel
echo -n "Chars per line: "
CHAR=$(cat ~/.mop/:parallel | wc -c)
LINES=$(cat ~/.mop/:parallel | wc -l)
echo "$CHAR/$LINES" | bc

View file

@ -4,9 +4,9 @@ b35d8e49be8d94899b719c40d3f1f4bb -
3 60000 348894
1foo bar2foo bar3 Afoo barBfoo barC
1foo2foo3 1bar2bar3 AfooBfooC AbarBbarC
1c0c49286e5b5b18437e51b438ea5475 -
51736abdee4738369ce04b354d40c887 -
6 119994 697800
1c0c49286e5b5b18437e51b438ea5475 -
51736abdee4738369ce04b354d40c887 -
Chars per line: 116300
'a'
'a'

View file

@ -0,0 +1,79 @@
1 1
2 1
2 2
3 1
3 2
3 3
4 1
4 2
4 3
4 4
5 1
5 2
5 3
5 4
5 5
6 1
6 2
6 3
6 4
6 5
6 6
7 1
7 2
7 3
7 4
7 5
7 6
7 7
8 1
8 2
8 3
8 4
8 5
8 6
8 7
8 8
9 1
9 2
9 3
9 4
9 5
9 6
9 7
9 8
9 9
10 1
10 2
10 3
10 4
10 5
10 6
10 7
10 8
10 9
10 10
a1 b1
a2 b1 b2
a3 b1 b2 b3
a4 b1 b2 b3 b4
a5 b1 b2 b3 b4 b5
a6 b1 b2 b3 b4 b5 b6
a7 b1 b2 b3 b4 b5 b6 b7
a8 b1 b2 b3 b4 b5 b6 b7 b8
a9 b1 b2 b3 b4 b5 b6 b7 b8 b9
a10 b1 b2 b3 b4 b5 b6 b7 b8 b9 b10
a1 b1
a2 b1 2
a3 b1 2 3
a4 b1 2 3 4
a5 b1 2 3 4 5
a6 b1 2 3 4 5 6
a7 b1 2 3 4 5 6 7
a8 b1 2 3 4 5 6 7 8
a9 b1 2 3 4 5 6 7 8 9
a10 b1 2 3 4 5 6 7 8 9 10
51736abdee4738369ce04b354d40c887 -
Chars per line: 116300
1471045299517233a8dc29b1c3227f2e -
Chars per line: 102223