diff --git a/parallel b/parallel index 36b42746..c16813cf 100755 --- a/parallel +++ b/parallel @@ -6,7 +6,7 @@ parallel - build and execute command lines from standard input in parallel =head1 SYNOPSIS -B [-0cfgkqsuvxX] [-j num] [command [arguments]] < list_of_arguments +B [-0cfgkqsuvxX] [-I str] [-j num] [command [arguments]] < list_of_arguments =head1 DESCRIPTION @@ -52,6 +52,10 @@ Group output. Output from each jobs is grouped together and is only printed when the command is finished. STDERR first followed by STDOUT. B<-g> is the default. Can be reversed with B<-u>. +=item B<-I> I + +Use the replacement string I instead of {}. + =item B<-j> I Run N jobs in parallel. 0 means as many as possible. Default is 10. @@ -458,7 +462,7 @@ use Getopt::Std; use strict; my ($processes,$command); -getopts("0cdfgj:kqsuvxX") || die_usage(); +getopts("0cdfgI:j:kqsuvxX") || die_usage(); # Defaults: $Global::debug = 0; @@ -468,6 +472,7 @@ $Global::verbose = 0; $Global::grouped = 1; $Global::keeporder = 0; $Global::quoting = 0; +$Global::replacestring = '{}'; $Global::input_is_filename = (@ARGV); $/="\n"; @@ -483,6 +488,8 @@ if(defined $::opt_c) { $Global::input_is_filename = 0; } if(defined $::opt_f) { $Global::input_is_filename = 1; } if(defined $::opt_0) { $/ = "\0"; } if(defined $::opt_q) { $Global::quoting = 1; } +if(defined $::opt_I) { $Global::replacestring = $::opt_I; } + if(@ARGV) { if($Global::quoting) { $Global::command = join(" ", shell_quote(@ARGV)); @@ -515,18 +522,18 @@ sub generate_command_line { my ($length_of_command_no_args); if($Global::xargs or $Global::Xargs) { # Count number of {}'s on the command line - $number_of_substitution = ($command =~ s/{}/{}/g); + $number_of_substitution = ($command =~ s/$Global::replacestring/$Global::replacestring/go); $number_of_substitution ||= 1; } if($Global::xargs) { my $c = $command; # remove all {}s - $c =~ s/{}//g; + $c =~ s/$Global::replacestring//go; $length_of_command_no_args = length($c); } if($Global::Xargs) { my $c = $command; - while($c =~ s/(\S*{}\S*)//) { + while($c =~ s/(\S*$Global::replacestring\S*)//o) { # Length of context minus the {} $length_of_context += length($1) - 2; } @@ -556,17 +563,17 @@ sub generate_command_line { } if(@quoted_args) { $job_line = $command; - if(defined $job_line and $job_line =~/{}/) { + if(defined $job_line and $job_line =~/$Global::replacestring/o) { # substitute {} with args if($Global::Xargs) { - # Context sensitive replace - while($job_line =~/{}/) { - $job_line =~ /(\S*{}\S*)/ or die ("This should never happen"); + # Context sensitive replace (foo{}bar with fooargsbar) + while($job_line =~/$Global::replacestring/o) { + $job_line =~ /(\S*$Global::replacestring\S*)/ or die ("This should never happen"); my $wordarg = $1; my @all_word_arg; for my $arg (@quoted_args) { my $substituted = $wordarg; - $substituted=~s/{}/$arg/g; + $substituted=~s/$Global::replacestring/$arg/go; push @all_word_arg, $substituted; } my $all_word_arg = join(" ",@all_word_arg); @@ -574,9 +581,9 @@ sub generate_command_line { $job_line =~ s/$quoted_wordarg/$all_word_arg/; } } else { - # Normal replace + # Normal replace {} with args my $arg=join(" ",@quoted_args); - $job_line =~ s/{}/$arg/g; + $job_line =~ s/$Global::replacestring/$arg/go; } } else { # append args diff --git a/parallel.1 b/parallel.1 index c16bfa21..cef511ec 100644 --- a/parallel.1 +++ b/parallel.1 @@ -124,7 +124,7 @@ .\" ======================================================================== .\" .IX Title "PARALLEL 1" -.TH PARALLEL 1 "2009-10-26" "perl v5.10.1" "User Contributed Perl Documentation" +.TH PARALLEL 1 "2009-11-10" "perl v5.10.1" "User Contributed Perl Documentation" .\" For nroff, turn off justification. Always turn off hyphenation; it makes .\" way too many mistakes in technical documents. .if n .ad l @@ -133,7 +133,7 @@ parallel \- build and execute command lines from standard input in parallel .SH "SYNOPSIS" .IX Header "SYNOPSIS" -\&\fBparallel\fR [\-0cfgkqsuvxX] [\-j num] [command [arguments]] < list_of_arguments +\&\fBparallel\fR [\-0cfgkqsuvxX] [\-I str] [\-j num] [command [arguments]] < list_of_arguments .SH "DESCRIPTION" .IX Header "DESCRIPTION" For each line of input \fBparallel\fR will execute \fBcommand\fR with the @@ -170,6 +170,9 @@ quoted so it is not evaluated by the shell. This is the default if Group output. Output from each jobs is grouped together and is only printed when the command is finished. \s-1STDERR\s0 first followed by \s-1STDOUT\s0. \&\fB\-g\fR is the default. Can be reversed with \fB\-u\fR. +.IP "\fB\-I\fR \fIstring\fR" 9 +.IX Item "-I string" +Use the replacement string \fIstring\fR instead of {}. .IP "\fB\-j\fR \fIN\fR" 9 .IX Item "-j N" Run N jobs in parallel. 0 means as many as possible. Default is 10. @@ -478,6 +481,8 @@ Report bugs to . xargs dropin-replacement. Implement the missing \-\-features .PP +\&\-I replacement string +.PP monitor to see which jobs are currently running http://code.google.com/p/ppss/ .PP diff --git a/unittest/actual-results/test10 b/unittest/actual-results/test10 index 041aae9e..765b331a 100644 --- a/unittest/actual-results/test10 +++ b/unittest/actual-results/test10 @@ -4,9 +4,9 @@ b35d8e49be8d94899b719c40d3f1f4bb - 3 60000 348894 1foo bar2foo bar3 Afoo barBfoo barC 1foo2foo3 1bar2bar3 AfooBfooC AbarBbarC -1c0c49286e5b5b18437e51b438ea5475 - +51736abdee4738369ce04b354d40c887 - 6 119994 697800 -1c0c49286e5b5b18437e51b438ea5475 - +51736abdee4738369ce04b354d40c887 - Chars per line: 116300 'a' 'a' diff --git a/unittest/actual-results/test14 b/unittest/actual-results/test14 new file mode 100644 index 00000000..bd90c37b --- /dev/null +++ b/unittest/actual-results/test14 @@ -0,0 +1,79 @@ +1 1 +2 1 +2 2 +3 1 +3 2 +3 3 +4 1 +4 2 +4 3 +4 4 +5 1 +5 2 +5 3 +5 4 +5 5 +6 1 +6 2 +6 3 +6 4 +6 5 +6 6 +7 1 +7 2 +7 3 +7 4 +7 5 +7 6 +7 7 +8 1 +8 2 +8 3 +8 4 +8 5 +8 6 +8 7 +8 8 +9 1 +9 2 +9 3 +9 4 +9 5 +9 6 +9 7 +9 8 +9 9 +10 1 +10 2 +10 3 +10 4 +10 5 +10 6 +10 7 +10 8 +10 9 +10 10 +a1 b1 +a2 b1 b2 +a3 b1 b2 b3 +a4 b1 b2 b3 b4 +a5 b1 b2 b3 b4 b5 +a6 b1 b2 b3 b4 b5 b6 +a7 b1 b2 b3 b4 b5 b6 b7 +a8 b1 b2 b3 b4 b5 b6 b7 b8 +a9 b1 b2 b3 b4 b5 b6 b7 b8 b9 +a10 b1 b2 b3 b4 b5 b6 b7 b8 b9 b10 +a1 b1 +a2 b1 2 +a3 b1 2 3 +a4 b1 2 3 4 +a5 b1 2 3 4 5 +a6 b1 2 3 4 5 6 +a7 b1 2 3 4 5 6 7 +a8 b1 2 3 4 5 6 7 8 +a9 b1 2 3 4 5 6 7 8 9 +a10 b1 2 3 4 5 6 7 8 9 10 +51736abdee4738369ce04b354d40c887 - +Chars per line: 116300 +1471045299517233a8dc29b1c3227f2e - +Chars per line: 102223 diff --git a/unittest/tests-to-run/test10.sh b/unittest/tests-to-run/test10.sh index c2e1becb..e3a25bad 100644 --- a/unittest/tests-to-run/test10.sh +++ b/unittest/tests-to-run/test10.sh @@ -8,8 +8,8 @@ seq 1 40 | parallel -j 0 seq 1 10 '| parallel -j 3 echo' | sort |md5sum seq 1 60000 | parallel -x echo | mop -d 4 "|sort |md5sum" "| wc" (echo foo;echo bar) | parallel -x echo 1{}2{}3 A{}B{}C (echo foo;echo bar) | parallel -X echo 1{}2{}3 A{}B{}C -seq 1 60000 | parallel -x echo 1{}2{}3 | mop -d 4 "|sort |md5sum" "| wc" -seq 1 60000 | parallel -x echo 1{}2{}3 | \ +seq 1 60000 | parallel -x echo a{}b{}c | mop -d 4 "|sort |md5sum" "| wc" +seq 1 60000 | parallel -x echo a{}b{}c | \ mop -q "|sort |md5sum" :parallel echo -n "Chars per line: " CHAR=$(cat ~/.mop/:parallel | wc -c) diff --git a/unittest/tests-to-run/test14.sh b/unittest/tests-to-run/test14.sh new file mode 100644 index 00000000..8edd41d2 --- /dev/null +++ b/unittest/tests-to-run/test14.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +# Test -I +seq 1 10 | parallel -k 'seq 1 {} | parallel -k -I :: echo {} ::' + +seq 1 10 | parallel -k 'seq 1 {} | parallel -X -k -I :: echo a{} b::' + +seq 1 10 | parallel -k 'seq 1 {} | parallel -x -k -I :: echo a{} b::' + +seq 1 60000 | parallel -I :: -x echo a::b::c | \ + mop -q "|sort |md5sum" :parallel +echo -n "Chars per line: " +CHAR=$(cat ~/.mop/:parallel | wc -c) +LINES=$(cat ~/.mop/:parallel | wc -l) +echo "$CHAR/$LINES" | bc + +seq 1 60000 | parallel -I :: -X echo a::b::c | \ + mop -q "|sort |md5sum" :parallel +echo -n "Chars per line: " +CHAR=$(cat ~/.mop/:parallel | wc -c) +LINES=$(cat ~/.mop/:parallel | wc -l) +echo "$CHAR/$LINES" | bc diff --git a/unittest/wanted-results/test10 b/unittest/wanted-results/test10 index 041aae9e..765b331a 100644 --- a/unittest/wanted-results/test10 +++ b/unittest/wanted-results/test10 @@ -4,9 +4,9 @@ b35d8e49be8d94899b719c40d3f1f4bb - 3 60000 348894 1foo bar2foo bar3 Afoo barBfoo barC 1foo2foo3 1bar2bar3 AfooBfooC AbarBbarC -1c0c49286e5b5b18437e51b438ea5475 - +51736abdee4738369ce04b354d40c887 - 6 119994 697800 -1c0c49286e5b5b18437e51b438ea5475 - +51736abdee4738369ce04b354d40c887 - Chars per line: 116300 'a' 'a' diff --git a/unittest/wanted-results/test14 b/unittest/wanted-results/test14 new file mode 100644 index 00000000..bd90c37b --- /dev/null +++ b/unittest/wanted-results/test14 @@ -0,0 +1,79 @@ +1 1 +2 1 +2 2 +3 1 +3 2 +3 3 +4 1 +4 2 +4 3 +4 4 +5 1 +5 2 +5 3 +5 4 +5 5 +6 1 +6 2 +6 3 +6 4 +6 5 +6 6 +7 1 +7 2 +7 3 +7 4 +7 5 +7 6 +7 7 +8 1 +8 2 +8 3 +8 4 +8 5 +8 6 +8 7 +8 8 +9 1 +9 2 +9 3 +9 4 +9 5 +9 6 +9 7 +9 8 +9 9 +10 1 +10 2 +10 3 +10 4 +10 5 +10 6 +10 7 +10 8 +10 9 +10 10 +a1 b1 +a2 b1 b2 +a3 b1 b2 b3 +a4 b1 b2 b3 b4 +a5 b1 b2 b3 b4 b5 +a6 b1 b2 b3 b4 b5 b6 +a7 b1 b2 b3 b4 b5 b6 b7 +a8 b1 b2 b3 b4 b5 b6 b7 b8 +a9 b1 b2 b3 b4 b5 b6 b7 b8 b9 +a10 b1 b2 b3 b4 b5 b6 b7 b8 b9 b10 +a1 b1 +a2 b1 2 +a3 b1 2 3 +a4 b1 2 3 4 +a5 b1 2 3 4 5 +a6 b1 2 3 4 5 6 +a7 b1 2 3 4 5 6 7 +a8 b1 2 3 4 5 6 7 8 +a9 b1 2 3 4 5 6 7 8 9 +a10 b1 2 3 4 5 6 7 8 9 10 +51736abdee4738369ce04b354d40c887 - +Chars per line: 116300 +1471045299517233a8dc29b1c3227f2e - +Chars per line: 102223