tange/parallel
1
0
Fork 0
mirror of https://git.savannah.gnu.org/git/parallel.git synced 2024-11-22 05:57:54 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

parallel: Made options compatible with xargs: -x -> -m, -s -> --silent

Browse source
This commit is contained in:
Ole Tange 2010-02-03 16:32:36 +01:00
parent aaa69ffa1b
commit 2280fd8215
4 changed files with 76 additions and 41 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

55
parallel
View file

@ -88,7 +88,7 @@ characters are not interpreted by the shell. See the section
QUOTING. Most people will never need this. Quoting is disabled by QUOTING. Most people will never need this. Quoting is disabled by
default. default.
=item B<-s> =item B<--silent>
Silent. The job to be run will not be printed. This is the default. Silent. The job to be run will not be printed. This is the default.
Can be reversed with B<-v>. Can be reversed with B<-v>.
@ -100,17 +100,17 @@ output from different commands to be mixed. Can be reversed with B<-g>.
=item B<-v> =item B<-v>
Verbose. Print the job to be run. Can be reversed with B<-s>. Verbose. Print the job to be run on standard output. Can be reversed with B<-s>.
=item B<-x> =item B<-m>
xargs. Insert as many arguments as the command line length permits. If Multiple. Insert as many arguments as the command line length permits. If
{} is not used the arguments will be appended to the line. If {} is {} is not used the arguments will be appended to the line. If {} is
used multiple times each {} will be replaced with all the arguments. used multiple times each {} will be replaced with all the arguments.
=item B<-X> =item B<-X>
xargs with context replace. This works like B<-x> except if {} is part xargs with context replace. This works like B<-m> except if {} is part
of a word (like I<pic{}.jpg>) then the whole word will be repeated. of a word (like I<pic{}.jpg>) then the whole word will be repeated.
=back =back
@ -522,11 +522,11 @@ my ($processes,$command);
# getopts("0cdfgI:j:kqsuvxX") || die_usage(); # getopts("0cdfgI:j:kqsuvxX") || die_usage();
Getopt::Long::Configure ("bundling","require_order"); Getopt::Long::Configure ("bundling","require_order");
GetOptions("debug|d" => \$::opt_d, GetOptions("debug|D" => \$::opt_D,
"xargs|x" => \$::opt_x, "xargs|m" => \$::opt_m,
"X" => \$::opt_X, "X" => \$::opt_X,
"verbose|v" => \$::opt_v, "v" => \$::opt_v,
"silent|s" => \$::opt_s, "silent" => \$::opt_silent,
"keeporder|k" => \$::opt_k, "keeporder|k" => \$::opt_k,
"group|g" => \$::opt_g, "group|g" => \$::opt_g,
"ungroup|u" => \$::opt_u, "ungroup|u" => \$::opt_u,
@ -534,8 +534,27 @@ GetOptions("debug|d" => \$::opt_d,
"file|f" => \$::opt_f, "file|f" => \$::opt_f,
"null|0" => \$::opt_0, "null|0" => \$::opt_0,
"quote|q" => \$::opt_q, "quote|q" => \$::opt_q,
"replace|I=s" => \$::opt_I, "I=s" => \$::opt_I,
"jobs|j=s" => \$::opt_j) || die_usage(); "jobs|j=s" => \$::opt_j,
# xargs-compatability - implemented
# xargs-compatability - unimplemented
"arg-fil|a=s" => \$::opt_a,
"delimiter|d=s" => \$::opt_d,
"E=s" => \$::opt_E,
"eof|e:s" => \$::opt_e,
"help|h" => \$::opt_help,
"replace|i:s" => \$::opt_i,
"L=i" => \$::opt_L,
"max-lines|l:i" => \$::opt_l,
"max-args|n=i" => \$::opt_n,
"interactive|p" => \$::opt_p,
"no-run-if-empty|r" => \$::opt_r,
"max-chars|s=i" => \$::opt_s,
"version" => \$::opt_version,
"verbose|t" => \$::opt_t,
"show-limits" => \$::opt_show_limits,
"exit|x" => \$::opt_x,
"max-procs|P=s" => \$::opt_P) || die_usage();
# Defaults: # Defaults:
$Global::debug = 0; $Global::debug = 0;
@ -549,11 +568,11 @@ $Global::replacestring = '{}';
$Global::input_is_filename = (@ARGV); $Global::input_is_filename = (@ARGV);
$/="\n"; $/="\n";
$Global::debug = (defined $::opt_d); $Global::debug = (defined $::opt_D);
if(defined $::opt_x) { $Global::xargs = 1; } if(defined $::opt_m) { $Global::xargs = 1; }
if(defined $::opt_X) { $Global::Xargs = 1; } if(defined $::opt_X) { $Global::Xargs = 1; }
if(defined $::opt_v) { $Global::verbose = 1; } if(defined $::opt_v) { $Global::verbose = 1; }
if(defined $::opt_s) { $Global::verbose = 0; } if(defined $::opt_silent) { $Global::verbose = 0; }
if(defined $::opt_k) { $Global::keeporder = 1; } if(defined $::opt_k) { $Global::keeporder = 1; }
if(defined $::opt_g) { $Global::grouped = 1; } if(defined $::opt_g) { $Global::grouped = 1; }
if(defined $::opt_u) { $Global::grouped = 0; } if(defined $::opt_u) { $Global::grouped = 0; }
@ -1244,6 +1263,10 @@ sub my_dump {
# Keep perl -w happy # Keep perl -w happy
$main::opt_u = $main::opt_c = $main::opt_f = $main::opt_q = $main::opt_u = $main::opt_c = $main::opt_f = $main::opt_q =
$main::opt_0 = $main::opt_s = $main::opt_v = $main::opt_g = $main::opt_0 = $main::opt_s = $main::opt_v = $main::opt_g =
$main::opt_j = $main::opt_d = $main::opt_x = $main::opt_X = $main::opt_j = $main::opt_D = $main::opt_m = $main::opt_X =
$main::opt_k = $main::opt_x =
$main::opt_k = $main::opt_d = $main::opt_P = $main::opt_i =
$main::opt_p = $main::opt_a = $main::opt_version = $main::opt_L =
$main::opt_l = $main::opt_show_limits = $main::opt_n = $main::opt_e =
$main::opt_t = $main::opt_E = $main::opt_r = $main::opt_help =
$Global::xargs = $Global::keeporder = 0; $Global::xargs = $Global::keeporder = 0;

50
parallel.1
View file

@ -124,7 +124,7 @@
.\" ======================================================================== .\" ========================================================================
.\" .\"
.IX Title "PARALLEL 1" .IX Title "PARALLEL 1"
.TH PARALLEL 1 "2010-01-29" "perl v5.10.1" "User Contributed Perl Documentation" .TH PARALLEL 1 "2010-02-03" "perl v5.10.1" "User Contributed Perl Documentation"
.\" For nroff, turn off justification. Always turn off hyphenation; it makes .\" For nroff, turn off justification. Always turn off hyphenation; it makes
.\" way too many mistakes in technical documents. .\" way too many mistakes in technical documents.
.if n .ad l .if n .ad l
@ -199,8 +199,8 @@ Quote \fBcommand\fR. This will quote the command line so special
characters are not interpreted by the shell. See the section characters are not interpreted by the shell. See the section
\&\s-1QUOTING\s0. Most people will never need this. Quoting is disabled by \&\s-1QUOTING\s0. Most people will never need this. Quoting is disabled by
default. default.
.IP "\fB\-s\fR" 9 .IP "\fB\-\-silent\fR" 9
.IX Item "-s" .IX Item "--silent"
Silent. The job to be run will not be printed. This is the default. Silent. The job to be run will not be printed. This is the default.
Can be reversed with \fB\-v\fR. Can be reversed with \fB\-v\fR.
.IP "\fB\-u\fR" 9 .IP "\fB\-u\fR" 9
@ -209,15 +209,15 @@ Ungroup output. Output is printed as soon as possible. This may cause
output from different commands to be mixed. Can be reversed with \fB\-g\fR. output from different commands to be mixed. Can be reversed with \fB\-g\fR.
.IP "\fB\-v\fR" 9 .IP "\fB\-v\fR" 9
.IX Item "-v" .IX Item "-v"
Verbose. Print the job to be run. Can be reversed with \fB\-s\fR. Verbose. Print the job to be run on standard output. Can be reversed with \fB\-s\fR.
.IP "\fB\-x\fR" 9 .IP "\fB\-m\fR" 9
.IX Item "-x" .IX Item "-m"
xargs. Insert as many arguments as the command line length permits. If Multiple. Insert as many arguments as the command line length permits. If
{} is not used the arguments will be appended to the line. If {} is {} is not used the arguments will be appended to the line. If {} is
used multiple times each {} will be replaced with all the arguments. used multiple times each {} will be replaced with all the arguments.
.IP "\fB\-X\fR" 9 .IP "\fB\-X\fR" 9
.IX Item "-X" .IX Item "-X"
xargs with context replace. This works like \fB\-x\fR except if {} is part xargs with context replace. This works like \fB\-m\fR except if {} is part
of a word (like \fIpic{}.jpg\fR) then the whole word will be repeated. of a word (like \fIpic{}.jpg\fR) then the whole word will be repeated.
.SH "EXAMPLE 1: Working as cat | sh. Ressource inexpensive jobs and evaluation" .SH "EXAMPLE 1: Working as cat | sh. Ressource inexpensive jobs and evaluation"
.IX Header "EXAMPLE 1: Working as cat | sh. Ressource inexpensive jobs and evaluation" .IX Header "EXAMPLE 1: Working as cat | sh. Ressource inexpensive jobs and evaluation"
@ -413,7 +413,7 @@ easier just to write a small script and have \fBparallel\fR call that
script. script.
.SH "LIST RUNNING JOBS" .SH "LIST RUNNING JOBS"
.IX Header "LIST RUNNING JOBS" .IX Header "LIST RUNNING JOBS"
To list the jobs currently running you can run: If you want a list of the jobs currently running you can run:
.PP .PP
\&\fBkillall \-USR1 parallel\fR \&\fBkillall \-USR1 parallel\fR
.PP .PP
@ -422,9 +422,9 @@ To list the jobs currently running you can run:
.IX Header "COMPLETE RUNNING JOBS BUT DO NOT START NEW JOBS" .IX Header "COMPLETE RUNNING JOBS BUT DO NOT START NEW JOBS"
If you regret starting a lot of jobs you can simply break \fBparallel\fR, If you regret starting a lot of jobs you can simply break \fBparallel\fR,
but if you want to make sure you do not have halfcompleted jobs you but if you want to make sure you do not have halfcompleted jobs you
should send the signal \fB\-USR2\fR to \fBparallel\fR: should send the signal \fB\s-1SIGTERM\s0\fR to \fBparallel\fR:
.PP .PP
\&\fBkillall \-USR2 parallel\fR \&\fBkillall \-TERM parallel\fR
.PP .PP
This will tell \fBparallel\fR to not start any new jobs, but wait until This will tell \fBparallel\fR to not start any new jobs, but wait until
the currently running jobs are finished. the currently running jobs are finished.
@ -443,6 +443,8 @@ hosts or URLs) will require creating these inputs as files. \fBfind
touch important_file touch important_file
touch 'not important_file' touch 'not important_file'
ls not* | xargs rm ls not* | xargs rm
mkdir \-p '12" records'
ls | xargs rmdir
.PP .PP
You can specify \fB\-0\fR or \fB\-d \*(L"\en\*(R"\fR, but many input generators are not You can specify \fB\-0\fR or \fB\-d \*(L"\en\*(R"\fR, but many input generators are not
optimized for using \fB\s-1NUL\s0\fR as separator but are optimized for optimized for using \fB\s-1NUL\s0\fR as separator but are optimized for
@ -513,22 +515,32 @@ Implement the missing \-\-features
monitor to see which jobs are currently running monitor to see which jobs are currently running
http://code.google.com/p/ppss/ http://code.google.com/p/ppss/
.PP .PP
Accept signal \s-1INT\s0 to complete current running jobs but do not start Accept signal \s-1INT\s0 instead of \s-1TERM\s0 to complete current running jobs but
new jobs. Print out the number of jobs waiting to complete on do not start new jobs. Print out the number of jobs waiting to
\&\s-1STDERR\s0. Accept sig \s-1INT\s0 again to kill now. This seems to be hard, as complete on \s-1STDERR\s0. Accept sig \s-1INT\s0 again to kill now. This seems to be
all foreground processes get the \s-1INT\s0 from the shell. hard, as all foreground processes get the \s-1INT\s0 from the shell.
.PP
If there are nomore jobs (\s-1STDIN\s0 is closed) then make sure to
distribute the arguments evenly if running \-X.
.PP .PP
Distibute jobs to computers with different speeds/no_of_cpu using ssh Distibute jobs to computers with different speeds/no_of_cpu using ssh
ask the computers how many cpus they have and spawn appropriately ask the computers how many cpus they have and spawn appropriately
according to \-j setting. according to \-j setting. Reuse ssh connection (\-M and \-S)
http://www.semicomplete.com/blog/geekery/distributed\-xargs.html?source=rss20 http://www.semicomplete.com/blog/geekery/distributed\-xargs.html?source=rss20
http://code.google.com/p/ppss/wiki/Manual2
.SS "\-S" .SS "\-S"
.IX Subsection "-S" .IX Subsection "-S"
\&\-S sshlogin[,sshlogin] \&\-S sshlogin[,sshlogin]
.PP .PP
sshlogin is [user@]host or filename with list of sshlogin sshlogin is [user@]host or filename with list of sshlogin
.PP .PP
What about copying data to/from remote host? What about copying data to remote host? Have an option that says the
argument is a file that should be copied.
.PP
What about copying data from remote host? Have an option that says
the argument is a file that should be copied.
.PP
Where will '>' be run? Local or remote?
.PP .PP
Parallelize so this can be done: Parallelize so this can be done:
mdm.screen find dir \-execdir mdm-run cmd {} \e; mdm.screen find dir \-execdir mdm-run cmd {} \e;
@ -553,7 +565,7 @@ Copyright (C) 2007\-2009 Free Software Foundation, Inc.
This program is free software; you can redistribute it and/or modify This program is free software; you can redistribute it and/or modify
it under the terms of the \s-1GNU\s0 General Public License as published by it under the terms of the \s-1GNU\s0 General Public License as published by
the Free Software Foundation; either version 3 of the License, or the Free Software Foundation; either version 3 of the License, or
(at your option) any later version. at your option any later version.
.PP .PP
This program is distributed in the hope that it will be useful, This program is distributed in the hope that it will be useful,
but \s-1WITHOUT\s0 \s-1ANY\s0 \s-1WARRANTY\s0; without even the implied warranty of but \s-1WITHOUT\s0 \s-1ANY\s0 \s-1WARRANTY\s0; without even the implied warranty of
@ -568,4 +580,4 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
Symbol, IO::File, \s-1POSIX\s0, and File::Temp. Symbol, IO::File, \s-1POSIX\s0, and File::Temp.
.SH "SEE ALSO" .SH "SEE ALSO"
.IX Header "SEE ALSO" .IX Header "SEE ALSO"
\&\fBfind\fR(1), \fBxargs\fR(1) \&\fBfind\fR(1), \fBxargs\fR(1), '

8
unittest/tests-to-run/test10.sh
View file

@ -5,11 +5,11 @@ seq 1 40 | parallel -j 0 seq 1 10 | sort |md5sum
seq 1 40 | parallel -j 0 seq 1 10 '| parallel -j 3 echo' | sort |md5sum seq 1 40 | parallel -j 0 seq 1 10 '| parallel -j 3 echo' | sort |md5sum
# Test of xargs # Test of xargs
seq 1 60000 | parallel -x echo | mop -d 4 "|sort |md5sum" "| wc" seq 1 60000 | parallel -m echo | mop -d 4 "|sort |md5sum" "| wc"
(echo foo;echo bar) | parallel -x echo 1{}2{}3 A{}B{}C (echo foo;echo bar) | parallel -m echo 1{}2{}3 A{}B{}C
(echo foo;echo bar) | parallel -X echo 1{}2{}3 A{}B{}C (echo foo;echo bar) | parallel -X echo 1{}2{}3 A{}B{}C
seq 1 60000 | parallel -x echo a{}b{}c | mop -d 4 "|sort |md5sum" "| wc" seq 1 60000 | parallel -m echo a{}b{}c | mop -d 4 "|sort |md5sum" "| wc"
seq 1 60000 | parallel -x echo a{}b{}c | \ seq 1 60000 | parallel -m echo a{}b{}c | \
mop -q "|sort |md5sum" :parallel mop -q "|sort |md5sum" :parallel
echo -n "Chars per line: " echo -n "Chars per line: "
CHAR=$(cat ~/.mop/:parallel | wc -c) CHAR=$(cat ~/.mop/:parallel | wc -c)

4
unittest/tests-to-run/test14.sh
View file

@ -5,9 +5,9 @@ seq 1 10 | parallel -k 'seq 1 {} | parallel -k -I :: echo {} ::'
seq 1 10 | parallel -k 'seq 1 {} | parallel -X -k -I :: echo a{} b::' seq 1 10 | parallel -k 'seq 1 {} | parallel -X -k -I :: echo a{} b::'
seq 1 10 | parallel -k 'seq 1 {} | parallel -x -k -I :: echo a{} b::' seq 1 10 | parallel -k 'seq 1 {} | parallel -m -k -I :: echo a{} b::'
seq 1 60000 | parallel -I :: -x echo a::b::c | \ seq 1 60000 | parallel -I :: -m echo a::b::c | \
mop -q "|sort |md5sum" :parallel mop -q "|sort |md5sum" :parallel
echo -n "Chars per line: " echo -n "Chars per line: "
CHAR=$(cat ~/.mop/:parallel | wc -c) CHAR=$(cat ~/.mop/:parallel | wc -c)