From 2280fd8215dee185ffe4f754696e1669fb947d5d Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Wed, 3 Feb 2010 16:32:36 +0100 Subject: [PATCH] parallel: Made options compatible with xargs: -x -> -m, -s -> --silent --- parallel | 55 +++++++++++++++++++++++---------- parallel.1 | 50 ++++++++++++++++++------------ unittest/tests-to-run/test10.sh | 8 ++--- unittest/tests-to-run/test14.sh | 4 +-- 4 files changed, 76 insertions(+), 41 deletions(-) diff --git a/parallel b/parallel index 5b27dfdf..295ef405 100755 --- a/parallel +++ b/parallel @@ -88,7 +88,7 @@ characters are not interpreted by the shell. See the section QUOTING. Most people will never need this. Quoting is disabled by default. -=item B<-s> +=item B<--silent> Silent. The job to be run will not be printed. This is the default. Can be reversed with B<-v>. @@ -100,17 +100,17 @@ output from different commands to be mixed. Can be reversed with B<-g>. =item B<-v> -Verbose. Print the job to be run. Can be reversed with B<-s>. +Verbose. Print the job to be run on standard output. Can be reversed with B<-s>. -=item B<-x> +=item B<-m> -xargs. Insert as many arguments as the command line length permits. If +Multiple. Insert as many arguments as the command line length permits. If {} is not used the arguments will be appended to the line. If {} is used multiple times each {} will be replaced with all the arguments. =item B<-X> -xargs with context replace. This works like B<-x> except if {} is part +xargs with context replace. This works like B<-m> except if {} is part of a word (like I) then the whole word will be repeated. =back @@ -522,11 +522,11 @@ my ($processes,$command); # getopts("0cdfgI:j:kqsuvxX") || die_usage(); Getopt::Long::Configure ("bundling","require_order"); -GetOptions("debug|d" => \$::opt_d, - "xargs|x" => \$::opt_x, +GetOptions("debug|D" => \$::opt_D, + "xargs|m" => \$::opt_m, "X" => \$::opt_X, - "verbose|v" => \$::opt_v, - "silent|s" => \$::opt_s, + "v" => \$::opt_v, + "silent" => \$::opt_silent, "keeporder|k" => \$::opt_k, "group|g" => \$::opt_g, "ungroup|u" => \$::opt_u, @@ -534,8 +534,27 @@ GetOptions("debug|d" => \$::opt_d, "file|f" => \$::opt_f, "null|0" => \$::opt_0, "quote|q" => \$::opt_q, - "replace|I=s" => \$::opt_I, - "jobs|j=s" => \$::opt_j) || die_usage(); + "I=s" => \$::opt_I, + "jobs|j=s" => \$::opt_j, + # xargs-compatability - implemented + # xargs-compatability - unimplemented + "arg-fil|a=s" => \$::opt_a, + "delimiter|d=s" => \$::opt_d, + "E=s" => \$::opt_E, + "eof|e:s" => \$::opt_e, + "help|h" => \$::opt_help, + "replace|i:s" => \$::opt_i, + "L=i" => \$::opt_L, + "max-lines|l:i" => \$::opt_l, + "max-args|n=i" => \$::opt_n, + "interactive|p" => \$::opt_p, + "no-run-if-empty|r" => \$::opt_r, + "max-chars|s=i" => \$::opt_s, + "version" => \$::opt_version, + "verbose|t" => \$::opt_t, + "show-limits" => \$::opt_show_limits, + "exit|x" => \$::opt_x, + "max-procs|P=s" => \$::opt_P) || die_usage(); # Defaults: $Global::debug = 0; @@ -549,11 +568,11 @@ $Global::replacestring = '{}'; $Global::input_is_filename = (@ARGV); $/="\n"; -$Global::debug = (defined $::opt_d); -if(defined $::opt_x) { $Global::xargs = 1; } +$Global::debug = (defined $::opt_D); +if(defined $::opt_m) { $Global::xargs = 1; } if(defined $::opt_X) { $Global::Xargs = 1; } if(defined $::opt_v) { $Global::verbose = 1; } -if(defined $::opt_s) { $Global::verbose = 0; } +if(defined $::opt_silent) { $Global::verbose = 0; } if(defined $::opt_k) { $Global::keeporder = 1; } if(defined $::opt_g) { $Global::grouped = 1; } if(defined $::opt_u) { $Global::grouped = 0; } @@ -1244,6 +1263,10 @@ sub my_dump { # Keep perl -w happy $main::opt_u = $main::opt_c = $main::opt_f = $main::opt_q = $main::opt_0 = $main::opt_s = $main::opt_v = $main::opt_g = -$main::opt_j = $main::opt_d = $main::opt_x = $main::opt_X = -$main::opt_k = +$main::opt_j = $main::opt_D = $main::opt_m = $main::opt_X = +$main::opt_x = +$main::opt_k = $main::opt_d = $main::opt_P = $main::opt_i = +$main::opt_p = $main::opt_a = $main::opt_version = $main::opt_L = +$main::opt_l = $main::opt_show_limits = $main::opt_n = $main::opt_e = +$main::opt_t = $main::opt_E = $main::opt_r = $main::opt_help = $Global::xargs = $Global::keeporder = 0; diff --git a/parallel.1 b/parallel.1 index 89bc736a..e3d9b894 100644 --- a/parallel.1 +++ b/parallel.1 @@ -124,7 +124,7 @@ .\" ======================================================================== .\" .IX Title "PARALLEL 1" -.TH PARALLEL 1 "2010-01-29" "perl v5.10.1" "User Contributed Perl Documentation" +.TH PARALLEL 1 "2010-02-03" "perl v5.10.1" "User Contributed Perl Documentation" .\" For nroff, turn off justification. Always turn off hyphenation; it makes .\" way too many mistakes in technical documents. .if n .ad l @@ -199,8 +199,8 @@ Quote \fBcommand\fR. This will quote the command line so special characters are not interpreted by the shell. See the section \&\s-1QUOTING\s0. Most people will never need this. Quoting is disabled by default. -.IP "\fB\-s\fR" 9 -.IX Item "-s" +.IP "\fB\-\-silent\fR" 9 +.IX Item "--silent" Silent. The job to be run will not be printed. This is the default. Can be reversed with \fB\-v\fR. .IP "\fB\-u\fR" 9 @@ -209,15 +209,15 @@ Ungroup output. Output is printed as soon as possible. This may cause output from different commands to be mixed. Can be reversed with \fB\-g\fR. .IP "\fB\-v\fR" 9 .IX Item "-v" -Verbose. Print the job to be run. Can be reversed with \fB\-s\fR. -.IP "\fB\-x\fR" 9 -.IX Item "-x" -xargs. Insert as many arguments as the command line length permits. If +Verbose. Print the job to be run on standard output. Can be reversed with \fB\-s\fR. +.IP "\fB\-m\fR" 9 +.IX Item "-m" +Multiple. Insert as many arguments as the command line length permits. If {} is not used the arguments will be appended to the line. If {} is used multiple times each {} will be replaced with all the arguments. .IP "\fB\-X\fR" 9 .IX Item "-X" -xargs with context replace. This works like \fB\-x\fR except if {} is part +xargs with context replace. This works like \fB\-m\fR except if {} is part of a word (like \fIpic{}.jpg\fR) then the whole word will be repeated. .SH "EXAMPLE 1: Working as cat | sh. Ressource inexpensive jobs and evaluation" .IX Header "EXAMPLE 1: Working as cat | sh. Ressource inexpensive jobs and evaluation" @@ -413,7 +413,7 @@ easier just to write a small script and have \fBparallel\fR call that script. .SH "LIST RUNNING JOBS" .IX Header "LIST RUNNING JOBS" -To list the jobs currently running you can run: +If you want a list of the jobs currently running you can run: .PP \&\fBkillall \-USR1 parallel\fR .PP @@ -422,9 +422,9 @@ To list the jobs currently running you can run: .IX Header "COMPLETE RUNNING JOBS BUT DO NOT START NEW JOBS" If you regret starting a lot of jobs you can simply break \fBparallel\fR, but if you want to make sure you do not have halfcompleted jobs you -should send the signal \fB\-USR2\fR to \fBparallel\fR: +should send the signal \fB\s-1SIGTERM\s0\fR to \fBparallel\fR: .PP -\&\fBkillall \-USR2 parallel\fR +\&\fBkillall \-TERM parallel\fR .PP This will tell \fBparallel\fR to not start any new jobs, but wait until the currently running jobs are finished. @@ -443,6 +443,8 @@ hosts or URLs) will require creating these inputs as files. \fBfind touch important_file touch 'not important_file' ls not* | xargs rm +mkdir \-p '12" records' +ls | xargs rmdir .PP You can specify \fB\-0\fR or \fB\-d \*(L"\en\*(R"\fR, but many input generators are not optimized for using \fB\s-1NUL\s0\fR as separator but are optimized for @@ -513,22 +515,32 @@ Implement the missing \-\-features monitor to see which jobs are currently running http://code.google.com/p/ppss/ .PP -Accept signal \s-1INT\s0 to complete current running jobs but do not start -new jobs. Print out the number of jobs waiting to complete on -\&\s-1STDERR\s0. Accept sig \s-1INT\s0 again to kill now. This seems to be hard, as -all foreground processes get the \s-1INT\s0 from the shell. +Accept signal \s-1INT\s0 instead of \s-1TERM\s0 to complete current running jobs but +do not start new jobs. Print out the number of jobs waiting to +complete on \s-1STDERR\s0. Accept sig \s-1INT\s0 again to kill now. This seems to be +hard, as all foreground processes get the \s-1INT\s0 from the shell. +.PP +If there are nomore jobs (\s-1STDIN\s0 is closed) then make sure to +distribute the arguments evenly if running \-X. .PP Distibute jobs to computers with different speeds/no_of_cpu using ssh ask the computers how many cpus they have and spawn appropriately -according to \-j setting. +according to \-j setting. Reuse ssh connection (\-M and \-S) http://www.semicomplete.com/blog/geekery/distributed\-xargs.html?source=rss20 +http://code.google.com/p/ppss/wiki/Manual2 .SS "\-S" .IX Subsection "-S" \&\-S sshlogin[,sshlogin] .PP sshlogin is [user@]host or filename with list of sshlogin .PP -What about copying data to/from remote host? +What about copying data to remote host? Have an option that says the +argument is a file that should be copied. +.PP +What about copying data from remote host? Have an option that says +the argument is a file that should be copied. +.PP +Where will '>' be run? Local or remote? .PP Parallelize so this can be done: mdm.screen find dir \-execdir mdm-run cmd {} \e; @@ -553,7 +565,7 @@ Copyright (C) 2007\-2009 Free Software Foundation, Inc. This program is free software; you can redistribute it and/or modify it under the terms of the \s-1GNU\s0 General Public License as published by the Free Software Foundation; either version 3 of the License, or -(at your option) any later version. +at your option any later version. .PP This program is distributed in the hope that it will be useful, but \s-1WITHOUT\s0 \s-1ANY\s0 \s-1WARRANTY\s0; without even the implied warranty of @@ -568,4 +580,4 @@ along with this program. If not, see . Symbol, IO::File, \s-1POSIX\s0, and File::Temp. .SH "SEE ALSO" .IX Header "SEE ALSO" -\&\fBfind\fR(1), \fBxargs\fR(1) +\&\fBfind\fR(1), \fBxargs\fR(1), ' diff --git a/unittest/tests-to-run/test10.sh b/unittest/tests-to-run/test10.sh index e3a25bad..deb95e30 100644 --- a/unittest/tests-to-run/test10.sh +++ b/unittest/tests-to-run/test10.sh @@ -5,11 +5,11 @@ seq 1 40 | parallel -j 0 seq 1 10 | sort |md5sum seq 1 40 | parallel -j 0 seq 1 10 '| parallel -j 3 echo' | sort |md5sum # Test of xargs -seq 1 60000 | parallel -x echo | mop -d 4 "|sort |md5sum" "| wc" -(echo foo;echo bar) | parallel -x echo 1{}2{}3 A{}B{}C +seq 1 60000 | parallel -m echo | mop -d 4 "|sort |md5sum" "| wc" +(echo foo;echo bar) | parallel -m echo 1{}2{}3 A{}B{}C (echo foo;echo bar) | parallel -X echo 1{}2{}3 A{}B{}C -seq 1 60000 | parallel -x echo a{}b{}c | mop -d 4 "|sort |md5sum" "| wc" -seq 1 60000 | parallel -x echo a{}b{}c | \ +seq 1 60000 | parallel -m echo a{}b{}c | mop -d 4 "|sort |md5sum" "| wc" +seq 1 60000 | parallel -m echo a{}b{}c | \ mop -q "|sort |md5sum" :parallel echo -n "Chars per line: " CHAR=$(cat ~/.mop/:parallel | wc -c) diff --git a/unittest/tests-to-run/test14.sh b/unittest/tests-to-run/test14.sh index 8edd41d2..10ddd08a 100644 --- a/unittest/tests-to-run/test14.sh +++ b/unittest/tests-to-run/test14.sh @@ -5,9 +5,9 @@ seq 1 10 | parallel -k 'seq 1 {} | parallel -k -I :: echo {} ::' seq 1 10 | parallel -k 'seq 1 {} | parallel -X -k -I :: echo a{} b::' -seq 1 10 | parallel -k 'seq 1 {} | parallel -x -k -I :: echo a{} b::' +seq 1 10 | parallel -k 'seq 1 {} | parallel -m -k -I :: echo a{} b::' -seq 1 60000 | parallel -I :: -x echo a::b::c | \ +seq 1 60000 | parallel -I :: -m echo a::b::c | \ mop -q "|sort |md5sum" :parallel echo -n "Chars per line: " CHAR=$(cat ~/.mop/:parallel | wc -c)