tange/parallel
1
0
Fork 0
mirror of https://git.savannah.gnu.org/git/parallel.git synced 2024-11-25 23:47:53 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

parallel: ::: and :::: can now be mixed. Test suite passes.

Browse source
This commit is contained in:
Ole Tange 2011-05-05 18:26:29 +02:00
parent 1f3af83d66
commit 0f7fdcdffb
2 changed files with 84 additions and 88 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

146
src/parallel
View file

@ -87,7 +87,7 @@ if($::opt_halt_on_error) {
sub spreadstdin { sub spreadstdin {
# read a record # read a record
# print it to the first jobs that is ready # Spawn a job and print the record to it.
my $record; my $record;
my $buf = ""; my $buf = "";
my ($recstart,$recend,$recerror); my ($recstart,$recend,$recerror);
@ -171,6 +171,9 @@ sub spreadstdin {
} }
sub nindex { sub nindex {
# See if string is in buffer N times
# Returns:
# the position where the Nth copy is found
my $buf_ref = shift; my $buf_ref = shift;
my $str = shift; my $str = shift;
my $n = shift; my $n = shift;
@ -183,6 +186,8 @@ sub nindex {
} }
sub flush_and_close_pipes { sub flush_and_close_pipes {
# Flush that that is cached to the open pipes
# and close them.
my $flush_done; my $flush_done;
my $sleep = 0.05; my $sleep = 0.05;
do { do {
@ -263,7 +268,7 @@ sub write_record_to_pipe {
sub acquire_semaphore { sub acquire_semaphore {
# Acquires semaphore. If needed: spawns to the background # Acquires semaphore. If needed: spawns to the background
# Returns: # Returns:
# The semaphore to be released when jobs is complete # The semaphore to be released when jobs is complete
$Global::host{':'} = SSHLogin->new(":"); $Global::host{':'} = SSHLogin->new(":");
my $sem = Semaphore->new($Semaphore::name,$Global::host{':'}->max_jobs_running()); my $sem = Semaphore->new($Semaphore::name,$Global::host{':'}->max_jobs_running());
$sem->acquire(); $sem->acquire();
@ -530,16 +535,11 @@ sub parse_options {
} }
%Global::replace_rev = reverse %Global::replace; %Global::replace_rev = reverse %Global::replace;
if(grep /^$Global::arg_sep$/o, @ARGV) { if(grep /^$Global::arg_sep$|^$Global::arg_file_sep$/o, @ARGV) {
# Deal with ::: # Deal with ::: and ::::
@ARGV=read_args_from_command_line(); @ARGV=read_args_from_command_line();
} }
if(grep /^$Global::arg_file_sep$/o, @ARGV) {
# Deal with ::::
@ARGV=convert_argfiles_from_command_line_to_multiple_opt_a();
}
# Semaphore defaults # Semaphore defaults
# Must be done before computing number of processes and max_line_length # Must be done before computing number of processes and max_line_length
# because when running as a semaphore GNU Parallel does not read args # because when running as a semaphore GNU Parallel does not read args
@ -655,88 +655,77 @@ sub read_options {
} }
sub read_args_from_command_line { sub read_args_from_command_line {
# Arguments given on the command line after ::: ($Global::arg_sep) # Arguments given on the command line after:
# Removes the arguments from @ARGV and puts it into the argument queue # ::: ($Global::arg_sep)
# Ignore STDIN by reading from /dev/null # :::: ($Global::arg_file_sep)
# or another file if user has given --arg-file # Removes the arguments from @ARGV and:
# - puts filenames into -a
# - puts arguments into files and add the files to -a
# Returns: # Returns:
# @ARGV without ::: and following args # @ARGV without ::: and :::: and following args
if(not @::opt_a) { push @::opt_a, "/dev/null"; } # Input: @ARGV = command option ::: arg arg arg :::: argfiles
# Input: @ARGV = command option ::: arg arg arg
my @new_argv = (); my @new_argv = ();
while(@ARGV) { for(my $arg = shift @ARGV; @ARGV; $arg = shift @ARGV) {
my $arg = shift @ARGV; if($arg eq $Global::arg_sep
if($arg eq $Global::arg_sep) { or
my $prepend=""; $arg eq $Global::arg_file_sep) {
while(@ARGV) { my $group = $arg; # This group of arguments is args or argfiles
my $arg = shift @ARGV; my @group;
if($Global::ignore_empty) { while(defined ($arg = shift @ARGV)) {
if($arg =~ /^\s*$/) { next; } if($arg eq $Global::arg_sep
} or
if($Global::end_of_file_string and $arg eq $Global::arg_file_sep) {
$arg eq $Global::end_of_file_string) { # exit while loop if finding new separator
# Ignore the rest of ARGV last;
@ARGV=(); } else {
if(defined $prepend) { # If not hitting ::: or ::::
push(@Global::unget_argv, [Arg->new($prepend)]); # Append it to the group
} push @group, $arg;
last; }
} }
if(defined $prepend) { if($group eq $Global::arg_sep) {
$arg = $prepend.$arg; # For line continuation # Group of arguments on the command line.
$prepend = undef; #undef; # Put them into a file.
} # Create argfile
if($Global::max_lines) { my ($outfh,$name) = ::tempfile(SUFFIX => ".arg", DELETE => 1);
if($arg =~ /\s$/) { # Put args into argfile
# Trailing space => continued on next line print $outfh map { $_,$/ } @group;
$prepend = $arg; seek $outfh, 0, 0;
redo; # Append filehandle to -a
} push @::opt_a, $outfh;
} } elsif($group eq $Global::arg_file_sep) {
push(@Global::unget_argv, [Arg->new($arg)]); # Group of file names on the command line.
} # Append args into -a
last; push @::opt_a, @group;
} else { } else {
push @new_argv, $arg; ::die_bug("Unknown command line group: $group");
} }
if(defined($arg)) {
# $arg is ::: or ::::
redo;
} else {
# $arg is undef -> @ARGV empty
last;
}
}
push @new_argv, $arg;
} }
# Output: @ARGV = command option # Output: @ARGV = command to run with options
return @new_argv;
}
sub convert_argfiles_from_command_line_to_multiple_opt_a {
# Convert :::: to multiple -a
# Remove :::: from @ARGV and move the following arguments to @::opt_a
# Returns:
# @ARGV without :::: and following args
my @new_argv = ();
my @argument_files;
while(@ARGV) {
my $arg = shift @ARGV;
# Ftq rudef oaawuq ime dqxqmeqp az 2011-01-24 mzp ime iaz nk
# MQhmd Mdzrvadp Nvmdymeaz az 2011-04-09. Ftue oaawuq ue m nuf
# tmdpqd me kag tmhq fa geq daf14. Bxqmeq qymux
# oaawuq2@fmzsq.pw itqz kag dqmp ftue.
if($arg eq $Global::arg_file_sep) {
@argument_files = @ARGV;
@ARGV=();
} else {
push @new_argv, $arg;
}
}
# Output: @ARGV = command option
push @::opt_a, @argument_files;
return @new_argv; return @new_argv;
} }
sub open_or_exit { sub open_or_exit {
# Returns: # Returns:
# file handle to read-opened file # file handle to read-opened file
# exits if file cannot be opened otherwise # exits if file cannot be opened
my $file = shift; my $file = shift;
if($file eq "-") { if($file eq "-") {
return ($Global::original_stdin || *STDIN); return ($Global::original_stdin || *STDIN);
} }
if(ref $file eq "GLOB") {
# This is an open filehandle
return $file;
}
my $fh = gensym; my $fh = gensym;
if(not open($fh,"<",$file)) { if(not open($fh,"<",$file)) {
print STDERR "$Global::progname: ". print STDERR "$Global::progname: ".
@ -758,7 +747,6 @@ sub cleanup {
# Generating the command line # Generating the command line
# #
sub shell_quote { sub shell_quote {
my @strings = (@_); my @strings = (@_);
for my $a (@strings) { for my $a (@strings) {

26
src/parallel.pod
View file

@ -9,9 +9,8 @@ parallel - build and execute shell command lines from standard input in parallel
B<parallel> [options] [I<command> [arguments]] < list_of_arguments B<parallel> [options] [I<command> [arguments]] < list_of_arguments
B<parallel> [options] [I<command> [arguments]] B<:::> arguments B<parallel> [options] [I<command> [arguments]] ( B<:::> arguments |
B<::::> argfile(s) ) ...
B<parallel> [options] [I<command> [arguments]] B<::::> argfile(s)
B<parallel> --semaphore [options] I<command> B<parallel> --semaphore [options] I<command>
@ -181,8 +180,17 @@ argument separator to something else. See also B<--arg-sep>.
stdin (standard input) will be passed to the first process run. stdin (standard input) will be passed to the first process run.
If B<--arg-file> is set arguments from that file will be appended. If multiple B<:::> are given, all combinations of group of arguments
will be generated. E.g. ::: 1 2 ::: a b c will result in the
combinations (1,a) (1,b) (1,c) (2,a) (2,b) (2,c). This is useful for
replacing nested for-loops.
B<:::> and B<::::> can be mixed. So these are equivalent:
parallel echo {1} {2} {3} :::: <(seq 6 7) ::: 4 5 ::: 1 2 3
parallel echo {1} {2} {3} :::: <(seq 6 7) <(seq 4 5) :::: <(seq 1 3)
parallel -a <(seq 6 7) echo {1} {2} {3} :::: <(seq 4 5) :::: <(seq 1 3)
parallel -a <(seq 6 7) -a <(seq 4 5) echo {1} {2} {3} ::: 1 2 3
=item B<::::> I<argfiles> =item B<::::> I<argfiles>
@ -208,8 +216,6 @@ Read items from the file I<input-file> instead of stdin (standard input). If
you use this option, stdin is given to the first process run. you use this option, stdin is given to the first process run.
Otherwise, stdin is redirected from /dev/null. Otherwise, stdin is redirected from /dev/null.
(Below is unimplemented. Acts like --xapply now)
If multiple B<-a> are given, all combinations of all lines will be If multiple B<-a> are given, all combinations of all lines will be
generated. E.g. 1 2 and a b c will result in the combinations (1,a) generated. E.g. 1 2 and a b c will result in the combinations (1,a)
(1,b) (1,c) (2,a) (2,b) (2,c). This is useful for replacing nested (1,b) (1,c) (2,a) (2,b) (2,c). This is useful for replacing nested
@ -540,13 +546,15 @@ This makes it possible to change the number of simultaneous running
jobs while GNU B<parallel> is running. jobs while GNU B<parallel> is running.
=item B<--keeporder> =item B<--keep-order>
=item B<-k> =item B<-k>
Keep sequence of output same as the order of input. If jobs 1 2 3 4 Keep sequence of output same as the order of input. Try this to see
end in the sequence 3 1 4 2 the output will still be 1 2 3 4. the difference:
parallel -j4 sleep {}\; echo {} ::: 2 1 4 3
parallel -j4 -k sleep {}\; echo {} ::: 2 1 4 3
=item B<-L> I<max-lines> =item B<-L> I<max-lines>