tange/parallel
1
0
Fork 0
mirror of https://git.savannah.gnu.org/git/parallel.git synced 2024-10-18 12:59:39 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

parallel: ::: and :::: can now be mixed. Test suite passes.

Browse source
This commit is contained in:
Ole Tange 2011-05-05 18:26:29 +02:00
parent 1f3af83d66
commit 0f7fdcdffb
2 changed files with 84 additions and 88 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

146
src/parallel
View file

@ -87,7 +87,7 @@ if($::opt_halt_on_error) {
sub spreadstdin {
# read a record
# print it to the first jobs that is ready
# Spawn a job and print the record to it.
my $record;
my $buf = "";
my ($recstart,$recend,$recerror);
@ -171,6 +171,9 @@ sub spreadstdin {
}
sub nindex {
# See if string is in buffer N times
# Returns:
# the position where the Nth copy is found
my $buf_ref = shift;
my $str = shift;
my $n = shift;
@ -183,6 +186,8 @@ sub nindex {
}
sub flush_and_close_pipes {
# Flush that that is cached to the open pipes
# and close them.
my $flush_done;
my $sleep = 0.05;
do {
@ -263,7 +268,7 @@ sub write_record_to_pipe {
sub acquire_semaphore {
# Acquires semaphore. If needed: spawns to the background
# Returns:
# The semaphore to be released when jobs is complete
# The semaphore to be released when jobs is complete
$Global::host{':'} = SSHLogin->new(":");
my $sem = Semaphore->new($Semaphore::name,$Global::host{':'}->max_jobs_running());
$sem->acquire();
@ -530,16 +535,11 @@ sub parse_options {
}
%Global::replace_rev = reverse %Global::replace;
if(grep /^$Global::arg_sep$/o, @ARGV) {
# Deal with :::
if(grep /^$Global::arg_sep$|^$Global::arg_file_sep$/o, @ARGV) {
# Deal with ::: and ::::
@ARGV=read_args_from_command_line();
}
if(grep /^$Global::arg_file_sep$/o, @ARGV) {
# Deal with ::::
@ARGV=convert_argfiles_from_command_line_to_multiple_opt_a();
}
# Semaphore defaults
# Must be done before computing number of processes and max_line_length
# because when running as a semaphore GNU Parallel does not read args
@ -655,88 +655,77 @@ sub read_options {
}
sub read_args_from_command_line {
# Arguments given on the command line after ::: ($Global::arg_sep)
# Removes the arguments from @ARGV and puts it into the argument queue
# Ignore STDIN by reading from /dev/null
# or another file if user has given --arg-file
# Arguments given on the command line after:
# ::: ($Global::arg_sep)
# :::: ($Global::arg_file_sep)
# Removes the arguments from @ARGV and:
# - puts filenames into -a
# - puts arguments into files and add the files to -a
# Returns:
# @ARGV without ::: and following args
if(not @::opt_a) { push @::opt_a, "/dev/null"; }
# Input: @ARGV = command option ::: arg arg arg
# @ARGV without ::: and :::: and following args
# Input: @ARGV = command option ::: arg arg arg :::: argfiles
my @new_argv = ();
while(@ARGV) {
my $arg = shift @ARGV;
if($arg eq $Global::arg_sep) {
my $prepend="";
while(@ARGV) {
my $arg = shift @ARGV;
if($Global::ignore_empty) {
if($arg =~ /^\s*$/) { next; }
}
if($Global::end_of_file_string and
$arg eq $Global::end_of_file_string) {
# Ignore the rest of ARGV
@ARGV=();
if(defined $prepend) {
push(@Global::unget_argv, [Arg->new($prepend)]);
}
last;
}
if(defined $prepend) {
$arg = $prepend.$arg; # For line continuation
$prepend = undef; #undef;
}
if($Global::max_lines) {
if($arg =~ /\s$/) {
# Trailing space => continued on next line
$prepend = $arg;
redo;
}
}
push(@Global::unget_argv, [Arg->new($arg)]);
}
last;
} else {
push @new_argv, $arg;
}
for(my $arg = shift @ARGV; @ARGV; $arg = shift @ARGV) {
if($arg eq $Global::arg_sep
or
$arg eq $Global::arg_file_sep) {
my $group = $arg; # This group of arguments is args or argfiles
my @group;
while(defined ($arg = shift @ARGV)) {
if($arg eq $Global::arg_sep
or
$arg eq $Global::arg_file_sep) {
# exit while loop if finding new separator
last;
} else {
# If not hitting ::: or ::::
# Append it to the group
push @group, $arg;
}
}
if($group eq $Global::arg_sep) {
# Group of arguments on the command line.
# Put them into a file.
# Create argfile
my ($outfh,$name) = ::tempfile(SUFFIX => ".arg", DELETE => 1);
# Put args into argfile
print $outfh map { $_,$/ } @group;
seek $outfh, 0, 0;
# Append filehandle to -a
push @::opt_a, $outfh;
} elsif($group eq $Global::arg_file_sep) {
# Group of file names on the command line.
# Append args into -a
push @::opt_a, @group;
} else {
::die_bug("Unknown command line group: $group");
}
if(defined($arg)) {
# $arg is ::: or ::::
redo;
} else {
# $arg is undef -> @ARGV empty
last;
}
}
push @new_argv, $arg;
}
# Output: @ARGV = command option
return @new_argv;
}
sub convert_argfiles_from_command_line_to_multiple_opt_a {
# Convert :::: to multiple -a
# Remove :::: from @ARGV and move the following arguments to @::opt_a
# Returns:
# @ARGV without :::: and following args
my @new_argv = ();
my @argument_files;
while(@ARGV) {
my $arg = shift @ARGV;
# Ftq rudef oaawuq ime dqxqmeqp az 2011-01-24 mzp ime iaz nk
# MQhmd Mdzrvadp Nvmdymeaz az 2011-04-09. Ftue oaawuq ue m nuf
# tmdpqd me kag tmhq fa geq daf14. Bxqmeq qymux
# oaawuq2@fmzsq.pw itqz kag dqmp ftue.
if($arg eq $Global::arg_file_sep) {
@argument_files = @ARGV;
@ARGV=();
} else {
push @new_argv, $arg;
}
}
# Output: @ARGV = command option
push @::opt_a, @argument_files;
# Output: @ARGV = command to run with options
return @new_argv;
}
sub open_or_exit {
# Returns:
# file handle to read-opened file
# exits if file cannot be opened otherwise
# exits if file cannot be opened
my $file = shift;
if($file eq "-") {
return ($Global::original_stdin || *STDIN);
}
if(ref $file eq "GLOB") {
# This is an open filehandle
return $file;
}
my $fh = gensym;
if(not open($fh,"<",$file)) {
print STDERR "$Global::progname: ".
@ -758,7 +747,6 @@ sub cleanup {
# Generating the command line
#
sub shell_quote {
my @strings = (@_);
for my $a (@strings) {

26
src/parallel.pod
View file

@ -9,9 +9,8 @@ parallel - build and execute shell command lines from standard input in parallel
B<parallel> [options] [I<command> [arguments]] < list_of_arguments
B<parallel> [options] [I<command> [arguments]] B<:::> arguments
B<parallel> [options] [I<command> [arguments]] B<::::> argfile(s)
B<parallel> [options] [I<command> [arguments]] ( B<:::> arguments |
B<::::> argfile(s) ) ...
B<parallel> --semaphore [options] I<command>
@ -181,8 +180,17 @@ argument separator to something else. See also B<--arg-sep>.
stdin (standard input) will be passed to the first process run.
If B<--arg-file> is set arguments from that file will be appended.
If multiple B<:::> are given, all combinations of group of arguments
will be generated. E.g. ::: 1 2 ::: a b c will result in the
combinations (1,a) (1,b) (1,c) (2,a) (2,b) (2,c). This is useful for
replacing nested for-loops.
B<:::> and B<::::> can be mixed. So these are equivalent:
parallel echo {1} {2} {3} :::: <(seq 6 7) ::: 4 5 ::: 1 2 3
parallel echo {1} {2} {3} :::: <(seq 6 7) <(seq 4 5) :::: <(seq 1 3)
parallel -a <(seq 6 7) echo {1} {2} {3} :::: <(seq 4 5) :::: <(seq 1 3)
parallel -a <(seq 6 7) -a <(seq 4 5) echo {1} {2} {3} ::: 1 2 3
=item B<::::> I<argfiles>
@ -208,8 +216,6 @@ Read items from the file I<input-file> instead of stdin (standard input). If
you use this option, stdin is given to the first process run.
Otherwise, stdin is redirected from /dev/null.
(Below is unimplemented. Acts like --xapply now)
If multiple B<-a> are given, all combinations of all lines will be
generated. E.g. 1 2 and a b c will result in the combinations (1,a)
(1,b) (1,c) (2,a) (2,b) (2,c). This is useful for replacing nested
@ -540,13 +546,15 @@ This makes it possible to change the number of simultaneous running
jobs while GNU B<parallel> is running.
=item B<--keeporder>
=item B<--keep-order>
=item B<-k>
Keep sequence of output same as the order of input. If jobs 1 2 3 4
end in the sequence 3 1 4 2 the output will still be 1 2 3 4.
Keep sequence of output same as the order of input. Try this to see
the difference:
parallel -j4 sleep {}\; echo {} ::: 2 1 4 3
parallel -j4 -k sleep {}\; echo {} ::: 2 1 4 3
=item B<-L> I<max-lines>