From a63e258bb1d3057c591bb3e5586934f4b2f3e803 Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Wed, 27 Apr 2011 17:12:35 +0200 Subject: [PATCH] parallel: Implemented {//}. Converted use into conditional use (hopefully with speedup). --- doc/release_new_version | 32 ++++++--------------------- src/parallel | 37 ++++++++++++++++++++------------ src/parallel.pod | 30 ++++++++++++++++++++------ testsuite/tests-to-run/test57.sh | 15 +++++++++++++ testsuite/wanted-results/test57 | 27 +++++++++++++++++++++++ 5 files changed, 95 insertions(+), 46 deletions(-) create mode 100644 testsuite/tests-to-run/test57.sh create mode 100644 testsuite/wanted-results/test57 diff --git a/doc/release_new_version b/doc/release_new_version index c37e721e..ab00a989 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -158,38 +158,18 @@ cc:Peter Simons , Sandro Cazzaniga , Christian Faulhammer , Ryoichiro Suzuki , Jesse Alama -Subject: GNU Parallel 2011XXXX ('Libya') released +Subject: GNU Parallel 2011XXXX ('?') released -GNU Parallel 2011XXXX ('Libya') has been released. It is +GNU Parallel 2011XXXX ('?') has been released. It is available for download at: http://ftp.gnu.org/gnu/parallel/ New in this release: -* {#} now works an alias for $PARALLEL_PID. +* Using GNU Parallel with EC2. Thanks to Kevin Wu. + http://blog.kevinformatics.com/post/4970574713/interested-in-your-experience-using-gnu-parallel-in -* --eta now estimates way more accurately. - -* CPU detection code for AIX thanks to Christian Netrwal. - -* --joblog contains exitcode. - -* Thanks to Ævar Arnfjörð Bjarmason for reading my code. - -* GNU Parallel was presented at: - - LUGA, Augsburg, 2011-03-26, http://www.luga.de/Aktionen/LIT-2011/Programm - - OSAA.dk, Aarhus, 2011-04-12 - -* Blog entry in Japanese. Thanks to Riywo. - http://blog.riywo.com/2011/04/19/022802 - -* Example of how to use GNU Parallel on PBS clusters. Thanks to Andrew - J Dolgert: http://web0.tc.cornell.edu/wiki/index.php?title=Gnu_Parallel - -* First example of using GNU Parallel in Hebrew. Thanks to omry. - http://nd.gd/jk - -* Intro video got past 10000 hits. - http://www.youtube.com/watch?v=OpaiGYxkSuQ +* Review with idea for {..} and {...} in Japanese. Thanks to ichii386. + http://d.hatena.ne.jp/ichii386/20110426 * Bug fixes and man page updates. diff --git a/src/parallel b/src/parallel index 5464efe0..2500980b 100755 --- a/src/parallel +++ b/src/parallel @@ -1,14 +1,17 @@ #!/usr/bin/perl -w +# open3 used in Job::start use IPC::Open3; -use Fcntl; -use Symbol qw(gensym); -use IO::File; +# &WNOHANG used in reaper use POSIX qw(:sys_wait_h setsid ceil :errno_h); +# gensym used in Job::start +use Symbol qw(gensym); +# tempfile used in Job::start use File::Temp qw(tempfile tempdir); +# GetOptions used in get_options_from_array use Getopt::Long; +# Used to ensure code quality use strict; -use Carp; $::oodebug=0; $SIG{TERM} ||= sub { exit 0; }; # $SIG{TERM} is not set on Mac OS X @@ -309,6 +312,7 @@ sub get_options_from_array { "I=s" => \$::opt_I, "extensionreplace|U=s" => \$::opt_U, "basenamereplace|bnr=s" => \$::opt_basenamereplace, + "dirnamereplace|dnr=s" => \$::opt_dirnamereplace, "basenameextensionreplace=s" => \$::opt_basenameextensionreplace, "seqreplace=s" => \$::opt_seqreplace, "jobs|j=s" => \$::opt_P, @@ -400,6 +404,7 @@ sub parse_options { $Global::replace{'{}'} = '{}'; $Global::replace{'{.}'} = '{.}'; $Global::replace{'{/}'} = '{/}'; + $Global::replace{'{//}'} = '{//}'; $Global::replace{'{/.}'} = '{/.}'; $Global::replace{'{#}'} = '{#}'; $/="\n"; @@ -436,6 +441,7 @@ sub parse_options { $Global::replace{'{}'} = $::opt_i eq "" ? "{}" : $::opt_i; } if(defined $::opt_basenamereplace) { $Global::replace{'{/}'} = $::opt_basenamereplace; } + if(defined $::opt_dirnamereplace) { $Global::replace{'{//}'} = $::opt_dirnamereplace; } if(defined $::opt_basenameextensionreplace) { $Global::replace{'{/.}'} = $::opt_basenameextensionreplace; } @@ -2594,8 +2600,7 @@ sub complete_write { my $self = shift; my $in = $self->{'stdin'}; my $len = syswrite($in,$self->{'remaining'}); - - if (!defined($len) && $! == ::EAGAIN) { + if (!defined($len) && $! == &::EAGAIN) { # write would block; } else { # Remove the part that was written @@ -3358,7 +3363,7 @@ sub len { $self->{'replacecount'}{$replstring}; } if($Global::replace{$replstring}) { - # This is a multi replacestring ({} {/} {.} {/.}) + # This is a multi replacestring ({} {/} {.} {/.} {//}) # Add each space between two arguments my $number_of_args = ($#{$self->{'arg_list'}[0]}+1)*$self->number_of_args(); $len += ($number_of_args-1) * $self->{'replacecount'}{$replstring}; @@ -3381,6 +3386,7 @@ sub multi_regexp { ($Global::replace{"{}"}, $Global::replace{"{.}"}, $Global::replace{"{/}"}, + $Global::replace{"{//}"}, $Global::replace{"{/.}"}) ).")"; } @@ -3431,7 +3437,7 @@ sub number_of_replacements { } for my $k (keys %count) { if(defined $Global::replace{$k}) { - # {} {/} {.} {/.} {#} + # {} {/} {//} {.} {/.} {#} $context -= (length $Global::replace{$k}) * $count{$k}; } else { # {n} @@ -3489,9 +3495,9 @@ sub replace_placeholders { # we have a matching argument for {n} $replace_single{$used} = $args[$positional-1]->replace($replacementfunction); } - } elsif($used =~ /^(\{\}|\{\/\}|\{\.\}|\{\/\.\})$/) { + } elsif($used =~ /^(\{\}|\{\/\}|\{\/\/\}|\{\.\}|\{\/\.\})$/) { # Add to the multireplacement - my $replacementfunction = $used; # {} {/} {.} or {/.} + my $replacementfunction = $used; # {} {/} {//} {.} or {/.} CORE::push @used_multi, $replacementfunction; if($self->{'context_replace'}) { for my $n (0 .. $#args) { @@ -3518,7 +3524,7 @@ sub replace_placeholders { my $orig_target = $target; if(@used_multi) { my $multi_regexp = join('|', map { - $a=$Global::replace{"$_"}; + $a=$Global::replace{$_}; $a=~s/(\W)/\\$1/g; $a } @used_multi); my %wordargs; @@ -3994,6 +4000,10 @@ sub replace { $s =~ s:\.[^/\.]*$::; # Remove .ext from argument } elsif($replacement_string eq "{/}") { $s =~ s:^.*/([^/]+)/?$:$1:; # Remove dir from argument. If ending in /, remove final / + } elsif($replacement_string eq "{//}") { + # Only load File::Basename if actually needed + $Global::use{"File::Basename"} ||= eval "use File::Basename;"; + $s = dirname($s); # Keep dir from argument. } elsif($replacement_string eq "{/.}") { $s =~ s:^.*/([^/]+)/?$:$1:; # Remove dir from argument. If ending in /, remove final / $s =~ s:\.[^/\.]*$::; # Remove .ext from argument @@ -4049,8 +4059,6 @@ package Semaphore; # process holding the entry. If the process dies, the entry can be # taken by another process. -use Fcntl qw(:DEFAULT :flock); - sub new { my $class = shift; my $id = shift; @@ -4159,6 +4167,7 @@ sub lock { open $self->{'lockfh'}, ">", $self->{'lockfile'} or ::die_bug("Can't open semaphore file $self->{'lockfile'}: $!"); chmod 0666, $self->{'lockfile'}; # assuming you want it a+rw + $Global::use{"Fcntl"} ||= eval "use Fcntl qw(:DEFAULT :flock);"; while(not flock $self->{'lockfh'}, LOCK_EX()|LOCK_NB()) { ::debug("Cannot lock $self->{'lockfile'}"); # TODO if timeout: last @@ -4179,5 +4188,5 @@ sub unlock { # Keep perl -w happy $::opt_x = $::opt_workdir = $Semaphore::timeout = $Semaphore::wait = -$::opt_skip_first_line = $::opt_shebang = 0 ; +$::opt_skip_first_line = $::opt_shebang = $Global::use = 0 ; diff --git a/src/parallel.pod b/src/parallel.pod index 120fe0e2..a7069512 100644 --- a/src/parallel.pod +++ b/src/parallel.pod @@ -48,9 +48,10 @@ after the list of options. That will give you an idea of what GNU B is capable of. You can also watch the intro video for a quick introduction: -http://www.youtube.com/watch?v=OpaiGYxkSuQ or at http://tinyogg.com/watch/TORaR/ http://tinyogg.com/watch/hfxKj/ and -http://tinyogg.com/watch/YQuXd/ +http://tinyogg.com/watch/YQuXd/ or +http://www.youtube.com/watch?v=OpaiGYxkSuQ and +http://www.youtube.com/watch?v=1ntxT-47VPA =head1 OPTIONS @@ -100,6 +101,15 @@ B<{/}> can be used the same places as B<{}>. The replacement string B<{/}> can be changed with B<--basenamereplace>. +=item B<{//}> + +Dirname of input line. This is a specialized replacement string +containing the dir of the input. See B(1). + +B<{//}> can be used the same places as B<{}>. The replacement string +B<{//}> can be changed with B<--dirnamereplace>. + + =item B<{/.}> Basename of input line without extension. This is a specialized @@ -326,6 +336,14 @@ as \n, or an octal or hexadecimal escape code. Octal and hexadecimal escape codes are understood as for the printf command. Multibyte characters are not supported. +=item B<--dirnamereplace> I (alpha testing) + +=item B<--dnr> I (alpha testing) + +Use the replacement string I instead of B<{//}> for +dirname of input line. + + =item B<-E> I Set the end of file string to eof-str. If the end of file string @@ -1263,7 +1281,7 @@ solution is to quote the whole command: B>B<{.}" ::: *.gz> -Other special shell charaters (such as * ; $ > < | >> <<) also needs +Other special shell charaters (such as * ; $ > < | >> <<) also need to be put in quotes, as they may otherwise be interpreted by the shell and not given to GNU B. @@ -2878,8 +2896,8 @@ it also uses rsync with ssh. =head1 SEE ALSO -B(1), B(1), B(1), B(1), B(1), -B(1), B(1), B(1), B(1), B(1), -B(1), +B(1), B(1), B(1), B(1), B, +B(1), B(1), B(1), B(1), B(1), +B(1), B(1), =cut diff --git a/testsuite/tests-to-run/test57.sh b/testsuite/tests-to-run/test57.sh new file mode 100644 index 00000000..4d83bb65 --- /dev/null +++ b/testsuite/tests-to-run/test57.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +echo '### Test {//}' +parallel -k echo {//} {} ::: a a/b a/b/c +parallel -k echo {//} {} ::: /a /a/b /a/b/c +parallel -k echo {//} {} ::: ./a ./a/b ./a/b/c +parallel -k echo {//} {} ::: a.jpg a/b.jpg a/b/c.jpg +parallel -k echo {//} {} ::: /a.jpg /a/b.jpg /a/b/c.jpg +parallel -k echo {//} {} ::: ./a.jpg ./a/b.jpg ./a/b/c.jpg + +echo '### Test --dnr' +parallel --dnr II -k echo II {} ::: a a/b a/b/c + +echo '### Test --dirnamereplace' +parallel --dirnamereplace II -k echo II {} ::: a a/b a/b/c diff --git a/testsuite/wanted-results/test57 b/testsuite/wanted-results/test57 new file mode 100644 index 00000000..18111583 --- /dev/null +++ b/testsuite/wanted-results/test57 @@ -0,0 +1,27 @@ +### Test {//} +. a +a a/b +a/b a/b/c +/ /a +/a /a/b +/a/b /a/b/c +. ./a +./a ./a/b +./a/b ./a/b/c +. a.jpg +a a/b.jpg +a/b a/b/c.jpg +/ /a.jpg +/a /a/b.jpg +/a/b /a/b/c.jpg +. ./a.jpg +./a ./a/b.jpg +./a/b ./a/b/c.jpg +### Test --dnr +. a +a a/b +a/b a/b/c +### Test --dirnamereplace +. a +a a/b +a/b a/b/c