From dbbd5829fe2ad059fac02b133d9002ebe275053b Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Tue, 9 Dec 2014 06:23:37 +0100 Subject: [PATCH] parallel: Fixed {= {= =} =}, {= =}=}, {={= =} --- doc/release_new_version | 25 +++-------- src/Makefile.in | 2 +- src/parallel | 43 ++++++++++++++++++- src/parallel.pod | 2 +- src/parallel_tutorial.1 | 2 +- testsuite/tests-to-run/parallel-local-0.3s.sh | 19 ++++++-- testsuite/wanted-results/parallel-local-0.3s | 30 +++++++++++-- 7 files changed, 91 insertions(+), 32 deletions(-) diff --git a/doc/release_new_version b/doc/release_new_version index c31fe174..fc679ba0 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -226,34 +226,21 @@ cc:Tim Cuthbertson , Ryoichiro Suzuki , Jesse Alama -Subject: GNU Parallel 20141122 ('Rosetta') released +Subject: GNU Parallel 20141222 ('Manila') released -GNU Parallel 20141122 ('Rosetta') has been released. It is available for download at: http://ftp.gnu.org/gnu/parallel/ +GNU Parallel 20141222 ('Manila') has been released. It is available for download at: http://ftp.gnu.org/gnu/parallel/ Haiku of the month: -Hadoop bit too much? -Want a simpler syntax now? -Use GNU Parallel. - -- Ole Tange - -A central piece of command generation was rewritten making this release beta quality. As always it passes the testsuite, so most functionality clearly works. +<<>> New in this release: -* Remote systems can be divided into hostgroups (e.g. web and db) by prepending '@groupname/' to the sshlogin. Multiple groups can be given by separating groups with '+'. E.g. @web/www1 @web+db/www2 @db/mariadb +* GNU Parallel was cited in: Parallel post-processing with MPI-Bash http://dl.acm.org/citation.cfm?id=2691137 -* Remote execution can be restricted to servers that are part of one or more groups by '@groupname' as an sshlogin. Multiple groups can be given by separating groups with '+'. E.g. -S @web or -S @db+web +* GNU Parallel: Open Source For You (OSFY) magazine, October 2013 edition http://www.shakthimaan.com/posts/2014/11/27/gnu-parallel/news.html -* With --hostgroup you can restrict arguments to certain hostgroups by appending '@groupname' to the argument. Multiple groups can be given by separating groups with '+'. E.g. my_web_arg@web db-or-web-arg@db+web db-only-arg@db Thanks to Michel Courtine for developing a prototype for this. - -* GNU Parallel was cited in: HTSeq-Hadoop: Extending HTSeq for Massively Parallel Sequencing Data Analysis using Hadoop http://essenceofescience.se/wp-content/uploads/2014/11/Siretskiy.pdf - -* GNU Parallel was cited in: SlideToolkit: An Assistive Toolset for the Histological Quantification of Whole Slide Images http://www.plosone.org/article/info%3Adoi%2F10.1371%2Fjournal.pone.0110289#close - -* GNU Parallel was cited in: Exploring a multiprocessor design space to analyze the impact of using STT-RAM in the memory hierarchy http://conservancy.umn.edu/bitstream/handle/11299/167286/Borse_umn_0130M_15431.pdf - -* Command-Line OCR with Tesseract on Mac OS X https://ryanfb.github.io/etc/2014/11/13/command_line_ocr_on_mac_os_x.html +* コマンドを並列に実行するGNU parallelがとても便利 http://bicycle1885.hatenablog.com/entry/2014/08/10/143612 * Bug fixes and man page updates. diff --git a/src/Makefile.in b/src/Makefile.in index ba2715f3..16f9df9d 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -78,7 +78,7 @@ NORMAL_UNINSTALL = : PRE_UNINSTALL = : POST_UNINSTALL = : subdir = src -DIST_COMMON = $(srcdir)/Makefile.in $(srcdir)/Makefile.am README +DIST_COMMON = $(srcdir)/Makefile.in $(srcdir)/Makefile.am ACLOCAL_M4 = $(top_srcdir)/aclocal.m4 am__aclocal_m4_deps = $(top_srcdir)/configure.ac am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \ diff --git a/src/parallel b/src/parallel index 77a5e131..bc0e7f9e 100755 --- a/src/parallel +++ b/src/parallel @@ -781,7 +781,7 @@ sub get_options_from_array { sub parse_options { # Returns: N/A # Defaults: - $Global::version = 20141123; + $Global::version = 20141209; $Global::progname = 'parallel'; $Global::infinity = 2**31; $Global::debug = 0; @@ -6865,6 +6865,7 @@ sub new { } } # Replace replacement strings with {= perl expr =} + @command = merge_rpl_parts(@command); # Protect matching inside {= perl expr =} # by replacing {= and =} with \257< and \257> for(@command) { @@ -6872,7 +6873,10 @@ sub new { ::error("Command cannot contain the character \257. Use a function for that.\n"); ::wait_and_exit(255); } - s/\Q$Global::parensleft\E(.*?)\Q$Global::parensright\E/\257<$1\257>/gx; + # Needs to match rightmost left parens (Perl defaults to leftmost) + # to deal with: {={==} + # Disallow \257 to avoid nested {= {= =} =} + while(s/([^\257]*) \Q$Global::parensleft\E ([^\257]*?) \Q$Global::parensright\E /$1\257<$2\257>/gx) {} } for my $rpl (keys %Global::rpl) { # Replace the short hand string with the {= perl expr =} in $command and $opt::tagstring @@ -6979,6 +6983,41 @@ sub new { }, ref($class) || $class; } +sub merge_rpl_parts { + # '{=' 'perlexpr' '=}' => '{= perlexpr =}' + # Input: + # @in = the @command as given by the user + # Uses: + # $Global::parensleft + # $Global::parensright + # Returns: + # @command with parts merged to keep {= and =} as one + my @in = @_; + my @out; + my $l = quotemeta($Global::parensleft); + my $r = quotemeta($Global::parensright); + + while(@in) { + my $s = shift @in; + $_ = $s; + # Remove matching (right most) parens + while(s/(.*)$l.*?$r/$1/o) {} + if(/$l/o) { + # Missing right parens + while(@in) { + $s .= " ".shift @in; + $_ = $s; + while(s/(.*)$l.*?$r/$1/o) {} + if(not /$l/o) { + last; + } + } + } + push @out, $s; + } + return @out; +} + sub get { my $self = shift; if(@{$self->{'unget'}}) { diff --git a/src/parallel.pod b/src/parallel.pod index 601d9eca..201762ed 100644 --- a/src/parallel.pod +++ b/src/parallel.pod @@ -3058,7 +3058,7 @@ To submit your jobs to the queue: You can of course use B<-S> to distribute the jobs to remote computers: - true >jobqueue; tail -f jobqueue | parallel -S .. + true >jobqueue; tail -n+0 -f jobqueue | parallel -S .. There is a a small issue when using GNU B as queue system/batch manager: You have to submit JobSlot number of jobs before diff --git a/src/parallel_tutorial.1 b/src/parallel_tutorial.1 index 219a35bb..788dbcae 100644 --- a/src/parallel_tutorial.1 +++ b/src/parallel_tutorial.1 @@ -133,7 +133,7 @@ .\" ======================================================================== .\" .IX Title "PARALLEL_TUTORIAL 1" -.TH PARALLEL_TUTORIAL 1 "2014-11-10" "20141022" "parallel" +.TH PARALLEL_TUTORIAL 1 "2014-11-26" "20141122" "parallel" .\" For nroff, turn off justification. Always turn off hyphenation; it makes .\" way too many mistakes in technical documents. .if n .ad l diff --git a/testsuite/tests-to-run/parallel-local-0.3s.sh b/testsuite/tests-to-run/parallel-local-0.3s.sh index 613efe9d..74742fb7 100644 --- a/testsuite/tests-to-run/parallel-local-0.3s.sh +++ b/testsuite/tests-to-run/parallel-local-0.3s.sh @@ -23,11 +23,22 @@ echo '### Test bug #43284: {%} and {#} with --xapply'; echo '**' echo '### Test bug #43376: {%} and {#} with --pipe' -echo foo | parallel -q --pipe -k echo {#} -echo foo | parallel --pipe -k echo {%} -echo foo | parallel -q --pipe -k echo {%} -echo foo | parallel --pipe -k echo {#} + echo foo | parallel -q --pipe -k echo {#} + echo foo | parallel --pipe -k echo {%} + echo foo | parallel -q --pipe -k echo {%} + echo foo | parallel --pipe -k echo {#} echo '**' +echo '### {= and =} in different groups' + parallel echo {= s/a/b/ =} ::: a + parallel echo {= s/a/b/=} ::: a + parallel echo {= s/a/b/=}{= s/a/b/=} ::: a + parallel echo {= s/a/b/=}{=s/a/b/=} ::: a + parallel echo {= s/a/b/=}{= {= s/a/b/=} ::: a + parallel echo {= s/a/b/=}{={=s/a/b/=} ::: a + parallel echo {= s/a/b/ =} {={==} ::: a + parallel echo {={= =} ::: a + parallel echo {= {= =} ::: a + parallel echo {= {= =} =} ::: a EOF diff --git a/testsuite/wanted-results/parallel-local-0.3s b/testsuite/wanted-results/parallel-local-0.3s index beaa6450..3459a5b9 100644 --- a/testsuite/wanted-results/parallel-local-0.3s +++ b/testsuite/wanted-results/parallel-local-0.3s @@ -16,13 +16,35 @@ echo '**' ** echo '### Test bug #43376: {%} and {#} with --pipe' ### Test bug #43376: {%} and {#} with --pipe -echo foo | parallel -q --pipe -k echo {#} + echo foo | parallel -q --pipe -k echo {#} 1 -echo foo | parallel --pipe -k echo {%} + echo foo | parallel --pipe -k echo {%} 1 -echo foo | parallel -q --pipe -k echo {%} + echo foo | parallel -q --pipe -k echo {%} 1 -echo foo | parallel --pipe -k echo {#} + echo foo | parallel --pipe -k echo {#} 1 echo '**' ** +echo '### {= and =} in different groups' +### {= and =} in different groups + parallel echo {= s/a/b/ =} ::: a +b + parallel echo {= s/a/b/=} ::: a +b + parallel echo {= s/a/b/=}{= s/a/b/=} ::: a +bb + parallel echo {= s/a/b/=}{=s/a/b/=} ::: a +bb + parallel echo {= s/a/b/=}{= {= s/a/b/=} ::: a +b{= b + parallel echo {= s/a/b/=}{={=s/a/b/=} ::: a +b{=b + parallel echo {= s/a/b/ =} {={==} ::: a +b {=a + parallel echo {={= =} ::: a +{=a + parallel echo {= {= =} ::: a +{= a + parallel echo {= {= =} =} ::: a +{= a =}