From 1aa965256c8586adda549ffbe188d5e3e4214a95 Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Tue, 14 Oct 2014 22:56:03 +0200 Subject: [PATCH] Test of bug #43376: {%} and {#} with --pipe. --- doc/release_new_version | 4 ++ src/parallel | 43 +++++++++++-------- src/parallel.pod | 4 +- testsuite/tests-to-run/parallel-local-0.3s.sh | 8 ++++ testsuite/wanted-results/parallel-local-0.3s | 12 ++++++ 5 files changed, 50 insertions(+), 21 deletions(-) diff --git a/doc/release_new_version b/doc/release_new_version index 78eb28bc..5cc8859d 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -248,10 +248,14 @@ New in this release: * GNU Parallel was cited in: Application of Machine Learning to Algorithm Selection for TSP http://www.dbai.tuwien.ac.at/staff/musliu/art_ictai_cam.pdf +* GNU Parallel was cited in: Performance Models for LTE-Advanced Random Access http://repository.asu.edu/attachments/137242/content/Tyagi_asu_0010E_14116.pdf + * GNU Parallel was used (unfortunately without citation) in: De Novo Insertions and Deletions of Predominantly Paternal Origin Are Associated with Autism Spectrum Disorder http://www.cell.com/cms/attachment/2019079667/2039310868/mmc1.pdf * GNU Parallel is demonstrated in: Data Science at the Command Line: Facing the Future with Time-Tested Tools +* GNU Parallel was covered in a talk at Sydney Perl Mongers group http://www.openfusion.net/talks/pwp/#24 + * GNU Parallel and how to get started with it http://www.jduck.net/blog/2014/09/30/gnu-paralell/ * Comparing Golang, Scala, Elixir and Ruby for ETL http://www.dimroc.com/2014/09/29/etl-language-showdown/ diff --git a/src/parallel b/src/parallel index baeac1eb..f5447f52 100755 --- a/src/parallel +++ b/src/parallel @@ -3137,6 +3137,7 @@ sub which { ( 'aix' => $sysv, 'cygwin' => $sysv, + 'msys' => $sysv, 'dec_osf' => $sysv, 'darwin' => $bsd, 'dragonfly' => $bsd, @@ -3172,6 +3173,8 @@ sub which { sub reap_usleep { # Reap dead children. # If no dead children: Sleep specified amount with exponential backoff + # Input: + # $ms = milliseconds to sleep # Returns: # $ms/2+0.001 if children reaped # $ms*1.1 if no children reaped @@ -3198,14 +3201,19 @@ sub reap_usleep { sub usleep { # Sleep this many milliseconds. - my $secs = shift; - ::debug(int($secs),"ms "); - select(undef, undef, undef, $secs/1000); + # Input: + # $ms = milliseconds to sleep + my $ms = shift; + ::debug(int($ms),"ms "); + select(undef, undef, undef, $ms/1000); } sub now { # Returns time since epoch as in seconds with 3 decimals - + # Uses: + # @Global::use + # Returns: + # $time = time now with millisecond accuracy if(not $Global::use{"Time::HiRes"}) { if(eval "use Time::HiRes qw ( time );") { eval "sub TimeHiRestime { return Time::HiRes::time };"; @@ -3225,6 +3233,10 @@ sub multiply_binary_prefix { # K =2^10, M =2^20, G =2^30, T =2^40, P =2^50, E =2^70, Z =2^80, Y =2^80 # k =10^3, m =10^6, g =10^9, t=10^12, p=10^15, e=10^18, z=10^21, y=10^24 # 13G = 13*1024*1024*1024 = 13958643712 + # Input: + # $s = string with prefixes + # Returns: + # $value = int with prefixes multiplied my $s = shift; $s =~ s/ki/*1024/gi; $s =~ s/mi/*1024*1024/gi; @@ -3262,11 +3274,18 @@ sub multiply_binary_prefix { } sub tmpfile { + # Create tempfile as $TMPDIR/parXXXXX + # Returns: + # $filename = file name created return ::tempfile(DIR=>$ENV{'TMPDIR'}, TEMPLATE => 'parXXXXX', @_); } + sub __DEBUGGING__ {} sub debug { + # Uses: + # $Global::debug + # %Global::fd # Returns: N/A $Global::debug or return; @_ = grep { defined $_ ? $_ : "" } @_; @@ -3306,7 +3325,7 @@ sub my_memory_usage { sub my_size { # Returns: - # size of object if Devel::Size is installed + # $size = size of object if Devel::Size is installed # -1 otherwise my @size_this = (@_); eval "use Devel::Size qw(size total_size)"; @@ -6710,20 +6729,6 @@ sub replace_placeholders { CORE::push(@{$replace{$ww}}, $val); } } - if(not @arg) { - die; - # No args: We can still have {%} or {#} as replacement string. - my $val = $w; - for my $perlexpr (keys %{$self->{'replacecount'}}) { - # Replace {= perl expr =} with value for each arg - $val =~ s/\257<\Q$perlexpr\E\257>/$_="";eval("$perlexpr");$_/eg; - } - my $ww = $word; - if($quote) { - $val = ::shell_quote_scalar($val); - } - CORE::push(@{$replace{$ww}}, $val); - } } if($quote) { diff --git a/src/parallel.pod b/src/parallel.pod index 51cd3950..f4db40cb 100644 --- a/src/parallel.pod +++ b/src/parallel.pod @@ -588,7 +588,7 @@ cannot be exported: parallel --env my_importer \ 'my_importer; echo "{}" "${indexed[{}]}" "${assoc[${indexed[{}]}]}"' ::: "${!indexed[@]}" -To copy the full environment use this function: +To copy the full environment use this function (e.g. by putting it in .bashrc): env_parallel() { export parallel_bash_environment='() { @@ -2748,7 +2748,7 @@ The following will start one B per big file in I to I on the server I: B +mkdir -p /dest-dir/{//}\;rsync -s -Havessh {} fooserver:/dest-dir/{}> The dirs created may end up with wrong permissions and smaller files are not being transferred. To fix those run B a final time: diff --git a/testsuite/tests-to-run/parallel-local-0.3s.sh b/testsuite/tests-to-run/parallel-local-0.3s.sh index 3b9a1ec1..613efe9d 100644 --- a/testsuite/tests-to-run/parallel-local-0.3s.sh +++ b/testsuite/tests-to-run/parallel-local-0.3s.sh @@ -22,4 +22,12 @@ echo '### Test bug #43284: {%} and {#} with --xapply'; echo '**' +echo '### Test bug #43376: {%} and {#} with --pipe' +echo foo | parallel -q --pipe -k echo {#} +echo foo | parallel --pipe -k echo {%} +echo foo | parallel -q --pipe -k echo {%} +echo foo | parallel --pipe -k echo {#} + +echo '**' + EOF diff --git a/testsuite/wanted-results/parallel-local-0.3s b/testsuite/wanted-results/parallel-local-0.3s index 22a6cab4..beaa6450 100644 --- a/testsuite/wanted-results/parallel-local-0.3s +++ b/testsuite/wanted-results/parallel-local-0.3s @@ -14,3 +14,15 @@ a 1 1 b 1 echo '**' ** +echo '### Test bug #43376: {%} and {#} with --pipe' +### Test bug #43376: {%} and {#} with --pipe +echo foo | parallel -q --pipe -k echo {#} +1 +echo foo | parallel --pipe -k echo {%} +1 +echo foo | parallel -q --pipe -k echo {%} +1 +echo foo | parallel --pipe -k echo {#} +1 +echo '**' +**