From dcc18d38f5443ac735cb4d30ce04dce18f6b4bb0 Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Sat, 27 Jun 2015 00:29:28 +0200 Subject: [PATCH] parallel: Fixed bug #45414: Use setpriority for --nice. --- src/parallel | 25 +++++-------------------- src/parallel.pod | 13 ++++++++----- src/parallel_design.pod | 11 ++++++++++- 3 files changed, 23 insertions(+), 26 deletions(-) diff --git a/src/parallel b/src/parallel index ab750257..6ef6c589 100755 --- a/src/parallel +++ b/src/parallel @@ -907,6 +907,7 @@ sub parse_options { if(defined $opt::max_args) { $Global::max_number_of_args = $opt::max_args; } if(defined $opt::timeout) { $Global::timeoutq = TimeoutQueue->new($opt::timeout); } if(defined $opt::tmpdir) { $ENV{'TMPDIR'} = $opt::tmpdir; } + $opt::nice ||= 0; if(defined $opt::help) { die_usage(); } if(defined $opt::colsep) { $Global::trim = 'lr'; } if(defined $opt::header) { $opt::colsep = defined $opt::colsep ? $opt::colsep : "\t"; } @@ -6387,24 +6388,6 @@ sub wrapped { $command = "echo " . ::shell_quote_scalar(::shell_quote_scalar($command)); } - if($opt::nice) { - # Prepend \nice -n19 $SHELL -c - # and quote. - # The '\' before nice is needed to avoid tcsh's built-in - my $sshlogin = $self->sshlogin(); - my $serverlogin = $sshlogin->serverlogin(); - if($serverlogin eq ":") { - # Local use $Global::shell - $command = '\nice'. " -n". $opt::nice. " ". - $Global::shell. " -c ". - ::shell_quote_scalar($command); - } else { - # Remote systems use $SHELL - $command = '\nice'. " -n". $opt::nice. - ' $SHELL -c '. - ::shell_quote_scalar($command); - } - } if($opt::cat) { # In '--cat' and '--fifo' {} == $PARALLEL_TMP. # This is to make it possible to compute $PARALLEL_TMP on @@ -6562,6 +6545,7 @@ sub sshlogin_wrap { # This will be packed in ', so only use " ::spacefree(0,'$shell = "'.($ENV{'PARALLEL_SHELL'} || '$ENV{SHELL}').'";'. '$tmpdir = "'.::perl_quote_scalar($ENV{'TMPDIR'}).'";'. + '$nice = '.$opt::nice.';'. q{ # Set $PARALLEL_TMP to a non-existent file name in $TMPDIR do { @@ -6573,6 +6557,7 @@ sub sshlogin_wrap { unless($pid) { # Make own process group to be able to kill HUP it later setpgrp; + setpriority(0,0,$nice); exec $shell, "-c", ($bashfunc."@ARGV"); die "exec: $!\n"; } @@ -7016,7 +7001,7 @@ sub start { open ERR, '>&', $stderr_fh or ::die_bug("Can't dup STDERR: $!"); my $pid; - my @setpgrp_wrap = ('perl','-e',"setpgrp\;exec '$Global::shell', '-c', \@ARGV"); + my @setpgrp_wrap = ('perl','-e',"setpgrp\;setpriority\(0,0,$opt::nice\)\;exec '$Global::shell', '-c', \@ARGV"); # The eval is needed to catch exception from open3 eval { $pid = ::open3($stdin_fh, ">&OUT", ">&ERR", @setpgrp_wrap, $command) @@ -7094,7 +7079,7 @@ sub start { open ERR, '>&', $stderr_fh or ::die_bug("Can't dup STDERR: $!"); *IN = *STDIN; # The eval is needed to catch exception from open3 - my @setpgrp_wrap = ('perl','-e',"setpgrp\;exec '$Global::shell', '-c', \@ARGV"); + my @setpgrp_wrap = ('perl','-e',"setpgrp\;setpriority\(0,0,$opt::nice\)\;exec '$Global::shell', '-c', \@ARGV"); eval { $pid = ::open3("<&IN", ">&OUT", ">&ERR", @setpgrp_wrap, $command) || ::die_bug("open3-. =item B<--block-size> I -Size of block in bytes. The I can be postfixed with K, M, G, T, P, -k, m, g, t, or p which would multiply the size with 1024, 1048576, -1073741824, 1099511627776, 1125899906842624, 1000, 1000000, -1000000000, 1000000000000, or 1000000000000000 respectively. +Size of block in bytes to read at a time. The I can be postfixed +with K, M, G, T, P, k, m, g, t, or p which would multiply the size +with 1024, 1048576, 1073741824, 1099511627776, 1125899906842624, 1000, +1000000, 1000000000, 1000000000000, or 1000000000000000 respectively. GNU B tries to meet the block size but can be off by the length of one record. For performance reasons I should be bigger -than a single record. +than a two records. GNU B will warn you and automatically +increase the size if you choose a I that is too small. + +If you use B<-N>, B<--block-size> should be bigger than N+1 records. I defaults to 1M. diff --git a/src/parallel_design.pod b/src/parallel_design.pod index bb1c1883..004a3847 100644 --- a/src/parallel_design.pod +++ b/src/parallel_design.pod @@ -282,6 +282,15 @@ B<--block-size> it increases the block size by 30%. A small B<--block-size> gives very poor performance; by exponentially increasing the block size performance will not suffer. +GNU B will waste CPU power if B<--block-size> does not +contain a full record, because it tries to find a full record and will +fail to do so. The recommendation is therefore to use a +B<--block-size> > 2 records, so you always get at least one full +record when you read one block. + +If you use B<-N> then B<--block-size> should be big enough to contain +N+1 records. + =head2 Convenience options --nice --basefile --transfer --return --cleanup --tmux --group --compress --cat --fifo --workdir @@ -317,7 +326,7 @@ up the tmpfile and keep the exit code of the command. The real killer comes when you try to combine several of these: Doing that correctly for all corner cases is next to impossible to do by -hard. +hand. =head2 Shell shock