mirror of
https://git.savannah.gnu.org/git/parallel.git
synced 2024-11-22 05:57:54 +00:00
Update of instant load calc on more platforms. Needs more tought, though.
This commit is contained in:
Ole Tange
parent
70d6afbbb9
commit
87f65b76fe
|
|
@ -226,34 +226,17 @@ cc:Tim Cuthbertson <tim3d.junk@gmail.com>,
|
|||
Ryoichiro Suzuki <ryoichiro.suzuki@gmail.com>,
|
||||
Jesse Alama <jesse.alama@gmail.com>
|
||||
|
||||
Subject: GNU Parallel 20141122 ('Rosetta') released
|
||||
Subject: GNU Parallel 20141222 ('') released
|
||||
|
||||
GNU Parallel 20141122 ('Rosetta') has been released. It is available for download at: http://ftp.gnu.org/gnu/parallel/
|
||||
GNU Parallel 20141222 ('') has been released. It is available for download at: http://ftp.gnu.org/gnu/parallel/
|
||||
|
||||
Haiku of the month:
|
||||
|
||||
Hadoop bit too much?
|
||||
Want a simpler syntax now?
|
||||
Use GNU Parallel.
|
||||
-- Ole Tange
|
||||
|
||||
A central piece of command generation was rewritten making this release beta quality. As always it passes the testsuite, so most functionality clearly works.
|
||||
<<>>
|
||||
|
||||
New in this release:
|
||||
|
||||
* Remote systems can be divided into hostgroups (e.g. web and db) by prepending '@groupname/' to the sshlogin. Multiple groups can be given by separating groups with '+'. E.g. @web/www1 @web+db/www2 @db/mariadb
|
||||
|
||||
* Remote execution can be restricted to servers that are part of one or more groups by '@groupname' as an sshlogin. Multiple groups can be given by separating groups with '+'. E.g. -S @web or -S @db+web
|
||||
|
||||
* With --hostgroup you can restrict arguments to certain hostgroups by appending '@groupname' to the argument. Multiple groups can be given by separating groups with '+'. E.g. my_web_arg@web db-or-web-arg@db+web db-only-arg@db Thanks to Michel Courtine for developing a prototype for this.
|
||||
|
||||
* GNU Parallel was cited in: HTSeq-Hadoop: Extending HTSeq for Massively Parallel Sequencing Data Analysis using Hadoop http://essenceofescience.se/wp-content/uploads/2014/11/Siretskiy.pdf
|
||||
|
||||
* GNU Parallel was cited in: SlideToolkit: An Assistive Toolset for the Histological Quantification of Whole Slide Images http://www.plosone.org/article/info%3Adoi%2F10.1371%2Fjournal.pone.0110289#close
|
||||
|
||||
* GNU Parallel was cited in: Exploring a multiprocessor design space to analyze the impact of using STT-RAM in the memory hierarchy http://conservancy.umn.edu/bitstream/handle/11299/167286/Borse_umn_0130M_15431.pdf
|
||||
|
||||
* Command-Line OCR with Tesseract on Mac OS X https://ryanfb.github.io/etc/2014/11/13/command_line_ocr_on_mac_os_x.html
|
||||
* GNU Parallel was cited in: Parallel post-processing with MPI-Bash http://dl.acm.org/citation.cfm?id=2691137
|
||||
|
||||
* Bug fixes and man page updates.
|
||||
|
||||
|
|
|
|||
101
src/parallel
101
src/parallel
|
|
@ -194,7 +194,6 @@ sub pipe_part_files {
|
|||
# Remote exec should look like:
|
||||
# ssh -oLogLevel=quiet lo 'eval `echo $SHELL | grep "/t\{0,1\}csh" > /dev/null && echo setenv PARALLEL_SEQ '$PARALLEL_SEQ'\; setenv PARALLEL_PID '$PARALLEL_PID' || echo PARALLEL_SEQ='$PARALLEL_SEQ'\;export PARALLEL_SEQ\; PARALLEL_PID='$PARALLEL_PID'\;export PARALLEL_PID` ;' tty\ \>/dev/null\ \&\&\ stty\ isig\ -onlcr\ -echo\;echo\ \$SHELL\ \|\ grep\ \"/t\\\{0,1\\\}csh\"\ \>\ /dev/null\ \&\&\ setenv\ FOO\ /tmp/foo\ \|\|\ export\ FOO=/tmp/foo\; \(wc\ -\ \$FOO\)
|
||||
# ssh -tt not allowed. Remote will die due to broken pipe anyway.
|
||||
# TODO test remote with --fifo / --cat
|
||||
return @cat_partials;
|
||||
}
|
||||
|
||||
|
|
@ -3134,15 +3133,15 @@ sub which {
|
|||
(
|
||||
'aix' => $sysv,
|
||||
'cygwin' => $sysv,
|
||||
'msys' => $sysv,
|
||||
'dec_osf' => $sysv,
|
||||
'darwin' => $bsd,
|
||||
'dec_osf' => $sysv,
|
||||
'dragonfly' => $bsd,
|
||||
'freebsd' => $bsd,
|
||||
'gnu' => $sysv,
|
||||
'hpux' => $sysv,
|
||||
'linux' => $sysv,
|
||||
'mirbsd' => $bsd,
|
||||
'msys' => $sysv,
|
||||
'netbsd' => $bsd,
|
||||
'nto' => $sysv,
|
||||
'openbsd' => $bsd,
|
||||
|
|
@ -3724,6 +3723,66 @@ sub loadavg_too_high {
|
|||
$loadavg > $self->max_loadavg());
|
||||
}
|
||||
|
||||
{
|
||||
my $cmd;
|
||||
sub loadavg_cmd {
|
||||
if(not $cmd) {
|
||||
# aix => "ps -ae -o state,command" # state wrong
|
||||
# bsd => "ps ax -o state,command"
|
||||
# sysv => "ps -ef -o s -o comm"
|
||||
# dec_osf => bsd
|
||||
# dragonfly => bsd
|
||||
# freebsd => bsd
|
||||
# gnu => bsd
|
||||
# hpux => ps -el|awk '{print $2,$14,$15}'
|
||||
# irix => ps -ef -o state -o comm
|
||||
# linux => bsd
|
||||
# minix => ps el|awk '{print \$1,\$11}'
|
||||
# mirbsd => bsd
|
||||
# netbsd => bsd
|
||||
# openbsd => bsd
|
||||
# solaris => sysv
|
||||
# svr5 => sysv
|
||||
# ultrix => ps -ax | awk '{print $3,$5}'
|
||||
# unixware => ps -el|awk '{print $2,$14,$15}'
|
||||
my $ps = q{
|
||||
$sysv="ps -ef -o s -o comm";
|
||||
$sysv2="ps -ef -o state -o comm";
|
||||
$bsd="ps ax -o state,command";
|
||||
$psel="ps -el|awk '{ print \$2,\$14,\$15 }'";
|
||||
$dummy="echo S COMMAND;echo R dummy";
|
||||
%ps=(
|
||||
'aix' => "uptime",
|
||||
'cygwin' => $sysv,
|
||||
'darwin' => $bsd,
|
||||
'dec_osf' => $sysv2,
|
||||
'dragonfly' => $bsd,
|
||||
'freebsd' => $bsd,
|
||||
'gnu' => $bsd,
|
||||
'hpux' => $psel,
|
||||
'irix' => 'ps -ef -o state -o comm',
|
||||
'linux' => $bsd,
|
||||
'minix' => "ps el|awk '{print \$1,\$11}'",
|
||||
'mirbsd' => $bsd,
|
||||
'msys' => $sysv,
|
||||
'netbsd' => $bsd,
|
||||
'nto' => $dummy,
|
||||
'openbsd' => $bsd,
|
||||
'solaris' => $sysv,
|
||||
'svr5' => $psel,
|
||||
'ultrix' => "ps -ax | awk '{print \$3,\$5}'",
|
||||
);
|
||||
print `$ps{$^O}`;
|
||||
};
|
||||
$ps =~ s/[ \t\n]+/ /g;
|
||||
$cmd = "perl -e ".::shell_quote_scalar($ps);
|
||||
$cmd =~ s/\^/\\^/g;
|
||||
}
|
||||
return $cmd;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
sub loadavg {
|
||||
# If the currently know loadavg is too old:
|
||||
# Recompute a new one in the background
|
||||
|
|
@ -3741,21 +3800,28 @@ sub loadavg {
|
|||
local $/ = undef;
|
||||
my $load_out = <$load_fh>;
|
||||
close $load_fh;
|
||||
my $load =()= ($load_out=~/(^[DR]....[^\[])/gm);
|
||||
my $load =()= ($load_out=~/(^\s?[DOR]\S* [^\[])/gm);
|
||||
if($load > 0) {
|
||||
# load is overestimated by 1
|
||||
$self->{'loadavg'} = $load - 1;
|
||||
::debug("load", "New loadavg: ", $self->{'loadavg'});
|
||||
} else {
|
||||
::die_bug("loadavg_invalid_content: $load_out");
|
||||
::debug("load", "New loadavg: ", $self->{'loadavg'},"\n");
|
||||
} elsif ($load_out=~/average: (\d+.\d+)/) {
|
||||
# Aix does not support instant load average
|
||||
# 04:11AM up 21 days, 12:55, 1 user, load average: 1.85, 1.57, 1.55
|
||||
$self->{'loadavg'} = $1;
|
||||
} else {
|
||||
::die_bug("loadavg_invalid_content: " .
|
||||
$self->{'loadavg_file'} . "\n$load_out");
|
||||
}
|
||||
::debug("load", "Last update: ", $self->{'last_loadavg_update'});
|
||||
if(time - $self->{'last_loadavg_update'} > 10) {
|
||||
# last loadavg was started 10 seconds ago
|
||||
::debug("load", time - $self->{'last_loadavg_update'}, " secs old: ",
|
||||
$self->{'loadavg_file'});
|
||||
# Because of instant load average, it should not be delayed 10 secs
|
||||
# The instant load does not give 2 R if there is only 1 cpu.
|
||||
# ::debug("load", "Last update: ", $self->{'last_loadavg_update'});
|
||||
# if(time - $self->{'last_loadavg_update'} > 10) {
|
||||
# # last loadavg was started 10 seconds ago
|
||||
# ::debug("load", time - $self->{'last_loadavg_update'}, " secs old: ",
|
||||
# $self->{'loadavg_file'});
|
||||
$update_loadavg_file = 1;
|
||||
}
|
||||
# }
|
||||
} else {
|
||||
::debug("load", "No loadavg file: ", $self->{'loadavg_file'});
|
||||
$self->{'loadavg'} = undef;
|
||||
|
|
@ -3768,12 +3834,15 @@ sub loadavg {
|
|||
-e $ENV{'HOME'}."/.parallel/tmp" or mkdir $ENV{'HOME'}."/.parallel/tmp";
|
||||
my $cmd = "";
|
||||
if($self->{'string'} ne ":") {
|
||||
$cmd = $self->sshcommand() . " " . $self->serverlogin() . " ";
|
||||
$cmd = $self->sshcommand() . " " . $self->serverlogin() . " " .
|
||||
::shell_quote_scalar(loadavg_cmd());
|
||||
} else {
|
||||
$cmd .= loadavg_cmd();
|
||||
}
|
||||
# TODO Is is called 'ps ax -o state,command' on other platforms?
|
||||
$cmd .= "ps ax -o state,command";
|
||||
# $cmd .= "ps ax -o state,command";
|
||||
# As the command can take long to run if run remote
|
||||
# save it to a tmp file before moving it to the correct file
|
||||
::debug("load", "Cmd: ", $cmd);
|
||||
my $file = $self->{'loadavg_file'};
|
||||
my ($dummy_fh, $tmpfile) = ::tmpfile(SUFFIX => ".loa");
|
||||
qx{ ($cmd > $tmpfile && mv $tmpfile $file || rm $tmpfile) & };
|
||||
|
|
|
|||
Loading…
Reference in a new issue