More work on memory and disk stat extensions.

This commit is contained in:
Mark Nellemann 2021-05-13 19:28:31 +02:00
parent bc43d687a0
commit 888cdf8dad
25 changed files with 647 additions and 304 deletions

View file

@ -18,3 +18,17 @@ subprojects {
sourceCompatibility = 1.8
targetCompatibility = 1.8
}
tasks.create("packages") {
group "build"
dependsOn ":client:buildDeb"
dependsOn ":client:buildRpm"
dependsOn ":server:buildDeb"
dependsOn ":server:buildRpm"
dependsOn ":plugins:buildDeb"
dependsOn ":plugins:buildRpm"
}

View file

@ -1,3 +1,5 @@
import org.redline_rpm.header.Os
plugins {
id 'application'
@ -25,6 +27,8 @@ dependencies {
implementation group: 'org.apache.camel', name: 'camel-stream', version: camelVersion
}
def projectName = "sysmon-client"
application {
// Define the main class for the application.
mainClassName = 'org.sysmon.client.Application'
@ -42,7 +46,7 @@ tasks.named('test') {
apply plugin: 'nebula.ospackage'
ospackage {
packageName = 'sysmon-client'
packageName = projectName
release = '1'
user = 'root'
packager = "Mark Nellemann <mark.nellemann@gmail.com>"
@ -67,15 +71,21 @@ ospackage {
}
buildRpm {
dependsOn startShadowScripts
os = "LINUX"
}
buildDeb {
dependsOn startShadowScripts
}
buildRpm {
dependsOn startShadowScripts
os = Os.LINUX
}
task buildRpmAix(type: Rpm) {
dependsOn startShadowScripts
packageName = "${projectName}-AIX"
os = Os.AIX
}
jar {
manifest {
attributes(
@ -91,7 +101,7 @@ jar {
}
shadowJar {
archiveBaseName.set('sysmon-client')
archiveBaseName.set(projectName)
archiveClassifier.set('')
archiveVersion.set('')
mergeServiceFiles() // Tell plugin to merge duplicate service files

View file

@ -8,12 +8,12 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import picocli.CommandLine;
import java.io.File;
import java.io.IOException;
import java.net.InetAddress;
import java.net.URL;
import java.net.UnknownHostException;
import java.util.Properties;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.concurrent.Callable;
@CommandLine.Command(name = "sysmon-client", mixinStandardHelpOptions = true)
@ -27,8 +27,8 @@ public class Application implements Callable<Integer> {
@CommandLine.Option(names = { "-n", "--hostname" }, description = "Client hostname (default: <hostname>).", paramLabel = "<name>")
private String hostname;
@CommandLine.Option(names = { "-p", "--plugins" }, description = "Plugin jar path (default: ${DEFAULT-VALUE}).", paramLabel = "<path>", defaultValue = "/opt/sysmon/plugins")
private File plugins;
@CommandLine.Option(names = { "-p", "--plugin-dir" }, description = "Plugin jar path (default: ${DEFAULT-VALUE}).", paramLabel = "<path>", defaultValue = "/opt/sysmon/plugins")
private String pluginPath;
public static void main(String... args) {
int exitCode = new CommandLine(new Application()).execute(args);
@ -48,7 +48,13 @@ public class Application implements Callable<Integer> {
}
}
String pf4jPluginsDir = System.getProperty("pf4j.pluginsDir");
if(pf4jPluginsDir != null) {
pluginPath = pf4jPluginsDir;
}
Main main = new Main();
main.bind("pluginPath", pluginPath);
main.bind("myServerUrl", serverUrl.toString());
main.bind("myHostname", hostname);
main.configure().addRoutesBuilder(ClientRouteBuilder.class);

View file

@ -13,6 +13,7 @@ import org.sysmon.shared.MetricResult;
import java.io.File;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
@ -26,8 +27,7 @@ public class ClientRouteBuilder extends RouteBuilder {
Registry registry = getContext().getRegistry();
Path[] pluginpaths = { new File("/opt/sysmon/plugins").toPath() };
Path[] pluginpaths = { Paths.get(registry.lookupByNameAndType("pluginPath", String.class)) };
PluginManager pluginManager = new JarPluginManager(pluginpaths);
pluginManager.loadPlugins();
pluginManager.startPlugins();

View file

@ -1,7 +1,10 @@
import org.redline_rpm.header.Os
plugins {
id "nebula.ospackage" version "8.5.6"
}
subprojects {
apply plugin: 'java'
apply plugin: 'groovy'
@ -36,11 +39,10 @@ subprojects {
into "../output/"
}
tasks.assemble.dependsOn {
tasks.build.dependsOn {
copyJar
}
test {
useJUnitPlatform()
}
@ -55,10 +57,12 @@ task customCleanUp(type:Delete) {
tasks.clean.dependsOn(tasks.customCleanUp)
def projectName = "sysmon-plugins"
apply plugin: 'nebula.ospackage'
ospackage {
packageName = 'sysmon-plugins'
packageName = projectName
release = '1'
user = 'root'
packager = "Mark Nellemann <mark.nellemann@gmail.com>"
@ -71,11 +75,17 @@ ospackage {
}
buildRpm {
dependsOn assemble
os = "LINUX"
buildDeb {
subprojects.each { dependsOn("${it.name}:build") }
}
buildDeb {
dependsOn assemble
buildRpm {
subprojects.each { dependsOn("${it.name}:build") }
os = Os.LINUX
}
task buildRpmAix(type: Rpm) {
subprojects.each { dependsOn("${it.name}:build") }
packageName = "${projectName}-AIX"
os = Os.AIX
}

View file

@ -9,7 +9,9 @@ public class AixDiskExtension implements MetricExtension {
@Override
public boolean isSupported() {
return System.getProperty("os.name").toLowerCase().contains("aix");
// TODO: Implement
//return System.getProperty("os.name").toLowerCase().contains("aix");
return false;
}
@Override

View file

@ -0,0 +1,67 @@
package org.sysmon.plugins.sysmon_aix;
import org.pf4j.Extension;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.sysmon.shared.Measurement;
import org.sysmon.shared.MetricExtension;
import org.sysmon.shared.MetricResult;
import org.sysmon.shared.PluginHelper;
import java.util.List;
import java.util.Map;
@Extension
public class AixMemoryExtension implements MetricExtension {
private static final Logger log = LoggerFactory.getLogger(AixMemoryExtension.class);
@Override
public boolean isSupported() {
if(!System.getProperty("os.name").toLowerCase().contains("aix")) {
log.warn("Requires AIX.");
return false;
}
if(!PluginHelper.canExecute("svmon")) {
log.warn("Requires the 'svmon' command.");
return false;
}
return true;
}
@Override
public String getName() {
return "aix-memory";
}
@Override
public String getProvides() {
return "memory";
}
@Override
public String getDescription() {
return "AIX Memory Metrics";
}
@Override
public MetricResult getMetrics() {
List<String> svmon = PluginHelper.executeCommand("svmon -G -O unit=KB");
AixMemoryStat memoryStat = processCommandOutput(svmon);
Map<String, String> tagsMap = memoryStat.getTags();
Map<String, Object> fieldsMap = memoryStat.getFields();
return new MetricResult("memory", new Measurement(tagsMap, fieldsMap));
}
protected AixMemoryStat processCommandOutput(List<String> inputLines) {
return new AixMemoryStat(inputLines);
}
}

View file

@ -0,0 +1,68 @@
package org.sysmon.plugins.sysmon_aix;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.math.BigDecimal;
import java.math.RoundingMode;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class AixMemoryStat {
private static final Logger log = LoggerFactory.getLogger(AixMemoryStat.class);
// size inuse free pin virtual available mmode
// memory 4194304 4036532 157772 1854772 2335076 1652640 Ded
private final Pattern patternAix = Pattern.compile("^memory\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(\\S+)");
private Long total;
private Long used;
private Long free;
private Long virtual;
private Long available;
private String mode;
AixMemoryStat(List<String> lines) {
for (String line : lines) {
Matcher matcher = patternAix.matcher(line);
if (matcher.find() && matcher.groupCount() == 7) {
total = Long.parseLong(matcher.group(1));
used = Long.parseLong(matcher.group(2));
free = Long.parseLong(matcher.group(3));
//pin = Long.parseLong(matcher.group(4));
virtual = Long.parseLong(matcher.group(5));
available = Long.parseLong(matcher.group(6));
mode = matcher.group(7);
break;
}
}
}
public Map<String, String> getTags() {
Map<String, String> tags = new HashMap<>();
tags.put("mode", mode);
return tags;
}
public Map<String, Object> getFields() {
double tmp = ((double) (total - available) / total ) * 100;
BigDecimal usage = new BigDecimal(tmp).setScale(2, RoundingMode.HALF_UP);
Map<String, Object> fields = new HashMap<>();
fields.put("total", total);
fields.put("used", used);
fields.put("free", free);
fields.put("virtual", virtual);
fields.put("available", available);
fields.put("usage", usage.doubleValue());
return fields;
}
}

View file

@ -8,7 +8,6 @@ import org.sysmon.shared.MetricExtension;
import org.sysmon.shared.MetricResult;
import org.sysmon.shared.PluginHelper;
import java.io.File;
import java.util.List;
import java.util.Map;
@ -22,12 +21,12 @@ public class AixProcessorExtension implements MetricExtension {
String osArch = System.getProperty("os.arch").toLowerCase();
if(!osArch.startsWith("ppc64")) {
log.warn("Wrong os arch: " + osArch);
log.warn("Requires CPU Architecture ppc64 or ppc64le, this is: " + osArch);
return false;
}
if(!PluginHelper.canExecute("lparstat")) {
log.warn("No lparstat command found.");
log.warn("Requires the 'lparstat' command.");
return false;
}
@ -52,8 +51,8 @@ public class AixProcessorExtension implements MetricExtension {
@Override
public MetricResult getMetrics() {
List<String> vmstat = PluginHelper.executeCommand("lparstat 1 1");
AixProcessorStat processorStat = processCommandOutput(vmstat);
List<String> lparstat = PluginHelper.executeCommand("lparstat 1 1");
AixProcessorStat processorStat = processCommandOutput(lparstat);
Map<String, String> tagsMap = processorStat.getTags();
Map<String, Object> fieldsMap = processorStat.getFields();

View file

@ -0,0 +1,29 @@
import org.sysmon.plugins.sysmon_aix.AixMemoryExtension
import org.sysmon.plugins.sysmon_aix.AixMemoryStat
import spock.lang.Specification
class AixMemoryTest extends Specification {
void "test AIX svmon output processing"() {
setup:
def testFile = new File(getClass().getResource('/svmon.txt').toURI())
List<String> lines = testFile.readLines("UTF-8")
when:
AixMemoryExtension extension = new AixMemoryExtension()
AixMemoryStat stats = extension.processCommandOutput(lines)
then:
stats.getFields().get("total") == 4194304l
stats.getFields().get("used") == 4036532l
stats.getFields().get("free") == 157772l
stats.getFields().get("virtual") == 2335076l
stats.getFields().get("available") == 1652640l
stats.getFields().get("usage") == 60.6d
stats.getTags().get("mode") == "Ded"
}
}

View file

@ -0,0 +1,3 @@
Disks: % tm_act Kbps tps Kb_read Kb_wrtn
cd0 0.0 0.0 0.0 0 0
hdisk0 1.0 752.0 81.0 740 12

View file

@ -0,0 +1,9 @@
Unit: KB
--------------------------------------------------------------------------------------
size inuse free pin virtual available mmode
memory 4194304 4036532 157772 1854772 2335076 1652640 Ded
pg space 524288 12400
work pers clnt other
pin 1407060 0 16736 430976
in use 2335076 0 1701456

View file

@ -1,6 +1,8 @@
package org.sysmon.plugins.sysmon_linux;
import org.pf4j.Extension;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.sysmon.shared.Measurement;
import org.sysmon.shared.MetricExtension;
import org.sysmon.shared.MetricResult;
@ -10,25 +12,17 @@ import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.concurrent.atomic.AtomicBoolean;
@Extension
public class LinuxDiskExtension implements MetricExtension {
private final static List<String> ignoreList = new ArrayList<String>() {{
add("dm-");
add("loop");
}};
private static final Logger log = LoggerFactory.getLogger(LinuxDiskExtension.class);
private List<LinuxDiskStat> currentDiskStats;
private List<LinuxDiskStat> previousDiskStats;
@Override
public boolean isSupported() {
//return System.getProperty("os.name").toLowerCase().contains("linux");
return false; // TODO: Not ready yet.
return System.getProperty("os.name").toLowerCase().contains("linux");
}
@Override
@ -46,75 +40,49 @@ public class LinuxDiskExtension implements MetricExtension {
return "Linux Disk Metrics";
}
@Override
public MetricResult getMetrics() {
MetricResult result = new MetricResult("disk");
LinuxDiskProcLine proc1 = processFileOutput(readProcFile());
try {
copyCurrentValues();
readProcFile();
result.setMeasurement(calculate());
} catch (IOException e) {
e.printStackTrace();
}
return result;
}
private void readProcFile() throws IOException {
currentDiskStats = new ArrayList<>();
List<String> allLines = Files.readAllLines(Paths.get("/proc/diskstats"), StandardCharsets.UTF_8);
for(String line : allLines) {
currentDiskStats.add(new LinuxDiskStat(line));
}
}
private void copyCurrentValues() {
if(currentDiskStats != null && currentDiskStats.size() > 0) {
previousDiskStats = new ArrayList<>(currentDiskStats);
}
}
private Measurement calculate() {
if(previousDiskStats == null || previousDiskStats.size() != currentDiskStats.size()) {
Thread.sleep(1 * 1000); // TODO: Configure sample collect time
} catch (InterruptedException e) {
log.warn("getMetrics() - sleep interrupted");
return null;
}
LinuxDiskProcLine proc2 = processFileOutput(readProcFile());
HashMap<String, String> tagsMap = new HashMap<>();
HashMap<String, Object> fieldsMap = new HashMap<>();
LinuxDiskStat stat = new LinuxDiskStat(proc2, proc1);
System.err.println("FOOBAR");
return new MetricResult("disk", new Measurement(stat.getTags(), stat.getFields()));
}
for(int i = 0; i < currentDiskStats.size(); i++) {
LinuxDiskStat curStat = currentDiskStats.get(i);
LinuxDiskStat preStat = previousDiskStats.get(i);
protected List<String> readProcFile() {
AtomicBoolean ignore = new AtomicBoolean(false);
ignoreList.forEach(str -> {
if(curStat.getDevice().startsWith(str)) {
ignore.set(true);
}
});
List<String> allLines = new ArrayList<>();
try {
allLines = Files.readAllLines(Paths.get("/proc/diskstats"), StandardCharsets.UTF_8);
} catch (IOException e) {
log.error(e.getMessage());
}
return allLines;
}
if(!ignore.get()) {
fieldsMap.put(curStat.getDevice() + "_iotime", curStat.getTimeSpentOnIo() - preStat.getTimeSpentOnIo());
//fieldsMap.put(curStat.getDevice() + "_readtime", curStat.getTimeSpentReading() - preStat.getTimeSpentReading());
//fieldsMap.put(curStat.getDevice() + "_writetime", curStat.getTimeSpentWriting() - preStat.getTimeSpentWriting());
fieldsMap.put(curStat.getDevice() + "_reads", curStat.getSectorsRead() - preStat.getSectorsRead());
fieldsMap.put(curStat.getDevice() + "_writes", curStat.getSectorsWritten() - preStat.getSectorsWritten());
protected LinuxDiskProcLine processFileOutput(List<String> inputLines) {
for(String line : inputLines) {
String[] splitStr = line.trim().split("\\s+");
String device = splitStr[2];
if (device.matches("[sv]d[a-z]{1}") || device.matches("nvme[0-9]n[0-9]")) {
//log.warn("Going for: " + line);
return new LinuxDiskProcLine(line);
}
}
return new Measurement(tagsMap, fieldsMap);
return null;
}
}

View file

@ -0,0 +1,132 @@
package org.sysmon.plugins.sysmon_linux;
public class LinuxDiskProcLine {
/*
== ===================================
1 major number
2 minor mumber
3 device name
4 reads completed successfully
5 reads merged
6 sectors read
7 time spent reading (ms)
8 writes completed
9 writes merged
10 sectors written
11 time spent writing (ms)
12 I/Os currently in progress
13 time spent doing I/Os (ms)
14 weighted time spent doing I/Os (ms)
== ===================================
Kernel 4.18+ appends four more fields for discard
tracking putting the total at 18:
== ===================================
15 discards completed successfully
16 discards merged
17 sectors discarded
18 time spent discarding
== ===================================
Kernel 5.5+ appends two more fields for flush requests:
== =====================================
19 flush requests completed successfully
20 time spent flushing
== =====================================
*/
private final int major;
private final int minor;
private final String device; // device name
private final Long readsCompleted; // successfully
private final Long readsMerged;
private final Long sectorsRead; // 512 bytes pr. sector
private final Long timeSpentReading; // ms
private final Long writesCompleted; // successfully
private final Long writesMerged;
private final Long sectorsWritten; // 512 bytes pr. sector
private final Long timeSpentWriting; // ms
private final Long ioInProgress;
private final Long timeSpentOnIo; // ms
private final Long timeSpentOnIoWeighted;
private final Long discardsCompleted; // successfully
private final Long discardsMerged;
private final Long sectorsDiscarded; // 512 bytes pr. sector
private final Long timeSpentDiscarding; // ms
private final Long flushRequestsCompleted;
private final Long timeSpentFlushing; // ms
LinuxDiskProcLine(String procString) {
String[] splitStr = procString.trim().split("\\s+");
if(splitStr.length < 14) {
throw new UnsupportedOperationException("Linux proc DISK string error: " + procString);
}
this.major = Integer.parseInt(splitStr[0]);
this.minor = Integer.parseInt(splitStr[1]);
this.device = splitStr[2];
this.readsCompleted = Long.parseLong(splitStr[3]);
this.readsMerged = Long.parseLong(splitStr[4]);
this.sectorsRead = Long.parseLong(splitStr[5]);
this.timeSpentReading = Long.parseLong(splitStr[6]);
this.writesCompleted = Long.parseLong(splitStr[7]);
this.writesMerged = Long.parseLong(splitStr[8]);
this.sectorsWritten = Long.parseLong(splitStr[9]);
this.timeSpentWriting = Long.parseLong(splitStr[10]);
this.ioInProgress = Long.parseLong(splitStr[11]);
this.timeSpentOnIo = Long.parseLong(splitStr[12]);
this.timeSpentOnIoWeighted = Long.parseLong(splitStr[13]);
if(splitStr.length >= 18) {
this.discardsCompleted = Long.parseLong(splitStr[10]);
this.discardsMerged = Long.parseLong(splitStr[11]);
this.sectorsDiscarded = Long.parseLong(splitStr[12]);
this.timeSpentDiscarding = Long.parseLong(splitStr[13]);
} else {
this.discardsCompleted = null;
this.discardsMerged = null;
this.sectorsDiscarded = null;
this.timeSpentDiscarding = null;
}
if(splitStr.length == 20) {
this.flushRequestsCompleted = Long.parseLong(splitStr[14]);
this.timeSpentFlushing = Long.parseLong(splitStr[15]);
} else {
this.flushRequestsCompleted = null;
this.timeSpentFlushing = null;
}
}
public String getDevice() {
return device;
}
public Long getTimeSpentOnIo() {
return timeSpentOnIo;
}
public Long getSectorsRead() {
return sectorsRead;
}
public Long getSectorsWritten() {
return sectorsWritten;
}
public Long getTimeSpentReading() {
return timeSpentReading;
}
public Long getTimeSpentWriting() {
return timeSpentWriting;
}
}

View file

@ -1,132 +1,42 @@
package org.sysmon.plugins.sysmon_linux;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.HashMap;
import java.util.Map;
public class LinuxDiskStat {
/*
== ===================================
1 major number
2 minor mumber
3 device name
4 reads completed successfully
5 reads merged
6 sectors read
7 time spent reading (ms)
8 writes completed
9 writes merged
10 sectors written
11 time spent writing (ms)
12 I/Os currently in progress
13 time spent doing I/Os (ms)
14 weighted time spent doing I/Os (ms)
== ===================================
Kernel 4.18+ appends four more fields for discard
tracking putting the total at 18:
private static final Logger log = LoggerFactory.getLogger(LinuxDiskStat.class);
== ===================================
15 discards completed successfully
16 discards merged
17 sectors discarded
18 time spent discarding
== ===================================
private String device;
private Long iotime;
private Long reads;
private Long writes;
private Long readTime;
private Long writeTime;
Kernel 5.5+ appends two more fields for flush requests:
LinuxDiskStat(LinuxDiskProcLine proc1, LinuxDiskProcLine proc2) {
== =====================================
19 flush requests completed successfully
20 time spent flushing
== =====================================
*/
private final int major;
private final int minor;
private final String device; // device name
private final Long readsCompleted; // successfully
private final Long readsMerged;
private final Long sectorsRead; // 512 bytes pr. sector
private final Long timeSpentReading; // ms
private final Long writesCompleted; // successfully
private final Long writesMerged;
private final Long sectorsWritten; // 512 bytes pr. sector
private final Long timeSpentWriting; // ms
private final Long ioInProgress;
private final Long timeSpentOnIo; // ms
private final Long timeSpentOnIoWeighted;
private final Long discardsCompleted; // successfully
private final Long discardsMerged;
private final Long sectorsDiscarded; // 512 bytes pr. sector
private final Long timeSpentDiscarding; // ms
private final Long flushRequestsCompleted;
private final Long timeSpentFlushing; // ms
LinuxDiskStat(String procString) {
String[] splitStr = procString.trim().split("\\s+");
if(splitStr.length < 14) {
throw new UnsupportedOperationException("Linux proc DISK string error: " + procString);
}
this.major = Integer.parseInt(splitStr[0]);
this.minor = Integer.parseInt(splitStr[1]);
this.device = splitStr[2];
this.readsCompleted = Long.parseLong(splitStr[3]);
this.readsMerged = Long.parseLong(splitStr[4]);
this.sectorsRead = Long.parseLong(splitStr[5]);
this.timeSpentReading = Long.parseLong(splitStr[6]);
this.writesCompleted = Long.parseLong(splitStr[7]);
this.writesMerged = Long.parseLong(splitStr[8]);
this.sectorsWritten = Long.parseLong(splitStr[9]);
this.timeSpentWriting = Long.parseLong(splitStr[10]);
this.ioInProgress = Long.parseLong(splitStr[11]);
this.timeSpentOnIo = Long.parseLong(splitStr[12]);
this.timeSpentOnIoWeighted = Long.parseLong(splitStr[13]);
if(splitStr.length >= 18) {
this.discardsCompleted = Long.parseLong(splitStr[10]);
this.discardsMerged = Long.parseLong(splitStr[11]);
this.sectorsDiscarded = Long.parseLong(splitStr[12]);
this.timeSpentDiscarding = Long.parseLong(splitStr[13]);
} else {
this.discardsCompleted = null;
this.discardsMerged = null;
this.sectorsDiscarded = null;
this.timeSpentDiscarding = null;
}
if(splitStr.length == 20) {
this.flushRequestsCompleted = Long.parseLong(splitStr[14]);
this.timeSpentFlushing = Long.parseLong(splitStr[15]);
} else {
this.flushRequestsCompleted = null;
this.timeSpentFlushing = null;
}
device = proc1.getDevice();
iotime = proc2.getTimeSpentOnIo() - proc1.getTimeSpentOnIo();
writes = proc2.getSectorsWritten() - proc1.getSectorsWritten();
reads = proc2.getSectorsRead() - proc1.getSectorsRead();
}
public String getDevice() {
return device;
public Map<String, String> getTags() {
Map<String, String> tags = new HashMap<>();
tags.put("device", device);
return tags;
}
public Long getTimeSpentOnIo() {
return timeSpentOnIo;
public Map<String, Object> getFields() {
Map<String, Object> fields = new HashMap<>();
fields.put("iotime", iotime);
fields.put("writes", writes);
fields.put("reads", reads);
return fields;
}
public Long getSectorsRead() {
return sectorsRead;
}
public Long getSectorsWritten() {
return sectorsWritten;
}
public Long getTimeSpentReading() {
return timeSpentReading;
}
public Long getTimeSpentWriting() {
return timeSpentWriting;
}
}

View file

@ -1,31 +1,35 @@
package org.sysmon.plugins.sysmon_linux;
import org.pf4j.Extension;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.sysmon.shared.Measurement;
import org.sysmon.shared.MetricExtension;
import org.sysmon.shared.MetricResult;
import org.sysmon.shared.PluginHelper;
import java.io.IOException;
import java.math.BigDecimal;
import java.math.RoundingMode;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@Extension
public class LinuxMemoryExtension implements MetricExtension {
private final Pattern pattern = Pattern.compile("^([a-zA-Z]+):\\s+(\\d+)\\s+");
private static final Logger log = LoggerFactory.getLogger(LinuxMemoryExtension.class);
@Override
public boolean isSupported() {
return System.getProperty("os.name").toLowerCase().contains("linux");
if(!System.getProperty("os.name").toLowerCase().contains("linux")) {
log.warn("Requires Linux.");
return false;
}
if(!PluginHelper.canExecute("free")) {
log.warn("Requires the 'free' command.");
return false;
}
return true;
}
@Override
@ -43,66 +47,21 @@ public class LinuxMemoryExtension implements MetricExtension {
return "Linux Memory Metrics";
}
@Override
public MetricResult getMetrics() {
MetricResult result = new MetricResult("memory");
try {
result.setMeasurement(processProcFile(readProcFile()));
} catch (IOException e) {
e.printStackTrace();
}
List<String> svmon = PluginHelper.executeCommand("free -k");
LinuxMemoryStat memoryStat = processCommandOutput(svmon);
return result;
Map<String, String> tagsMap = memoryStat.getTags();
Map<String, Object> fieldsMap = memoryStat.getFields();
return new MetricResult("memory", new Measurement(tagsMap, fieldsMap));
}
protected LinuxMemoryStat processCommandOutput(List<String> inputLines) {
return new LinuxMemoryStat(inputLines);
}
protected List<String> readProcFile() throws IOException {
List<String> allLines = Files.readAllLines(Paths.get("/proc/meminfo"), StandardCharsets.UTF_8);
return allLines;
}
protected Measurement processProcFile(List<String> lines) {
Map<String, String> tagsMap = new HashMap<>();
Map<String, Object> fieldsMap = new HashMap<>();
Long total = null;
Long available = null;
for (String line : lines) {
if (line.startsWith("Mem")) {
Matcher matcher = pattern.matcher(line);
if (matcher.find() && matcher.groupCount() == 2) {
String key = matcher.group(1).substring(3).toLowerCase(); // remove "Mem" and lowercase
String value = matcher.group(2);
switch (key) {
case "total":
total = Long.parseLong(value);
fieldsMap.put(key, total);
break;
case "available":
available = Long.parseLong(value);
fieldsMap.put(key, available);
break;
}
}
}
}
if(total != null && available != null) {
BigDecimal usage = BigDecimal.valueOf(((float)(total - available) / total) * 100);
fieldsMap.put("usage", usage.setScale(2, RoundingMode.HALF_EVEN));
}
return new Measurement(tagsMap, fieldsMap);
}
}

View file

@ -0,0 +1,72 @@
package org.sysmon.plugins.sysmon_linux;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.math.BigDecimal;
import java.math.RoundingMode;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class LinuxMemoryStat {
private static final Logger log = LoggerFactory.getLogger(LinuxMemoryStat.class);
/*
total used free shared buff/cache available
Mem: 16069172 5896832 4597860 639780 5574480 9192992
Swap: 3985404 0 3985404
*/
private final Pattern pattern = Pattern.compile("^Mem:\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)");
private Long total;
private Long used;
private Long free;
private Long shared;
private Long buffers;
private Long available;
private String mode;
LinuxMemoryStat(List<String> lines) {
for (String line : lines) {
Matcher matcher = pattern.matcher(line);
if (matcher.find() && matcher.groupCount() == 6) {
total = Long.parseLong(matcher.group(1));
used = Long.parseLong(matcher.group(2));
free = Long.parseLong(matcher.group(3));
shared = Long.parseLong(matcher.group(4));
buffers = Long.parseLong(matcher.group(5));
available = Long.parseLong(matcher.group(6));
break;
}
}
}
public Map<String, String> getTags() {
Map<String, String> tags = new HashMap<>();
return tags;
}
public Map<String, Object> getFields() {
double tmp = ((double) (total - available) / total ) * 100;
BigDecimal usage = new BigDecimal(tmp).setScale(2, RoundingMode.HALF_UP);
Map<String, Object> fields = new HashMap<>();
fields.put("total", total);
fields.put("used", used);
fields.put("free", free);
fields.put("shared", shared);
fields.put("buffers", buffers);
fields.put("available", available);
fields.put("usage", usage.doubleValue());
return fields;
}
}

View file

@ -0,0 +1,43 @@
import org.sysmon.plugins.sysmon_linux.LinuxDiskExtension
import org.sysmon.plugins.sysmon_linux.LinuxDiskProcLine
import org.sysmon.plugins.sysmon_linux.LinuxDiskStat
import spock.lang.Specification
class LinuxDiskTest extends Specification {
void "test proc file processing"() {
setup:
def testFile = new File(getClass().getResource('/diskstats1.txt').toURI())
List<String> lines = testFile.readLines("UTF-8")
when:
LinuxDiskExtension extension = new LinuxDiskExtension()
LinuxDiskProcLine procLine = extension.processFileOutput(lines)
then:
procLine.getDevice() == "nvme0n1"
procLine.getTimeSpentOnIo() == 79560l
}
void "test disk utilization"() {
setup:
def testFile1 = new File(getClass().getResource('/diskstats1.txt').toURI())
def testFile2 = new File(getClass().getResource('/diskstats2.txt').toURI())
LinuxDiskExtension extension = new LinuxDiskExtension()
LinuxDiskProcLine procLine1 = extension.processFileOutput(testFile1.readLines())
LinuxDiskProcLine procLine2 = extension.processFileOutput(testFile2.readLines())
when:
LinuxDiskStat diskStat = new LinuxDiskStat(procLine1, procLine2)
then:
diskStat.getTags().get("device") == "nvme0n1"
diskStat.getFields().get("iotime") == 272l
diskStat.getFields().get("writes") == 78920l
diskStat.getFields().get("reads") == 0l
}
}

View file

@ -1,23 +1,28 @@
import org.sysmon.plugins.sysmon_linux.LinuxMemoryExtension
import org.sysmon.shared.Measurement
import org.sysmon.plugins.sysmon_linux.LinuxMemoryStat
import spock.lang.Specification
class LinuxMemoryTest extends Specification {
void "test proc file processing"() {
void "test Linux free output processing"() {
setup:
def testFile = new File(getClass().getResource('/meminfo.txt').toURI())
def testFile = new File(getClass().getResource('/free.txt').toURI())
List<String> lines = testFile.readLines("UTF-8")
when:
LinuxMemoryExtension extension = new LinuxMemoryExtension()
Measurement m = extension.processProcFile(lines);
LinuxMemoryStat stats = extension.processCommandOutput(lines)
then:
m.getFields().get("total") == 16069616
m.getFields().get("available") == 7968744
m.getFields().get("usage") == 50.41
stats.getFields().get("total") == 16069172l
stats.getFields().get("used") == 5896832l
stats.getFields().get("free") == 4597860l
stats.getFields().get("shared") == 639780l
stats.getFields().get("buffers") == 5574480l
stats.getFields().get("available") == 9192992l
stats.getFields().get("usage") == 42.79d
}
}

View file

@ -24,7 +24,7 @@ class LinuxProcessorTest extends Specification {
}
void "test processor utlization"() {
void "test processor utilization"() {
setup:
def testFile1 = new File(getClass().getResource('/proc1.txt').toURI())

View file

@ -0,0 +1,15 @@
7 0 loop0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
7 1 loop1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
7 2 loop2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
7 3 loop3 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
7 4 loop4 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
7 5 loop5 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
7 6 loop6 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
7 7 loop7 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
259 0 nvme0n1 89537 20714 7233785 14446 43973 46453 4226674 41656 0 79560 58009 0 0 0 0 2066 1906
259 1 nvme0n1p1 126 510 7421 38 2 0 2 0 0 84 38 0 0 0 0 0 0
259 2 nvme0n1p2 100 0 7383 20 14 13 216 12 0 100 32 0 0 0 0 0 0
259 3 nvme0n1p3 89207 20204 7213629 14363 41894 46440 4226456 39721 0 79456 54085 0 0 0 0 0 0
253 0 dm-0 109349 0 7211632 27872 89689 0 4226456 291556 0 80344 319428 0 0 0 0 0 0
253 1 dm-1 109109 0 7201162 28096 88502 0 4241000 276900 0 80356 304996 0 0 0 0 0 0
253 2 dm-2 194 0 8616 16 0 0 0 0 0 52 16 0 0 0 0 0 0

View file

@ -0,0 +1,15 @@
7 0 loop0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
7 1 loop1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
7 2 loop2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
7 3 loop3 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
7 4 loop4 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
7 5 loop5 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
7 6 loop6 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
7 7 loop7 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
259 0 nvme0n1 89537 20714 7233785 14446 44354 46795 4305594 42289 0 79832 58659 0 0 0 0 2078 1923
259 1 nvme0n1p1 126 510 7421 38 2 0 2 0 0 84 38 0 0 0 0 0 0
259 2 nvme0n1p2 100 0 7383 20 14 13 216 12 0 100 32 0 0 0 0 0 0
259 3 nvme0n1p3 89207 20204 7213629 14363 42263 46782 4305376 40338 0 79728 54701 0 0 0 0 0 0
253 0 dm-0 109349 0 7211632 27872 90394 0 4305376 293624 0 80628 321496 0 0 0 0 0 0
253 1 dm-1 109109 0 7201162 28096 89181 0 4320008 278720 0 80640 306816 0 0 0 0 0 0
253 2 dm-2 194 0 8616 16 0 0 0 0 0 52 16 0 0 0 0 0 0

View file

@ -0,0 +1,3 @@
total used free shared buff/cache available
Mem: 16069172 5896832 4597860 639780 5574480 9192992
Swap: 3985404 0 3985404

View file

@ -1,3 +1,5 @@
import org.redline_rpm.header.Os
plugins {
id 'application'
@ -22,6 +24,8 @@ dependencies {
implementation group: 'org.apache.camel', name: 'camel-influxdb', version: camelVersion
}
def projectName = "sysmon-server"
application {
// Define the main class for the application.
mainClassName = 'org.sysmon.server.Application'
@ -34,7 +38,7 @@ tasks.named('test') {
apply plugin: 'nebula.ospackage'
ospackage {
packageName = 'sysmon-server'
packageName = projectName
release = '1'
user = 'root'
packager = "Mark Nellemann <mark.nellemann@gmail.com>"
@ -59,19 +63,19 @@ ospackage {
}
buildRpm {
dependsOn startShadowScripts
os = "LINUX"
}
buildDeb {
dependsOn startShadowScripts
}
buildRpm {
dependsOn startShadowScripts
os = Os.LINUX
}
task buildRpmAix(type: Rpm) {
dependsOn startShadowScripts
os "AIX"
packageName = "${projectName}-AIX"
os = Os.AIX
}
jar {
@ -90,7 +94,7 @@ jar {
}
shadowJar {
archiveBaseName.set('sysmon-server')
archiveBaseName.set(projectName)
archiveClassifier.set('')
archiveVersion.set('')
mergeServiceFiles() // Tell plugin to merge duplicate service files

View file

@ -27,7 +27,7 @@ camel.main.name = sysmon-server
#camel.main.beanIntrospectionLoggingLevel=INFO
# run in lightweight mode to be tiny as possible
#camel.main.lightweight = true
camel.main.lightweight = true
# and eager load classes
#camel.main.eager-classloading = true