diff --git a/build.gradle b/build.gradle index d40349a..5197f53 100644 --- a/build.gradle +++ b/build.gradle @@ -18,3 +18,17 @@ subprojects { sourceCompatibility = 1.8 targetCompatibility = 1.8 } + +tasks.create("packages") { + group "build" + + dependsOn ":client:buildDeb" + dependsOn ":client:buildRpm" + + dependsOn ":server:buildDeb" + dependsOn ":server:buildRpm" + + dependsOn ":plugins:buildDeb" + dependsOn ":plugins:buildRpm" + +} diff --git a/client/build.gradle b/client/build.gradle index 178c0d4..bd311ab 100644 --- a/client/build.gradle +++ b/client/build.gradle @@ -1,3 +1,5 @@ +import org.redline_rpm.header.Os + plugins { id 'application' @@ -25,6 +27,8 @@ dependencies { implementation group: 'org.apache.camel', name: 'camel-stream', version: camelVersion } +def projectName = "sysmon-client" + application { // Define the main class for the application. mainClassName = 'org.sysmon.client.Application' @@ -42,7 +46,7 @@ tasks.named('test') { apply plugin: 'nebula.ospackage' ospackage { - packageName = 'sysmon-client' + packageName = projectName release = '1' user = 'root' packager = "Mark Nellemann " @@ -67,15 +71,21 @@ ospackage { } -buildRpm { - dependsOn startShadowScripts - os = "LINUX" -} - buildDeb { dependsOn startShadowScripts } +buildRpm { + dependsOn startShadowScripts + os = Os.LINUX +} + +task buildRpmAix(type: Rpm) { + dependsOn startShadowScripts + packageName = "${projectName}-AIX" + os = Os.AIX +} + jar { manifest { attributes( @@ -91,7 +101,7 @@ jar { } shadowJar { - archiveBaseName.set('sysmon-client') + archiveBaseName.set(projectName) archiveClassifier.set('') archiveVersion.set('') mergeServiceFiles() // Tell plugin to merge duplicate service files diff --git a/client/src/main/java/org/sysmon/client/Application.java b/client/src/main/java/org/sysmon/client/Application.java index 097ef74..623375f 100644 --- a/client/src/main/java/org/sysmon/client/Application.java +++ b/client/src/main/java/org/sysmon/client/Application.java @@ -8,12 +8,12 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import picocli.CommandLine; -import java.io.File; import java.io.IOException; import java.net.InetAddress; import java.net.URL; import java.net.UnknownHostException; -import java.util.Properties; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.concurrent.Callable; @CommandLine.Command(name = "sysmon-client", mixinStandardHelpOptions = true) @@ -27,8 +27,8 @@ public class Application implements Callable { @CommandLine.Option(names = { "-n", "--hostname" }, description = "Client hostname (default: ).", paramLabel = "") private String hostname; - @CommandLine.Option(names = { "-p", "--plugins" }, description = "Plugin jar path (default: ${DEFAULT-VALUE}).", paramLabel = "", defaultValue = "/opt/sysmon/plugins") - private File plugins; + @CommandLine.Option(names = { "-p", "--plugin-dir" }, description = "Plugin jar path (default: ${DEFAULT-VALUE}).", paramLabel = "", defaultValue = "/opt/sysmon/plugins") + private String pluginPath; public static void main(String... args) { int exitCode = new CommandLine(new Application()).execute(args); @@ -48,7 +48,13 @@ public class Application implements Callable { } } + String pf4jPluginsDir = System.getProperty("pf4j.pluginsDir"); + if(pf4jPluginsDir != null) { + pluginPath = pf4jPluginsDir; + } + Main main = new Main(); + main.bind("pluginPath", pluginPath); main.bind("myServerUrl", serverUrl.toString()); main.bind("myHostname", hostname); main.configure().addRoutesBuilder(ClientRouteBuilder.class); diff --git a/client/src/main/java/org/sysmon/client/ClientRouteBuilder.java b/client/src/main/java/org/sysmon/client/ClientRouteBuilder.java index 41a5f79..63c96d0 100644 --- a/client/src/main/java/org/sysmon/client/ClientRouteBuilder.java +++ b/client/src/main/java/org/sysmon/client/ClientRouteBuilder.java @@ -13,6 +13,7 @@ import org.sysmon.shared.MetricResult; import java.io.File; import java.nio.file.Path; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; import java.util.Properties; @@ -26,8 +27,7 @@ public class ClientRouteBuilder extends RouteBuilder { Registry registry = getContext().getRegistry(); - Path[] pluginpaths = { new File("/opt/sysmon/plugins").toPath() }; - + Path[] pluginpaths = { Paths.get(registry.lookupByNameAndType("pluginPath", String.class)) }; PluginManager pluginManager = new JarPluginManager(pluginpaths); pluginManager.loadPlugins(); pluginManager.startPlugins(); diff --git a/plugins/build.gradle b/plugins/build.gradle index cfe30a7..b9f9cad 100644 --- a/plugins/build.gradle +++ b/plugins/build.gradle @@ -1,7 +1,10 @@ +import org.redline_rpm.header.Os + plugins { id "nebula.ospackage" version "8.5.6" } + subprojects { apply plugin: 'java' apply plugin: 'groovy' @@ -36,11 +39,10 @@ subprojects { into "../output/" } - tasks.assemble.dependsOn { + tasks.build.dependsOn { copyJar } - test { useJUnitPlatform() } @@ -55,10 +57,12 @@ task customCleanUp(type:Delete) { tasks.clean.dependsOn(tasks.customCleanUp) +def projectName = "sysmon-plugins" + apply plugin: 'nebula.ospackage' ospackage { - packageName = 'sysmon-plugins' + packageName = projectName release = '1' user = 'root' packager = "Mark Nellemann " @@ -71,11 +75,17 @@ ospackage { } -buildRpm { - dependsOn assemble - os = "LINUX" +buildDeb { + subprojects.each { dependsOn("${it.name}:build") } } -buildDeb { - dependsOn assemble +buildRpm { + subprojects.each { dependsOn("${it.name}:build") } + os = Os.LINUX +} + +task buildRpmAix(type: Rpm) { + subprojects.each { dependsOn("${it.name}:build") } + packageName = "${projectName}-AIX" + os = Os.AIX } diff --git a/plugins/sysmon-aix/src/main/java/org/sysmon/plugins/sysmon_aix/AixDiskExtension.java b/plugins/sysmon-aix/src/main/java/org/sysmon/plugins/sysmon_aix/AixDiskExtension.java index 0ec1fc9..0605f47 100644 --- a/plugins/sysmon-aix/src/main/java/org/sysmon/plugins/sysmon_aix/AixDiskExtension.java +++ b/plugins/sysmon-aix/src/main/java/org/sysmon/plugins/sysmon_aix/AixDiskExtension.java @@ -9,7 +9,9 @@ public class AixDiskExtension implements MetricExtension { @Override public boolean isSupported() { - return System.getProperty("os.name").toLowerCase().contains("aix"); + // TODO: Implement + //return System.getProperty("os.name").toLowerCase().contains("aix"); + return false; } @Override diff --git a/plugins/sysmon-aix/src/main/java/org/sysmon/plugins/sysmon_aix/AixMemoryExtension.java b/plugins/sysmon-aix/src/main/java/org/sysmon/plugins/sysmon_aix/AixMemoryExtension.java new file mode 100644 index 0000000..881e89f --- /dev/null +++ b/plugins/sysmon-aix/src/main/java/org/sysmon/plugins/sysmon_aix/AixMemoryExtension.java @@ -0,0 +1,67 @@ +package org.sysmon.plugins.sysmon_aix; + +import org.pf4j.Extension; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.sysmon.shared.Measurement; +import org.sysmon.shared.MetricExtension; +import org.sysmon.shared.MetricResult; +import org.sysmon.shared.PluginHelper; + +import java.util.List; +import java.util.Map; + +@Extension +public class AixMemoryExtension implements MetricExtension { + + private static final Logger log = LoggerFactory.getLogger(AixMemoryExtension.class); + + @Override + public boolean isSupported() { + + if(!System.getProperty("os.name").toLowerCase().contains("aix")) { + log.warn("Requires AIX."); + return false; + } + + if(!PluginHelper.canExecute("svmon")) { + log.warn("Requires the 'svmon' command."); + return false; + } + + return true; + } + + @Override + public String getName() { + return "aix-memory"; + } + + @Override + public String getProvides() { + return "memory"; + } + + @Override + public String getDescription() { + return "AIX Memory Metrics"; + } + + @Override + public MetricResult getMetrics() { + + List svmon = PluginHelper.executeCommand("svmon -G -O unit=KB"); + AixMemoryStat memoryStat = processCommandOutput(svmon); + + Map tagsMap = memoryStat.getTags(); + Map fieldsMap = memoryStat.getFields(); + + return new MetricResult("memory", new Measurement(tagsMap, fieldsMap)); + } + + protected AixMemoryStat processCommandOutput(List inputLines) { + return new AixMemoryStat(inputLines); + } + + +} diff --git a/plugins/sysmon-aix/src/main/java/org/sysmon/plugins/sysmon_aix/AixMemoryStat.java b/plugins/sysmon-aix/src/main/java/org/sysmon/plugins/sysmon_aix/AixMemoryStat.java new file mode 100644 index 0000000..ac1f14d --- /dev/null +++ b/plugins/sysmon-aix/src/main/java/org/sysmon/plugins/sysmon_aix/AixMemoryStat.java @@ -0,0 +1,68 @@ +package org.sysmon.plugins.sysmon_aix; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.math.BigDecimal; +import java.math.RoundingMode; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class AixMemoryStat { + + private static final Logger log = LoggerFactory.getLogger(AixMemoryStat.class); + + // size inuse free pin virtual available mmode + // memory 4194304 4036532 157772 1854772 2335076 1652640 Ded + private final Pattern patternAix = Pattern.compile("^memory\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(\\S+)"); + + private Long total; + private Long used; + private Long free; + private Long virtual; + private Long available; + private String mode; + + AixMemoryStat(List lines) { + for (String line : lines) { + Matcher matcher = patternAix.matcher(line); + if (matcher.find() && matcher.groupCount() == 7) { + total = Long.parseLong(matcher.group(1)); + used = Long.parseLong(matcher.group(2)); + free = Long.parseLong(matcher.group(3)); + //pin = Long.parseLong(matcher.group(4)); + virtual = Long.parseLong(matcher.group(5)); + available = Long.parseLong(matcher.group(6)); + mode = matcher.group(7); + break; + } + } + } + + + public Map getTags() { + Map tags = new HashMap<>(); + tags.put("mode", mode); + return tags; + } + + + public Map getFields() { + + double tmp = ((double) (total - available) / total ) * 100; + BigDecimal usage = new BigDecimal(tmp).setScale(2, RoundingMode.HALF_UP); + + Map fields = new HashMap<>(); + fields.put("total", total); + fields.put("used", used); + fields.put("free", free); + fields.put("virtual", virtual); + fields.put("available", available); + fields.put("usage", usage.doubleValue()); + return fields; + } + +} diff --git a/plugins/sysmon-aix/src/main/java/org/sysmon/plugins/sysmon_aix/AixProcessorExtension.java b/plugins/sysmon-aix/src/main/java/org/sysmon/plugins/sysmon_aix/AixProcessorExtension.java index d61b387..4d37676 100644 --- a/plugins/sysmon-aix/src/main/java/org/sysmon/plugins/sysmon_aix/AixProcessorExtension.java +++ b/plugins/sysmon-aix/src/main/java/org/sysmon/plugins/sysmon_aix/AixProcessorExtension.java @@ -8,7 +8,6 @@ import org.sysmon.shared.MetricExtension; import org.sysmon.shared.MetricResult; import org.sysmon.shared.PluginHelper; -import java.io.File; import java.util.List; import java.util.Map; @@ -22,12 +21,12 @@ public class AixProcessorExtension implements MetricExtension { String osArch = System.getProperty("os.arch").toLowerCase(); if(!osArch.startsWith("ppc64")) { - log.warn("Wrong os arch: " + osArch); + log.warn("Requires CPU Architecture ppc64 or ppc64le, this is: " + osArch); return false; } if(!PluginHelper.canExecute("lparstat")) { - log.warn("No lparstat command found."); + log.warn("Requires the 'lparstat' command."); return false; } @@ -52,8 +51,8 @@ public class AixProcessorExtension implements MetricExtension { @Override public MetricResult getMetrics() { - List vmstat = PluginHelper.executeCommand("lparstat 1 1"); - AixProcessorStat processorStat = processCommandOutput(vmstat); + List lparstat = PluginHelper.executeCommand("lparstat 1 1"); + AixProcessorStat processorStat = processCommandOutput(lparstat); Map tagsMap = processorStat.getTags(); Map fieldsMap = processorStat.getFields(); diff --git a/plugins/sysmon-aix/src/test/groovy/AixMemoryTest.groovy b/plugins/sysmon-aix/src/test/groovy/AixMemoryTest.groovy new file mode 100644 index 0000000..acc65c9 --- /dev/null +++ b/plugins/sysmon-aix/src/test/groovy/AixMemoryTest.groovy @@ -0,0 +1,29 @@ +import org.sysmon.plugins.sysmon_aix.AixMemoryExtension +import org.sysmon.plugins.sysmon_aix.AixMemoryStat +import spock.lang.Specification + +class AixMemoryTest extends Specification { + + void "test AIX svmon output processing"() { + + setup: + def testFile = new File(getClass().getResource('/svmon.txt').toURI()) + List lines = testFile.readLines("UTF-8") + + when: + AixMemoryExtension extension = new AixMemoryExtension() + AixMemoryStat stats = extension.processCommandOutput(lines) + + then: + + stats.getFields().get("total") == 4194304l + stats.getFields().get("used") == 4036532l + stats.getFields().get("free") == 157772l + stats.getFields().get("virtual") == 2335076l + stats.getFields().get("available") == 1652640l + stats.getFields().get("usage") == 60.6d + stats.getTags().get("mode") == "Ded" + + } + +} diff --git a/plugins/sysmon-aix/src/test/resources/iostat.txt b/plugins/sysmon-aix/src/test/resources/iostat.txt new file mode 100644 index 0000000..9b0b3b0 --- /dev/null +++ b/plugins/sysmon-aix/src/test/resources/iostat.txt @@ -0,0 +1,3 @@ +Disks: % tm_act Kbps tps Kb_read Kb_wrtn +cd0 0.0 0.0 0.0 0 0 +hdisk0 1.0 752.0 81.0 740 12 diff --git a/plugins/sysmon-aix/src/test/resources/svmon.txt b/plugins/sysmon-aix/src/test/resources/svmon.txt new file mode 100644 index 0000000..6b9ee1e --- /dev/null +++ b/plugins/sysmon-aix/src/test/resources/svmon.txt @@ -0,0 +1,9 @@ +Unit: KB +-------------------------------------------------------------------------------------- + size inuse free pin virtual available mmode +memory 4194304 4036532 157772 1854772 2335076 1652640 Ded +pg space 524288 12400 + + work pers clnt other +pin 1407060 0 16736 430976 +in use 2335076 0 1701456 diff --git a/plugins/sysmon-linux/src/main/java/org/sysmon/plugins/sysmon_linux/LinuxDiskExtension.java b/plugins/sysmon-linux/src/main/java/org/sysmon/plugins/sysmon_linux/LinuxDiskExtension.java index b941a16..312bdcf 100644 --- a/plugins/sysmon-linux/src/main/java/org/sysmon/plugins/sysmon_linux/LinuxDiskExtension.java +++ b/plugins/sysmon-linux/src/main/java/org/sysmon/plugins/sysmon_linux/LinuxDiskExtension.java @@ -1,6 +1,8 @@ package org.sysmon.plugins.sysmon_linux; import org.pf4j.Extension; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.sysmon.shared.Measurement; import org.sysmon.shared.MetricExtension; import org.sysmon.shared.MetricResult; @@ -10,25 +12,17 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Paths; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.concurrent.atomic.AtomicBoolean; @Extension public class LinuxDiskExtension implements MetricExtension { - private final static List ignoreList = new ArrayList() {{ - add("dm-"); - add("loop"); - }}; + private static final Logger log = LoggerFactory.getLogger(LinuxDiskExtension.class); - private List currentDiskStats; - private List previousDiskStats; @Override public boolean isSupported() { - //return System.getProperty("os.name").toLowerCase().contains("linux"); - return false; // TODO: Not ready yet. + return System.getProperty("os.name").toLowerCase().contains("linux"); } @Override @@ -46,75 +40,49 @@ public class LinuxDiskExtension implements MetricExtension { return "Linux Disk Metrics"; } + @Override public MetricResult getMetrics() { - MetricResult result = new MetricResult("disk"); + LinuxDiskProcLine proc1 = processFileOutput(readProcFile()); try { - copyCurrentValues(); - readProcFile(); - result.setMeasurement(calculate()); - } catch (IOException e) { - e.printStackTrace(); - } - - return result; - } - - - - private void readProcFile() throws IOException { - - currentDiskStats = new ArrayList<>(); - List allLines = Files.readAllLines(Paths.get("/proc/diskstats"), StandardCharsets.UTF_8); - for(String line : allLines) { - currentDiskStats.add(new LinuxDiskStat(line)); - } - - } - - - private void copyCurrentValues() { - - if(currentDiskStats != null && currentDiskStats.size() > 0) { - previousDiskStats = new ArrayList<>(currentDiskStats); - } - - } - - - private Measurement calculate() { - - if(previousDiskStats == null || previousDiskStats.size() != currentDiskStats.size()) { + Thread.sleep(1 * 1000); // TODO: Configure sample collect time + } catch (InterruptedException e) { + log.warn("getMetrics() - sleep interrupted"); return null; } + LinuxDiskProcLine proc2 = processFileOutput(readProcFile()); - HashMap tagsMap = new HashMap<>(); - HashMap fieldsMap = new HashMap<>(); - - for(int i = 0; i < currentDiskStats.size(); i++) { - - LinuxDiskStat curStat = currentDiskStats.get(i); - LinuxDiskStat preStat = previousDiskStats.get(i); - - AtomicBoolean ignore = new AtomicBoolean(false); - ignoreList.forEach(str -> { - if(curStat.getDevice().startsWith(str)) { - ignore.set(true); - } - }); - - if(!ignore.get()) { - fieldsMap.put(curStat.getDevice() + "_iotime", curStat.getTimeSpentOnIo() - preStat.getTimeSpentOnIo()); - //fieldsMap.put(curStat.getDevice() + "_readtime", curStat.getTimeSpentReading() - preStat.getTimeSpentReading()); - //fieldsMap.put(curStat.getDevice() + "_writetime", curStat.getTimeSpentWriting() - preStat.getTimeSpentWriting()); - fieldsMap.put(curStat.getDevice() + "_reads", curStat.getSectorsRead() - preStat.getSectorsRead()); - fieldsMap.put(curStat.getDevice() + "_writes", curStat.getSectorsWritten() - preStat.getSectorsWritten()); - } - - } - - return new Measurement(tagsMap, fieldsMap); + LinuxDiskStat stat = new LinuxDiskStat(proc2, proc1); + System.err.println("FOOBAR"); + return new MetricResult("disk", new Measurement(stat.getTags(), stat.getFields())); } -} + + protected List readProcFile() { + + List allLines = new ArrayList<>(); + try { + allLines = Files.readAllLines(Paths.get("/proc/diskstats"), StandardCharsets.UTF_8); + } catch (IOException e) { + log.error(e.getMessage()); + } + return allLines; + } + + + protected LinuxDiskProcLine processFileOutput(List inputLines) { + + for(String line : inputLines) { + String[] splitStr = line.trim().split("\\s+"); + String device = splitStr[2]; + if (device.matches("[sv]d[a-z]{1}") || device.matches("nvme[0-9]n[0-9]")) { + //log.warn("Going for: " + line); + return new LinuxDiskProcLine(line); + } + } + + return null; + } + +} \ No newline at end of file diff --git a/plugins/sysmon-linux/src/main/java/org/sysmon/plugins/sysmon_linux/LinuxDiskProcLine.java b/plugins/sysmon-linux/src/main/java/org/sysmon/plugins/sysmon_linux/LinuxDiskProcLine.java new file mode 100644 index 0000000..2d8bd24 --- /dev/null +++ b/plugins/sysmon-linux/src/main/java/org/sysmon/plugins/sysmon_linux/LinuxDiskProcLine.java @@ -0,0 +1,132 @@ +package org.sysmon.plugins.sysmon_linux; + +public class LinuxDiskProcLine { + /* + == =================================== + 1 major number + 2 minor mumber + 3 device name + 4 reads completed successfully + 5 reads merged + 6 sectors read + 7 time spent reading (ms) + 8 writes completed + 9 writes merged + 10 sectors written + 11 time spent writing (ms) + 12 I/Os currently in progress + 13 time spent doing I/Os (ms) + 14 weighted time spent doing I/Os (ms) + == =================================== + + Kernel 4.18+ appends four more fields for discard + tracking putting the total at 18: + + == =================================== + 15 discards completed successfully + 16 discards merged + 17 sectors discarded + 18 time spent discarding + == =================================== + + Kernel 5.5+ appends two more fields for flush requests: + + == ===================================== + 19 flush requests completed successfully + 20 time spent flushing + == ===================================== + */ + + private final int major; + private final int minor; + private final String device; // device name + private final Long readsCompleted; // successfully + private final Long readsMerged; + private final Long sectorsRead; // 512 bytes pr. sector + private final Long timeSpentReading; // ms + private final Long writesCompleted; // successfully + private final Long writesMerged; + private final Long sectorsWritten; // 512 bytes pr. sector + private final Long timeSpentWriting; // ms + private final Long ioInProgress; + private final Long timeSpentOnIo; // ms + private final Long timeSpentOnIoWeighted; + + private final Long discardsCompleted; // successfully + private final Long discardsMerged; + private final Long sectorsDiscarded; // 512 bytes pr. sector + private final Long timeSpentDiscarding; // ms + + private final Long flushRequestsCompleted; + private final Long timeSpentFlushing; // ms + + + LinuxDiskProcLine(String procString) { + + String[] splitStr = procString.trim().split("\\s+"); + if(splitStr.length < 14) { + throw new UnsupportedOperationException("Linux proc DISK string error: " + procString); + } + + this.major = Integer.parseInt(splitStr[0]); + this.minor = Integer.parseInt(splitStr[1]); + this.device = splitStr[2]; + this.readsCompleted = Long.parseLong(splitStr[3]); + this.readsMerged = Long.parseLong(splitStr[4]); + this.sectorsRead = Long.parseLong(splitStr[5]); + this.timeSpentReading = Long.parseLong(splitStr[6]); + this.writesCompleted = Long.parseLong(splitStr[7]); + this.writesMerged = Long.parseLong(splitStr[8]); + this.sectorsWritten = Long.parseLong(splitStr[9]); + this.timeSpentWriting = Long.parseLong(splitStr[10]); + this.ioInProgress = Long.parseLong(splitStr[11]); + this.timeSpentOnIo = Long.parseLong(splitStr[12]); + this.timeSpentOnIoWeighted = Long.parseLong(splitStr[13]); + + if(splitStr.length >= 18) { + this.discardsCompleted = Long.parseLong(splitStr[10]); + this.discardsMerged = Long.parseLong(splitStr[11]); + this.sectorsDiscarded = Long.parseLong(splitStr[12]); + this.timeSpentDiscarding = Long.parseLong(splitStr[13]); + } else { + this.discardsCompleted = null; + this.discardsMerged = null; + this.sectorsDiscarded = null; + this.timeSpentDiscarding = null; + } + + if(splitStr.length == 20) { + this.flushRequestsCompleted = Long.parseLong(splitStr[14]); + this.timeSpentFlushing = Long.parseLong(splitStr[15]); + } else { + this.flushRequestsCompleted = null; + this.timeSpentFlushing = null; + } + + } + + public String getDevice() { + return device; + } + + public Long getTimeSpentOnIo() { + return timeSpentOnIo; + } + + public Long getSectorsRead() { + return sectorsRead; + } + + public Long getSectorsWritten() { + return sectorsWritten; + } + + public Long getTimeSpentReading() { + return timeSpentReading; + } + + public Long getTimeSpentWriting() { + return timeSpentWriting; + } + +} diff --git a/plugins/sysmon-linux/src/main/java/org/sysmon/plugins/sysmon_linux/LinuxDiskStat.java b/plugins/sysmon-linux/src/main/java/org/sysmon/plugins/sysmon_linux/LinuxDiskStat.java index f17da9b..a72cc78 100644 --- a/plugins/sysmon-linux/src/main/java/org/sysmon/plugins/sysmon_linux/LinuxDiskStat.java +++ b/plugins/sysmon-linux/src/main/java/org/sysmon/plugins/sysmon_linux/LinuxDiskStat.java @@ -1,132 +1,42 @@ package org.sysmon.plugins.sysmon_linux; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.HashMap; +import java.util.Map; + public class LinuxDiskStat { - /* - == =================================== - 1 major number - 2 minor mumber - 3 device name - 4 reads completed successfully - 5 reads merged - 6 sectors read - 7 time spent reading (ms) - 8 writes completed - 9 writes merged - 10 sectors written - 11 time spent writing (ms) - 12 I/Os currently in progress - 13 time spent doing I/Os (ms) - 14 weighted time spent doing I/Os (ms) - == =================================== - Kernel 4.18+ appends four more fields for discard - tracking putting the total at 18: + private static final Logger log = LoggerFactory.getLogger(LinuxDiskStat.class); - == =================================== - 15 discards completed successfully - 16 discards merged - 17 sectors discarded - 18 time spent discarding - == =================================== + private String device; + private Long iotime; + private Long reads; + private Long writes; + private Long readTime; + private Long writeTime; - Kernel 5.5+ appends two more fields for flush requests: + LinuxDiskStat(LinuxDiskProcLine proc1, LinuxDiskProcLine proc2) { - == ===================================== - 19 flush requests completed successfully - 20 time spent flushing - == ===================================== - */ - - private final int major; - private final int minor; - private final String device; // device name - private final Long readsCompleted; // successfully - private final Long readsMerged; - private final Long sectorsRead; // 512 bytes pr. sector - private final Long timeSpentReading; // ms - private final Long writesCompleted; // successfully - private final Long writesMerged; - private final Long sectorsWritten; // 512 bytes pr. sector - private final Long timeSpentWriting; // ms - private final Long ioInProgress; - private final Long timeSpentOnIo; // ms - private final Long timeSpentOnIoWeighted; - - private final Long discardsCompleted; // successfully - private final Long discardsMerged; - private final Long sectorsDiscarded; // 512 bytes pr. sector - private final Long timeSpentDiscarding; // ms - - private final Long flushRequestsCompleted; - private final Long timeSpentFlushing; // ms - - - LinuxDiskStat(String procString) { - - String[] splitStr = procString.trim().split("\\s+"); - if(splitStr.length < 14) { - throw new UnsupportedOperationException("Linux proc DISK string error: " + procString); - } - - this.major = Integer.parseInt(splitStr[0]); - this.minor = Integer.parseInt(splitStr[1]); - this.device = splitStr[2]; - this.readsCompleted = Long.parseLong(splitStr[3]); - this.readsMerged = Long.parseLong(splitStr[4]); - this.sectorsRead = Long.parseLong(splitStr[5]); - this.timeSpentReading = Long.parseLong(splitStr[6]); - this.writesCompleted = Long.parseLong(splitStr[7]); - this.writesMerged = Long.parseLong(splitStr[8]); - this.sectorsWritten = Long.parseLong(splitStr[9]); - this.timeSpentWriting = Long.parseLong(splitStr[10]); - this.ioInProgress = Long.parseLong(splitStr[11]); - this.timeSpentOnIo = Long.parseLong(splitStr[12]); - this.timeSpentOnIoWeighted = Long.parseLong(splitStr[13]); - - if(splitStr.length >= 18) { - this.discardsCompleted = Long.parseLong(splitStr[10]); - this.discardsMerged = Long.parseLong(splitStr[11]); - this.sectorsDiscarded = Long.parseLong(splitStr[12]); - this.timeSpentDiscarding = Long.parseLong(splitStr[13]); - } else { - this.discardsCompleted = null; - this.discardsMerged = null; - this.sectorsDiscarded = null; - this.timeSpentDiscarding = null; - } - - if(splitStr.length == 20) { - this.flushRequestsCompleted = Long.parseLong(splitStr[14]); - this.timeSpentFlushing = Long.parseLong(splitStr[15]); - } else { - this.flushRequestsCompleted = null; - this.timeSpentFlushing = null; - } + device = proc1.getDevice(); + iotime = proc2.getTimeSpentOnIo() - proc1.getTimeSpentOnIo(); + writes = proc2.getSectorsWritten() - proc1.getSectorsWritten(); + reads = proc2.getSectorsRead() - proc1.getSectorsRead(); } - public String getDevice() { - return device; + public Map getTags() { + Map tags = new HashMap<>(); + tags.put("device", device); + return tags; } - public Long getTimeSpentOnIo() { - return timeSpentOnIo; + public Map getFields() { + Map fields = new HashMap<>(); + fields.put("iotime", iotime); + fields.put("writes", writes); + fields.put("reads", reads); + return fields; } - - public Long getSectorsRead() { - return sectorsRead; - } - - public Long getSectorsWritten() { - return sectorsWritten; - } - - public Long getTimeSpentReading() { - return timeSpentReading; - } - - public Long getTimeSpentWriting() { - return timeSpentWriting; - } - } \ No newline at end of file diff --git a/plugins/sysmon-linux/src/main/java/org/sysmon/plugins/sysmon_linux/LinuxMemoryExtension.java b/plugins/sysmon-linux/src/main/java/org/sysmon/plugins/sysmon_linux/LinuxMemoryExtension.java index fdeeccb..d69bf7f 100644 --- a/plugins/sysmon-linux/src/main/java/org/sysmon/plugins/sysmon_linux/LinuxMemoryExtension.java +++ b/plugins/sysmon-linux/src/main/java/org/sysmon/plugins/sysmon_linux/LinuxMemoryExtension.java @@ -1,31 +1,35 @@ package org.sysmon.plugins.sysmon_linux; import org.pf4j.Extension; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.sysmon.shared.Measurement; import org.sysmon.shared.MetricExtension; import org.sysmon.shared.MetricResult; +import org.sysmon.shared.PluginHelper; -import java.io.IOException; -import java.math.BigDecimal; -import java.math.RoundingMode; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.regex.Matcher; -import java.util.regex.Pattern; @Extension public class LinuxMemoryExtension implements MetricExtension { - private final Pattern pattern = Pattern.compile("^([a-zA-Z]+):\\s+(\\d+)\\s+"); + private static final Logger log = LoggerFactory.getLogger(LinuxMemoryExtension.class); @Override public boolean isSupported() { - return System.getProperty("os.name").toLowerCase().contains("linux"); + + if(!System.getProperty("os.name").toLowerCase().contains("linux")) { + log.warn("Requires Linux."); + return false; + } + + if(!PluginHelper.canExecute("free")) { + log.warn("Requires the 'free' command."); + return false; + } + + return true; } @Override @@ -43,66 +47,21 @@ public class LinuxMemoryExtension implements MetricExtension { return "Linux Memory Metrics"; } - @Override public MetricResult getMetrics() { - MetricResult result = new MetricResult("memory"); - try { - result.setMeasurement(processProcFile(readProcFile())); - } catch (IOException e) { - e.printStackTrace(); - } + List svmon = PluginHelper.executeCommand("free -k"); + LinuxMemoryStat memoryStat = processCommandOutput(svmon); - return result; + Map tagsMap = memoryStat.getTags(); + Map fieldsMap = memoryStat.getFields(); + + return new MetricResult("memory", new Measurement(tagsMap, fieldsMap)); + } + + protected LinuxMemoryStat processCommandOutput(List inputLines) { + return new LinuxMemoryStat(inputLines); } - protected List readProcFile() throws IOException { - List allLines = Files.readAllLines(Paths.get("/proc/meminfo"), StandardCharsets.UTF_8); - return allLines; - } - - protected Measurement processProcFile(List lines) { - - Map tagsMap = new HashMap<>(); - Map fieldsMap = new HashMap<>(); - - Long total = null; - Long available = null; - - for (String line : lines) { - - if (line.startsWith("Mem")) { - - Matcher matcher = pattern.matcher(line); - if (matcher.find() && matcher.groupCount() == 2) { - - String key = matcher.group(1).substring(3).toLowerCase(); // remove "Mem" and lowercase - String value = matcher.group(2); - - switch (key) { - case "total": - total = Long.parseLong(value); - fieldsMap.put(key, total); - break; - case "available": - available = Long.parseLong(value); - fieldsMap.put(key, available); - break; - } - - } - } - - - } - - if(total != null && available != null) { - BigDecimal usage = BigDecimal.valueOf(((float)(total - available) / total) * 100); - fieldsMap.put("usage", usage.setScale(2, RoundingMode.HALF_EVEN)); - } - - return new Measurement(tagsMap, fieldsMap); - } -} \ No newline at end of file +} diff --git a/plugins/sysmon-linux/src/main/java/org/sysmon/plugins/sysmon_linux/LinuxMemoryStat.java b/plugins/sysmon-linux/src/main/java/org/sysmon/plugins/sysmon_linux/LinuxMemoryStat.java new file mode 100644 index 0000000..2a8f300 --- /dev/null +++ b/plugins/sysmon-linux/src/main/java/org/sysmon/plugins/sysmon_linux/LinuxMemoryStat.java @@ -0,0 +1,72 @@ +package org.sysmon.plugins.sysmon_linux; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.math.BigDecimal; +import java.math.RoundingMode; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class LinuxMemoryStat { + + private static final Logger log = LoggerFactory.getLogger(LinuxMemoryStat.class); + + /* + total used free shared buff/cache available + Mem: 16069172 5896832 4597860 639780 5574480 9192992 + Swap: 3985404 0 3985404 + */ + private final Pattern pattern = Pattern.compile("^Mem:\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)"); + + private Long total; + private Long used; + private Long free; + private Long shared; + private Long buffers; + private Long available; + private String mode; + + LinuxMemoryStat(List lines) { + for (String line : lines) { + Matcher matcher = pattern.matcher(line); + if (matcher.find() && matcher.groupCount() == 6) { + total = Long.parseLong(matcher.group(1)); + used = Long.parseLong(matcher.group(2)); + free = Long.parseLong(matcher.group(3)); + shared = Long.parseLong(matcher.group(4)); + buffers = Long.parseLong(matcher.group(5)); + available = Long.parseLong(matcher.group(6)); + + break; + } + } + } + + + public Map getTags() { + Map tags = new HashMap<>(); + return tags; + } + + + public Map getFields() { + + double tmp = ((double) (total - available) / total ) * 100; + BigDecimal usage = new BigDecimal(tmp).setScale(2, RoundingMode.HALF_UP); + + Map fields = new HashMap<>(); + fields.put("total", total); + fields.put("used", used); + fields.put("free", free); + fields.put("shared", shared); + fields.put("buffers", buffers); + fields.put("available", available); + fields.put("usage", usage.doubleValue()); + return fields; + } + +} diff --git a/plugins/sysmon-linux/src/test/groovy/LinuxDiskTest.groovy b/plugins/sysmon-linux/src/test/groovy/LinuxDiskTest.groovy new file mode 100644 index 0000000..a788405 --- /dev/null +++ b/plugins/sysmon-linux/src/test/groovy/LinuxDiskTest.groovy @@ -0,0 +1,43 @@ +import org.sysmon.plugins.sysmon_linux.LinuxDiskExtension +import org.sysmon.plugins.sysmon_linux.LinuxDiskProcLine +import org.sysmon.plugins.sysmon_linux.LinuxDiskStat +import spock.lang.Specification + +class LinuxDiskTest extends Specification { + + void "test proc file processing"() { + + setup: + def testFile = new File(getClass().getResource('/diskstats1.txt').toURI()) + List lines = testFile.readLines("UTF-8") + + when: + LinuxDiskExtension extension = new LinuxDiskExtension() + LinuxDiskProcLine procLine = extension.processFileOutput(lines) + + then: + procLine.getDevice() == "nvme0n1" + procLine.getTimeSpentOnIo() == 79560l + } + + + void "test disk utilization"() { + + setup: + def testFile1 = new File(getClass().getResource('/diskstats1.txt').toURI()) + def testFile2 = new File(getClass().getResource('/diskstats2.txt').toURI()) + LinuxDiskExtension extension = new LinuxDiskExtension() + LinuxDiskProcLine procLine1 = extension.processFileOutput(testFile1.readLines()) + LinuxDiskProcLine procLine2 = extension.processFileOutput(testFile2.readLines()) + + when: + LinuxDiskStat diskStat = new LinuxDiskStat(procLine1, procLine2) + + then: + diskStat.getTags().get("device") == "nvme0n1" + diskStat.getFields().get("iotime") == 272l + diskStat.getFields().get("writes") == 78920l + diskStat.getFields().get("reads") == 0l + + } +} diff --git a/plugins/sysmon-linux/src/test/groovy/LinuxMemoryTest.groovy b/plugins/sysmon-linux/src/test/groovy/LinuxMemoryTest.groovy index a18dea0..0778ff7 100644 --- a/plugins/sysmon-linux/src/test/groovy/LinuxMemoryTest.groovy +++ b/plugins/sysmon-linux/src/test/groovy/LinuxMemoryTest.groovy @@ -1,23 +1,28 @@ import org.sysmon.plugins.sysmon_linux.LinuxMemoryExtension -import org.sysmon.shared.Measurement +import org.sysmon.plugins.sysmon_linux.LinuxMemoryStat import spock.lang.Specification class LinuxMemoryTest extends Specification { - void "test proc file processing"() { + void "test Linux free output processing"() { setup: - def testFile = new File(getClass().getResource('/meminfo.txt').toURI()) + def testFile = new File(getClass().getResource('/free.txt').toURI()) List lines = testFile.readLines("UTF-8") when: LinuxMemoryExtension extension = new LinuxMemoryExtension() - Measurement m = extension.processProcFile(lines); + LinuxMemoryStat stats = extension.processCommandOutput(lines) then: - m.getFields().get("total") == 16069616 - m.getFields().get("available") == 7968744 - m.getFields().get("usage") == 50.41 + stats.getFields().get("total") == 16069172l + stats.getFields().get("used") == 5896832l + stats.getFields().get("free") == 4597860l + stats.getFields().get("shared") == 639780l + stats.getFields().get("buffers") == 5574480l + stats.getFields().get("available") == 9192992l + stats.getFields().get("usage") == 42.79d + } } diff --git a/plugins/sysmon-linux/src/test/groovy/LinuxProcessorTest.groovy b/plugins/sysmon-linux/src/test/groovy/LinuxProcessorTest.groovy index 7e1c06d..95a0afe 100644 --- a/plugins/sysmon-linux/src/test/groovy/LinuxProcessorTest.groovy +++ b/plugins/sysmon-linux/src/test/groovy/LinuxProcessorTest.groovy @@ -24,7 +24,7 @@ class LinuxProcessorTest extends Specification { } - void "test processor utlization"() { + void "test processor utilization"() { setup: def testFile1 = new File(getClass().getResource('/proc1.txt').toURI()) diff --git a/plugins/sysmon-linux/src/test/resources/diskstats1.txt b/plugins/sysmon-linux/src/test/resources/diskstats1.txt new file mode 100644 index 0000000..19e9741 --- /dev/null +++ b/plugins/sysmon-linux/src/test/resources/diskstats1.txt @@ -0,0 +1,15 @@ + 7 0 loop0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 + 7 1 loop1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 + 7 2 loop2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 + 7 3 loop3 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 + 7 4 loop4 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 + 7 5 loop5 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 + 7 6 loop6 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 + 7 7 loop7 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 + 259 0 nvme0n1 89537 20714 7233785 14446 43973 46453 4226674 41656 0 79560 58009 0 0 0 0 2066 1906 + 259 1 nvme0n1p1 126 510 7421 38 2 0 2 0 0 84 38 0 0 0 0 0 0 + 259 2 nvme0n1p2 100 0 7383 20 14 13 216 12 0 100 32 0 0 0 0 0 0 + 259 3 nvme0n1p3 89207 20204 7213629 14363 41894 46440 4226456 39721 0 79456 54085 0 0 0 0 0 0 + 253 0 dm-0 109349 0 7211632 27872 89689 0 4226456 291556 0 80344 319428 0 0 0 0 0 0 + 253 1 dm-1 109109 0 7201162 28096 88502 0 4241000 276900 0 80356 304996 0 0 0 0 0 0 + 253 2 dm-2 194 0 8616 16 0 0 0 0 0 52 16 0 0 0 0 0 0 diff --git a/plugins/sysmon-linux/src/test/resources/diskstats2.txt b/plugins/sysmon-linux/src/test/resources/diskstats2.txt new file mode 100644 index 0000000..6635a81 --- /dev/null +++ b/plugins/sysmon-linux/src/test/resources/diskstats2.txt @@ -0,0 +1,15 @@ + 7 0 loop0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 + 7 1 loop1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 + 7 2 loop2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 + 7 3 loop3 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 + 7 4 loop4 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 + 7 5 loop5 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 + 7 6 loop6 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 + 7 7 loop7 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 + 259 0 nvme0n1 89537 20714 7233785 14446 44354 46795 4305594 42289 0 79832 58659 0 0 0 0 2078 1923 + 259 1 nvme0n1p1 126 510 7421 38 2 0 2 0 0 84 38 0 0 0 0 0 0 + 259 2 nvme0n1p2 100 0 7383 20 14 13 216 12 0 100 32 0 0 0 0 0 0 + 259 3 nvme0n1p3 89207 20204 7213629 14363 42263 46782 4305376 40338 0 79728 54701 0 0 0 0 0 0 + 253 0 dm-0 109349 0 7211632 27872 90394 0 4305376 293624 0 80628 321496 0 0 0 0 0 0 + 253 1 dm-1 109109 0 7201162 28096 89181 0 4320008 278720 0 80640 306816 0 0 0 0 0 0 + 253 2 dm-2 194 0 8616 16 0 0 0 0 0 52 16 0 0 0 0 0 0 diff --git a/plugins/sysmon-linux/src/test/resources/free.txt b/plugins/sysmon-linux/src/test/resources/free.txt new file mode 100644 index 0000000..6457c1c --- /dev/null +++ b/plugins/sysmon-linux/src/test/resources/free.txt @@ -0,0 +1,3 @@ + total used free shared buff/cache available +Mem: 16069172 5896832 4597860 639780 5574480 9192992 +Swap: 3985404 0 3985404 diff --git a/server/build.gradle b/server/build.gradle index fa17294..a7a5c27 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -1,3 +1,5 @@ +import org.redline_rpm.header.Os + plugins { id 'application' @@ -22,6 +24,8 @@ dependencies { implementation group: 'org.apache.camel', name: 'camel-influxdb', version: camelVersion } +def projectName = "sysmon-server" + application { // Define the main class for the application. mainClassName = 'org.sysmon.server.Application' @@ -34,7 +38,7 @@ tasks.named('test') { apply plugin: 'nebula.ospackage' ospackage { - packageName = 'sysmon-server' + packageName = projectName release = '1' user = 'root' packager = "Mark Nellemann " @@ -59,19 +63,19 @@ ospackage { } -buildRpm { - dependsOn startShadowScripts - os = "LINUX" -} - - buildDeb { dependsOn startShadowScripts } +buildRpm { + dependsOn startShadowScripts + os = Os.LINUX +} + task buildRpmAix(type: Rpm) { dependsOn startShadowScripts - os "AIX" + packageName = "${projectName}-AIX" + os = Os.AIX } jar { @@ -90,7 +94,7 @@ jar { } shadowJar { - archiveBaseName.set('sysmon-server') + archiveBaseName.set(projectName) archiveClassifier.set('') archiveVersion.set('') mergeServiceFiles() // Tell plugin to merge duplicate service files diff --git a/server/src/main/resources/application.properties b/server/src/main/resources/application.properties index 891efb0..cc7923e 100644 --- a/server/src/main/resources/application.properties +++ b/server/src/main/resources/application.properties @@ -27,7 +27,7 @@ camel.main.name = sysmon-server #camel.main.beanIntrospectionLoggingLevel=INFO # run in lightweight mode to be tiny as possible -#camel.main.lightweight = true +camel.main.lightweight = true # and eager load classes #camel.main.eager-classloading = true