From a83930452567e4008c652c37d0425f40022cd379 Mon Sep 17 00:00:00 2001 From: Mark Nellemann Date: Mon, 10 May 2021 16:56:56 +0200 Subject: [PATCH] Measurements simplified again, to ensure influxdb route works. More work on AIX plugin. --- client/build.gradle | 1 + .../java/org/sysmon/client/Application.java | 12 +- .../org/sysmon/client/ClientRouteBuilder.java | 2 - .../sysmon/client/MetricEnrichProcessor.java | 2 +- plugins/build.gradle | 1 - .../plugins/sysmon_aix/AixDiskExtension.java | 2 +- .../sysmon_aix/AixProcessorExtension.java | 36 +---- .../plugins/sysmon_aix/AixProcessorStat.java | 130 +++++++++++++----- .../src/test/groovy/AixProcessorTest.groovy | 18 +-- .../src/test/resources/lparstat.txt | 6 + .../src/test/resources/mpstat-v.txt | 15 ++ .../sysmon-aix/src/test/resources/mpstat1.txt | 15 -- .../sysmon-aix/src/test/resources/mpstat2.txt | 15 -- .../src/test/resources/vmstat-Iwt.txt | 7 + .../sysmon_linux/LinuxDiskExtension.java | 30 ++-- .../sysmon_linux/LinuxMemoryExtension.java | 9 +- .../sysmon_linux/LinuxProcessorExtension.java | 38 ++--- .../sysmon_linux/LinuxProcessorStat.java | 27 ++-- .../src/test/groovy/LinuxProcessorTest.groovy | 17 ++- server/build.gradle | 7 +- .../server/MetricResultToPointProcessor.java | 44 +++--- .../org/sysmon/server/ServerRouteBuilder.java | 3 +- .../java/org/sysmon/shared/Measurement.java | 7 +- .../java/org/sysmon/shared/MetricResult.java | 40 +++--- .../java/org/sysmon/shared/PluginHelper.java | 11 +- 25 files changed, 271 insertions(+), 224 deletions(-) create mode 100644 plugins/sysmon-aix/src/test/resources/lparstat.txt create mode 100644 plugins/sysmon-aix/src/test/resources/mpstat-v.txt delete mode 100644 plugins/sysmon-aix/src/test/resources/mpstat1.txt delete mode 100644 plugins/sysmon-aix/src/test/resources/mpstat2.txt create mode 100644 plugins/sysmon-aix/src/test/resources/vmstat-Iwt.txt diff --git a/client/build.gradle b/client/build.gradle index 7b60692..46055e5 100644 --- a/client/build.gradle +++ b/client/build.gradle @@ -9,6 +9,7 @@ plugins { dependencies { testImplementation project(':shared') implementation project(':shared') + implementation project(':plugins') annotationProcessor(group: 'org.pf4j', name: 'pf4j', version: "${pf4jVersion}") implementation group: 'org.pf4j', name: 'pf4j', version: "${pf4jVersion}" diff --git a/client/src/main/java/org/sysmon/client/Application.java b/client/src/main/java/org/sysmon/client/Application.java index 67109e8..146b751 100644 --- a/client/src/main/java/org/sysmon/client/Application.java +++ b/client/src/main/java/org/sysmon/client/Application.java @@ -4,16 +4,21 @@ package org.sysmon.client; import org.apache.camel.main.Main; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import picocli.CommandLine; import java.io.IOException; import java.net.InetAddress; import java.net.URL; +import java.net.UnknownHostException; import java.util.concurrent.Callable; @CommandLine.Command(name = "sysmon-client", mixinStandardHelpOptions = true) public class Application implements Callable { + private static final Logger log = LoggerFactory.getLogger(Application.class); + @CommandLine.Option(names = { "-s", "--server-url" }, description = "Server URL (default: ${DEFAULT-VALUE}).", defaultValue = "http://127.0.0.1:9925/metrics", paramLabel = "") private URL serverUrl; @@ -31,7 +36,12 @@ public class Application implements Callable { public Integer call() throws IOException { if(hostname == null || hostname.isEmpty()) { - hostname = InetAddress.getLocalHost().getHostName(); + try { + hostname = InetAddress.getLocalHost().getHostName(); + } catch (UnknownHostException e) { + log.warn(e.getMessage()); + hostname = "unknown"; + } } Main main = new Main(); diff --git a/client/src/main/java/org/sysmon/client/ClientRouteBuilder.java b/client/src/main/java/org/sysmon/client/ClientRouteBuilder.java index 46ff09d..a9b12d7 100644 --- a/client/src/main/java/org/sysmon/client/ClientRouteBuilder.java +++ b/client/src/main/java/org/sysmon/client/ClientRouteBuilder.java @@ -27,7 +27,6 @@ public class ClientRouteBuilder extends RouteBuilder { pluginManager.startPlugins(); List metricExtensions = pluginManager.getExtensions(MetricExtension.class); - //log.info(String.format("Found %d extensions for extension point '%s':", metricExtensions.size(), MetricExtension.class.getName())); for (MetricExtension ext : metricExtensions) { if(ext.isSupported()) { log.info(">>> Enabling extension: " + ext.getDescription()); @@ -42,7 +41,6 @@ public class ClientRouteBuilder extends RouteBuilder { .stop() .otherwise() .to("seda:metrics"); - } } diff --git a/client/src/main/java/org/sysmon/client/MetricEnrichProcessor.java b/client/src/main/java/org/sysmon/client/MetricEnrichProcessor.java index e441b33..97cf4a5 100644 --- a/client/src/main/java/org/sysmon/client/MetricEnrichProcessor.java +++ b/client/src/main/java/org/sysmon/client/MetricEnrichProcessor.java @@ -20,7 +20,7 @@ public class MetricEnrichProcessor implements Processor { MetricResult metricResult = exchange.getIn().getBody(MetricResult.class); // We make sure MetricResults with no measurements are not sent further down the line - if(metricResult == null || metricResult.getMeasurements().size() < 1) { + if(metricResult == null || metricResult.getMeasurement() == null) { exchange.setProperty("skip", true); return; } diff --git a/plugins/build.gradle b/plugins/build.gradle index 87b4191..c88be84 100644 --- a/plugins/build.gradle +++ b/plugins/build.gradle @@ -47,7 +47,6 @@ subprojects { task customCleanUp(type:Delete) { delete "output" - //delete files("${buildDir}/test/*.jar") } tasks.clean.dependsOn(tasks.customCleanUp) \ No newline at end of file diff --git a/plugins/sysmon-aix/src/main/java/org/sysmon/plugins/sysmon_aix/AixDiskExtension.java b/plugins/sysmon-aix/src/main/java/org/sysmon/plugins/sysmon_aix/AixDiskExtension.java index 598593f..d6aaef0 100644 --- a/plugins/sysmon-aix/src/main/java/org/sysmon/plugins/sysmon_aix/AixDiskExtension.java +++ b/plugins/sysmon-aix/src/main/java/org/sysmon/plugins/sysmon_aix/AixDiskExtension.java @@ -24,7 +24,7 @@ public class AixDiskExtension implements MetricExtension { @Override public MetricResult getMetrics() { - return null; + return new MetricResult("disk"); } } \ No newline at end of file diff --git a/plugins/sysmon-aix/src/main/java/org/sysmon/plugins/sysmon_aix/AixProcessorExtension.java b/plugins/sysmon-aix/src/main/java/org/sysmon/plugins/sysmon_aix/AixProcessorExtension.java index eb7351b..b441ae3 100644 --- a/plugins/sysmon-aix/src/main/java/org/sysmon/plugins/sysmon_aix/AixProcessorExtension.java +++ b/plugins/sysmon-aix/src/main/java/org/sysmon/plugins/sysmon_aix/AixProcessorExtension.java @@ -8,7 +8,6 @@ import org.sysmon.shared.MetricExtension; import org.sysmon.shared.MetricResult; import org.sysmon.shared.PluginHelper; -import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -36,39 +35,18 @@ public class AixProcessorExtension implements MetricExtension { @Override public MetricResult getMetrics() { - MetricResult result = new MetricResult("processor"); - List measurementList = new ArrayList<>(); + List vmstat = PluginHelper.executeCommand("/usr/bin/lparstat 1 1"); + AixProcessorStat processorStat = processCommandOutput(vmstat); - List mpstat = PluginHelper.executeCommand("mpstat", "-a"); - List processorStats = processCommandOutput(mpstat); + Map tagsMap = processorStat.getTags(); + Map fieldsMap = processorStat.getFields(); - for(AixProcessorStat stat : processorStats) { - - Map tagsMap = new HashMap<>(); - tagsMap.put("cpu", stat.getName()); - // TODO: entitlements as tag or field ? - - Map fieldsMap = new HashMap<>(); - fieldsMap.put("utilization", stat.getUtilizationPercentage()); - - measurementList.add(new Measurement(tagsMap, fieldsMap)); - } - result.addMeasurements(measurementList); - - return result; + return new MetricResult("processor", new Measurement(tagsMap, fieldsMap)); } - protected List processCommandOutput(List inputLines) { - List processorStatList = new ArrayList<>(); - - for(String line : inputLines) { - if(line.matches("^\\s+[0-9]+\\s+.*")) { - processorStatList.add(new AixProcessorStat(line)); - } - } - - return processorStatList; + protected AixProcessorStat processCommandOutput(List inputLines) { + return new AixProcessorStat(inputLines); } diff --git a/plugins/sysmon-aix/src/main/java/org/sysmon/plugins/sysmon_aix/AixProcessorStat.java b/plugins/sysmon-aix/src/main/java/org/sysmon/plugins/sysmon_aix/AixProcessorStat.java index adcc330..43b51dc 100644 --- a/plugins/sysmon-aix/src/main/java/org/sysmon/plugins/sysmon_aix/AixProcessorStat.java +++ b/plugins/sysmon-aix/src/main/java/org/sysmon/plugins/sysmon_aix/AixProcessorStat.java @@ -1,61 +1,127 @@ package org.sysmon.plugins.sysmon_aix; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + public class AixProcessorStat { - private final String name; - private final Float userTime; - private final Float systemTime; - private final Float waitTime; - private final Float idleTime; + private static final Logger log = LoggerFactory.getLogger(AixProcessorStat.class); - AixProcessorStat(String procString) { + private final Pattern pattern = Pattern.compile("^System configuration: type=(\\S+) mode=(\\S+) smt=(\\d+) lcpu=(\\d+) mem=(\\d+)MB psize=(\\d+) ent=(\\d+\\.?\\d*)"); - // cpu min maj mpcs mpcr dev soft dec ph cs ics bound rq push S3pull S3grd S0rd S1rd S2rd S3rd S4rd S5rd sysc us sy wa id pc %ec ilcs vlcs S3hrd S4hrd S5hrd %nsp - String[] splitStr = procString.trim().split("\\s+"); - if(splitStr.length != 35) { - throw new UnsupportedOperationException("AIX mpstat CPU string error: " + procString); + private String type; + private String mode; + private Integer smt; + private Integer lcpu; + private Integer psize; + private Float ent; + + private final Float user; // Indicates the percentage of the entitled processing capacity used while executing at the user level (application). + private final Float sys; // Indicates the percentage of the entitled processing capacity used while executing at the system level (kernel). + private final Float wait; // Indicates the percentage of the entitled processing capacity unused while the partition was idle and had outstanding disk I/O request(s). + private final Float idle; // Indicates the percentage of the entitled processing capacity unused while the partition was idle and did not have any outstanding disk I/O request. + private final Float physc; // Indicates the number of physical processors consumed. + private final Float entc; // Indicates the percentage of the entitled capacity consumed. + private final Float lbusy; // Indicates the percentage of logical processor(s) utilization that occurred while executing at the user and system level. + + + /* + + System configuration: type=Shared mode=Uncapped smt=8 lcpu=8 mem=4096MB psize=19 ent=0.50 + + + %user %sys %wait %idle physc %entc lbusy vcsw phint %nsp %utcyc + ----- ----- ------ ------ ----- ----- ------ ----- ----- ----- ------ + 0.1 0.0 0.0 99.9 0.00 0.2 1.9 37441986 316 149 33.06 + */ + AixProcessorStat(List vmstatLines) { + + for(String line : vmstatLines) { + Matcher matcher = pattern.matcher(line); + if (matcher.find() && matcher.groupCount() == 7) { + type=matcher.group(1); + mode=matcher.group(2); + smt = Integer.parseInt(matcher.group(3)); + lcpu = Integer.parseInt(matcher.group(4)); + psize = Integer.parseInt(matcher.group(5)); + ent = Float.parseFloat(matcher.group(7)); + break; + } } - this.name = "cpu" + splitStr[0]; - this.userTime = Float.parseFloat(splitStr[23]); - this.systemTime = Float.parseFloat(splitStr[24]); - this.waitTime = Float.parseFloat(splitStr[25]); - this.idleTime = Float.parseFloat(splitStr[26]); + String vmstat = vmstatLines.get(vmstatLines.size() -1); + String[] splitStr = vmstat.trim().split("\\s+"); + if(splitStr.length != 11) { + throw new UnsupportedOperationException("vmstat string error: " + splitStr.length); + } + this.user = Float.parseFloat(splitStr[0]); + this.sys = Float.parseFloat(splitStr[1]); + this.wait = Float.parseFloat(splitStr[2]); + this.idle = Float.parseFloat(splitStr[3]); + this.physc = Float.parseFloat(splitStr[4]); + this.entc = Float.parseFloat(splitStr[5]); + this.lbusy = Float.parseFloat(splitStr[6]); } - public String getName() { - return name; + public Float getUser() { + return user; } - public Float getUserTime() { - return userTime; + public Float getSys() { + return sys; } - public Float getSystemTime() { - return systemTime; + public Float getIdle() { + return idle; } - public Float getIdleTime() { - return idleTime; + public Float getWait() { + return wait; } - public Float getWaitTime() { - return waitTime; + public Float getPhysc() { + return physc; } - - public Float getCombinedWorkTime() { - return userTime + systemTime; + public Float getEntc() { + return entc; } - public Float getCombinedTime() { - return getIdleTime() + getCombinedWorkTime(); + public Float getLbusy() { + return lbusy; } - public float getUtilizationPercentage() { - return 100 - idleTime; + public float getUsage() { + return 100 - idle; } + public Map getTags() { + Map tags = new HashMap<>(); + tags.put("mode", mode); + tags.put("type", type); + return tags; + } + + public Map getFields() { + Map fields = new HashMap<>(); + fields.put("lcpu", lcpu); + fields.put("ent", ent); + fields.put("user", user); + fields.put("sys", sys); + fields.put("idle", idle); + fields.put("wait", wait); + fields.put("physc", physc); + fields.put("entc", entc); + fields.put("lbusy", lbusy); + + return fields; + } } diff --git a/plugins/sysmon-aix/src/test/groovy/AixProcessorTest.groovy b/plugins/sysmon-aix/src/test/groovy/AixProcessorTest.groovy index cee6460..35d3937 100644 --- a/plugins/sysmon-aix/src/test/groovy/AixProcessorTest.groovy +++ b/plugins/sysmon-aix/src/test/groovy/AixProcessorTest.groovy @@ -4,24 +4,24 @@ import spock.lang.Specification class AixProcessorTest extends Specification { - void "test mpstat output processing"() { + void "test lparstat output processing"() { setup: - def testFile = new File(getClass().getResource('/mpstat1.txt').toURI()) + def testFile = new File(getClass().getResource('/lparstat.txt').toURI()) List lines = testFile.readLines("UTF-8") when: AixProcessorExtension extension = new AixProcessorExtension() - List stats = extension.processCommandOutput(lines) + AixProcessorStat stats = extension.processCommandOutput(lines) then: - stats[0].getCombinedWorkTime() == 85.1f - stats[0].getCombinedTime() == 100.0f - stats[0].getSystemTime() == 18.4f - stats[0].getUserTime() == 66.7f - stats[0].getWaitTime() == 0.0f - stats[0].getIdleTime() == 14.9f + stats.getUser() == 83.7f + stats.getSys() == 3.3f + stats.getWait() == 0.0f + stats.getIdle() == 13.0f + stats.getFields().get("ent") == 0.50f } + } diff --git a/plugins/sysmon-aix/src/test/resources/lparstat.txt b/plugins/sysmon-aix/src/test/resources/lparstat.txt new file mode 100644 index 0000000..7ac01eb --- /dev/null +++ b/plugins/sysmon-aix/src/test/resources/lparstat.txt @@ -0,0 +1,6 @@ + +System configuration: type=Shared mode=Uncapped smt=8 lcpu=8 mem=4096MB psize=19 ent=0.50 + +%user %sys %wait %idle physc %entc lbusy vcsw phint %nsp %utcyc +----- ----- ------ ------ ----- ----- ------ ----- ----- ----- ------ + 83.7 3.3 0.0 13.0 1.00 199.6 66.4 1288 0 149 32.96 diff --git a/plugins/sysmon-aix/src/test/resources/mpstat-v.txt b/plugins/sysmon-aix/src/test/resources/mpstat-v.txt new file mode 100644 index 0000000..7922964 --- /dev/null +++ b/plugins/sysmon-aix/src/test/resources/mpstat-v.txt @@ -0,0 +1,15 @@ + +System configuration: lcpu=8 ent=0.5 mode=Uncapped + + +vcpu lcpu us sy wa id pbusy pc VTB(ms) +---- ---- ---- ---- ----- ----- ----- ----- ------- +0 33.51 5.62 0.11 60.77 0.00[ 39.1%] 0.00[ 0.1%] 1784440 + 0 13.38 3.96 0.05 4.81 0.00[ 17.3%] 0.00[ 22.2%] - + 1 8.41 0.72 0.04 7.59 0.00[ 9.1%] 0.00[ 16.8%] - + 2 5.19 0.35 0.01 6.45 0.00[ 5.5%] 0.00[ 12.0%] - + 3 4.42 0.28 0.01 8.42 0.00[ 4.7%] 0.00[ 13.1%] - + 4 0.57 0.07 0.00 7.47 0.00[ 0.6%] 0.00[ 8.1%] - + 5 0.51 0.06 0.00 9.28 0.00[ 0.6%] 0.00[ 9.8%] - + 6 0.49 0.07 0.00 7.49 0.00[ 0.6%] 0.00[ 8.1%] - + 7 0.54 0.10 0.00 9.26 0.00[ 0.6%] 0.00[ 9.9%] - diff --git a/plugins/sysmon-aix/src/test/resources/mpstat1.txt b/plugins/sysmon-aix/src/test/resources/mpstat1.txt deleted file mode 100644 index 80f39e7..0000000 --- a/plugins/sysmon-aix/src/test/resources/mpstat1.txt +++ /dev/null @@ -1,15 +0,0 @@ - -System configuration: lcpu=8 ent=0.2 mode=Uncapped - -cpu min maj mpcs mpcr dev soft dec ph cs ics bound rq push S3pull S3grd S0rd S1rd S2rd S3rd S4rd S5rd sysc us sy wa id pc %ec ilcs vlcs S3hrd S4hrd S5hrd -cpu min maj mpcs mpcr dev soft dec ph cs ics bound rq push S3pull S3grd S0rd S1rd S2rd S3rd S4rd S5rd sysc us sy wa id pc %ec ilcs vlcs S3hrd S4hrd S5hrd %nsp - 0 22324631 12995 2450 1 117140 14071941 37427759 4 25044259 445796 1 3 0 0 0 99.9 0.1 0.0 0.0 0.0 0.0 77548535 66.7 18.4 0.0 14.9 0.00 0.1 237843 38774212 100.0 0.0 0.0 150 - 1 275936 176 7 350 68318 1505 3604407 41 31072 25403 0 0 0 0 0 24.5 75.5 0.0 0.0 0.0 0.0 608541 0.2 0.8 0.0 99.0 0.00 0.0 916 3701367 100.0 0.0 0.0 150 - 2 4713 1 0 351 58162 833 2400472 13 24152 23991 0 0 0 0 0 0.7 99.3 0.0 0.0 0.0 0.0 1445 0.0 1.0 0.0 99.0 0.00 0.0 163 2483263 100.0 0.0 0.0 150 - 3 4587 2 0 351 57777 836 2400544 17 24094 23987 0 0 0 0 0 0.7 99.3 0.0 0.0 0.0 0.0 1199 0.0 0.4 0.0 99.6 0.00 0.0 167 2482920 100.0 0.0 0.0 150 - 4 5 0 0 351 56903 784 2400095 16 23999 23965 0 0 0 0 0 0.0 100.0 0.0 0.0 0.0 0.0 7 0.0 1.0 0.0 99.0 0.00 0.0 153 2481522 100.0 0.0 0.0 150 - 5 13 0 0 351 57171 815 2399248 12 23965 23962 0 0 0 0 0 0.0 100.0 0.0 0.0 0.0 0.0 0 0.0 0.4 0.0 99.6 0.00 0.0 145 2480922 100.0 0.0 0.0 150 - 6 23529 102 0 351 56443 805 2401503 6 24217 24124 0 0 0 0 0 0.9 99.1 0.0 0.0 0.0 0.0 461 0.0 1.0 0.0 99.0 0.00 0.0 162 2482459 100.0 0.0 0.0 150 - 7 1523 75 0 351 56335 783 4150673 11 24209 24040 0 0 0 0 0 1.1 98.9 0.0 0.0 0.0 0.0 471 0.0 0.9 0.0 99.1 0.00 0.0 479 4231232 100.0 0.0 0.0 150 - U - - - - - - - - - - - - - - - - - - - - - - - - 0.0 99.9 0.25 99.8 - - - - - -ALL 22634937 13351 2457 2457 528249 14078302 57184701 120 25219967 615268 1 3 0 0 0 99.3 0.7 0.0 0.0 0.0 0.0 78160659 0.0 0.0 0.0 99.9 0.00 0.2 240028 59117897 100.0 0.0 0.0 0 diff --git a/plugins/sysmon-aix/src/test/resources/mpstat2.txt b/plugins/sysmon-aix/src/test/resources/mpstat2.txt deleted file mode 100644 index 0807efe..0000000 --- a/plugins/sysmon-aix/src/test/resources/mpstat2.txt +++ /dev/null @@ -1,15 +0,0 @@ - -System configuration: lcpu=8 ent=0.2 mode=Uncapped - -cpu min maj mpcs mpcr dev soft dec ph cs ics bound rq push S3pull S3grd S0rd S1rd S2rd S3rd S4rd S5rd sysc us sy wa id pc %ec ilcs vlcs S3hrd S4hrd S5hrd -cpu min maj mpcs mpcr dev soft dec ph cs ics bound rq push S3pull S3grd S0rd S1rd S2rd S3rd S4rd S5rd sysc us sy wa id pc %ec ilcs vlcs S3hrd S4hrd S5hrd %nsp - 0 22334111 14138 2450 1 117391 14078152 37441241 4 25056579 446411 1 4 0 0 0 99.9 0.1 0.0 0.0 0.0 0.0 77590574 66.7 18.4 0.0 14.9 0.00 0.1 237925 38789423 100.0 0.0 0.0 150 - 1 275936 176 7 350 68538 1505 3605599 41 31080 25411 0 0 0 0 0 24.5 75.5 0.0 0.0 0.0 0.0 608541 0.2 0.8 0.0 99.0 0.00 0.0 916 3702787 100.0 0.0 0.0 150 - 2 4713 1 0 351 58375 833 2401267 13 24160 23999 0 0 0 0 0 0.7 99.3 0.0 0.0 0.0 0.0 1445 0.0 1.0 0.0 99.0 0.00 0.0 163 2484279 100.0 0.0 0.0 150 - 3 4587 2 0 351 57986 836 2401339 17 24102 23995 0 0 0 0 0 0.6 99.4 0.0 0.0 0.0 0.0 1199 0.0 0.4 0.0 99.6 0.00 0.0 167 2483932 100.0 0.0 0.0 150 - 4 5 0 0 351 57114 786 2400891 16 24007 23973 0 0 0 0 0 0.0 100.0 0.0 0.0 0.0 0.0 7 0.0 1.0 0.0 99.0 0.00 0.0 154 2482537 100.0 0.0 0.0 150 - 5 13 0 0 351 57379 815 2400045 12 23973 23970 0 0 0 0 0 0.0 100.0 0.0 0.0 0.0 0.0 0 0.0 0.4 0.0 99.6 0.00 0.0 145 2481936 100.0 0.0 0.0 150 - 6 23529 102 0 351 56651 805 2402300 6 24225 24132 0 0 0 0 0 0.9 99.1 0.0 0.0 0.0 0.0 461 0.0 1.0 0.0 99.0 0.00 0.0 162 2483472 100.0 0.0 0.0 150 - 7 1523 75 0 351 56544 783 4152029 11 24217 24048 0 0 0 0 0 1.1 98.9 0.0 0.0 0.0 0.0 471 0.0 0.9 0.0 99.1 0.00 0.0 479 4232805 100.0 0.0 0.0 150 - U - - - - - - - - - - - - - - - - - - - - - - - - 0.0 99.9 0.25 99.8 - - - - - -ALL 22644417 14494 2457 2457 529978 14084515 57204711 120 25232343 615939 1 4 0 0 0 99.3 0.7 0.0 0.0 0.0 0.0 78202698 0.0 0.0 0.0 99.9 0.00 0.2 240111 59141171 100.0 0.0 0.0 0 diff --git a/plugins/sysmon-aix/src/test/resources/vmstat-Iwt.txt b/plugins/sysmon-aix/src/test/resources/vmstat-Iwt.txt new file mode 100644 index 0000000..e294e51 --- /dev/null +++ b/plugins/sysmon-aix/src/test/resources/vmstat-Iwt.txt @@ -0,0 +1,7 @@ + +System configuration: lcpu=8 mem=4096MB ent=0.50 + + kthr memory page faults cpu time +----------- --------------------- ------------------------------------ ------------------ ----------------------- -------- + r b p avm fre fi fo pi po fr sr in sy cs us sy id wa pc ec hr mi se + 2 1 0 633739 130144 0 13 0 0 2 2 7 969 529 0 0 99 0 0.00 0.1 07:53:28 diff --git a/plugins/sysmon-linux/src/main/java/org/sysmon/plugins/sysmon_linux/LinuxDiskExtension.java b/plugins/sysmon-linux/src/main/java/org/sysmon/plugins/sysmon_linux/LinuxDiskExtension.java index 7505c5f..bce8138 100644 --- a/plugins/sysmon-linux/src/main/java/org/sysmon/plugins/sysmon_linux/LinuxDiskExtension.java +++ b/plugins/sysmon-linux/src/main/java/org/sysmon/plugins/sysmon_linux/LinuxDiskExtension.java @@ -47,7 +47,7 @@ public class LinuxDiskExtension implements MetricExtension { try { copyCurrentValues(); readProcFile(); - result.addMeasurements(calculate()); + result.setMeasurement(calculate()); } catch (IOException e) { e.printStackTrace(); } @@ -77,14 +77,15 @@ public class LinuxDiskExtension implements MetricExtension { } - private List calculate() { - - List measurementList = new ArrayList<>(); + private Measurement calculate() { if(previousDiskStats == null || previousDiskStats.size() != currentDiskStats.size()) { - return measurementList; + return null; } + HashMap tagsMap = new HashMap<>(); + HashMap fieldsMap = new HashMap<>(); + for(int i = 0; i < currentDiskStats.size(); i++) { LinuxDiskStat curStat = currentDiskStats.get(i); @@ -98,23 +99,16 @@ public class LinuxDiskExtension implements MetricExtension { }); if(!ignore.get()) { - HashMap tagsMap = new HashMap<>(); - tagsMap.put("device", curStat.getDevice()); - - HashMap fieldsMap = new HashMap<>(); - fieldsMap.put("iotime", curStat.getTimeSpentOnIo() - preStat.getTimeSpentOnIo()); - fieldsMap.put("readtime", curStat.getTimeSpentReading() - preStat.getTimeSpentReading()); - fieldsMap.put("writetime", curStat.getTimeSpentWriting() - preStat.getTimeSpentWriting()); - fieldsMap.put("reads", curStat.getSectorsRead() - preStat.getSectorsRead()); - fieldsMap.put("writes", curStat.getSectorsWritten() - preStat.getSectorsWritten()); - - measurementList.add(new Measurement(tagsMap, fieldsMap)); - + fieldsMap.put(curStat.getDevice() + "_iotime", curStat.getTimeSpentOnIo() - preStat.getTimeSpentOnIo()); + //fieldsMap.put(curStat.getDevice() + "_readtime", curStat.getTimeSpentReading() - preStat.getTimeSpentReading()); + //fieldsMap.put(curStat.getDevice() + "_writetime", curStat.getTimeSpentWriting() - preStat.getTimeSpentWriting()); + fieldsMap.put(curStat.getDevice() + "_reads", curStat.getSectorsRead() - preStat.getSectorsRead()); + fieldsMap.put(curStat.getDevice() + "_writes", curStat.getSectorsWritten() - preStat.getSectorsWritten()); } } - return measurementList; + return new Measurement(tagsMap, fieldsMap); } } diff --git a/plugins/sysmon-linux/src/main/java/org/sysmon/plugins/sysmon_linux/LinuxMemoryExtension.java b/plugins/sysmon-linux/src/main/java/org/sysmon/plugins/sysmon_linux/LinuxMemoryExtension.java index 5449bc7..2829238 100644 --- a/plugins/sysmon-linux/src/main/java/org/sysmon/plugins/sysmon_linux/LinuxMemoryExtension.java +++ b/plugins/sysmon-linux/src/main/java/org/sysmon/plugins/sysmon_linux/LinuxMemoryExtension.java @@ -42,7 +42,7 @@ public class LinuxMemoryExtension implements MetricExtension { MetricResult result = new MetricResult("memory"); try { - result.addMeasurements(readProcFile()); + result.setMeasurement(readProcFile()); } catch (IOException e) { e.printStackTrace(); } @@ -51,9 +51,7 @@ public class LinuxMemoryExtension implements MetricExtension { } - private List readProcFile() throws IOException { - - List measurementList = new ArrayList<>(); + private Measurement readProcFile() throws IOException { Map tagsMap = new HashMap<>(); Map fieldsMap = new HashMap<>(); @@ -73,8 +71,7 @@ public class LinuxMemoryExtension implements MetricExtension { } - measurementList.add(new Measurement(tagsMap, fieldsMap)); - return measurementList; + return new Measurement(tagsMap, fieldsMap); } } \ No newline at end of file diff --git a/plugins/sysmon-linux/src/main/java/org/sysmon/plugins/sysmon_linux/LinuxProcessorExtension.java b/plugins/sysmon-linux/src/main/java/org/sysmon/plugins/sysmon_linux/LinuxProcessorExtension.java index e642ea2..a67ea2a 100644 --- a/plugins/sysmon-linux/src/main/java/org/sysmon/plugins/sysmon_linux/LinuxProcessorExtension.java +++ b/plugins/sysmon-linux/src/main/java/org/sysmon/plugins/sysmon_linux/LinuxProcessorExtension.java @@ -44,31 +44,20 @@ public class LinuxProcessorExtension implements MetricExtension { @Override public MetricResult getMetrics() { - if(currentProcessorProc != null && currentProcessorProc.size() > 0) { - previousProcessorProc = new ArrayList<>(currentProcessorProc); - } - currentProcessorProc = processFileOutput(readProcFile()); + LinuxProcessorProcLine proc1 = processFileOutput(readProcFile()); - MetricResult result = new MetricResult("processor"); - if(previousProcessorProc == null || previousProcessorProc.size() != currentProcessorProc.size()) { - return result; + try { + Thread.sleep(1 * 1000); // TODO: Configure sample collect time + } catch (InterruptedException e) { + log.warn("getMetrics() - sleep interrupted"); + return null; } + LinuxProcessorProcLine proc2 = processFileOutput(readProcFile()); - List measurementList = new ArrayList<>(); - for(int i = 0; i < currentProcessorProc.size(); i++) { - LinuxProcessorStat stat = new LinuxProcessorStat(currentProcessorProc.get(i), previousProcessorProc.get(i)); + LinuxProcessorStat stat = new LinuxProcessorStat(proc2, proc1); - Map tagsMap = new HashMap<>(); - tagsMap.put("cpu", stat.getName()); - - Map fieldsMap = stat.getFields(); - - measurementList.add(new Measurement(tagsMap, fieldsMap)); - } - - result.addMeasurements(measurementList); - return result; + return new MetricResult("processor", new Measurement(stat.getTags(), stat.getFields())); } @@ -86,16 +75,15 @@ public class LinuxProcessorExtension implements MetricExtension { } - protected List processFileOutput(List inputLines) { + protected LinuxProcessorProcLine processFileOutput(List inputLines) { - List processorStats = new ArrayList<>(); for(String line : inputLines) { - if(line.matches("^cpu\\d+.*")) { - processorStats.add(new LinuxProcessorProcLine(line)); + if(line.matches("^cpu\\S+.*")) { + return new LinuxProcessorProcLine(line); } } - return processorStats; + return null; } diff --git a/plugins/sysmon-linux/src/main/java/org/sysmon/plugins/sysmon_linux/LinuxProcessorStat.java b/plugins/sysmon-linux/src/main/java/org/sysmon/plugins/sysmon_linux/LinuxProcessorStat.java index 27a1d6a..37f3c6e 100644 --- a/plugins/sysmon-linux/src/main/java/org/sysmon/plugins/sysmon_linux/LinuxProcessorStat.java +++ b/plugins/sysmon-linux/src/main/java/org/sysmon/plugins/sysmon_linux/LinuxProcessorStat.java @@ -6,7 +6,11 @@ import java.util.Map; public class LinuxProcessorStat { private final String cpuName; - private final float utilizationPercentage; + //private final float user; + //private final float sys; + //private final float wait; + //private final float idle; + private final float busy; public LinuxProcessorStat(LinuxProcessorProcLine current, LinuxProcessorProcLine previous) { cpuName = current.getCpuName(); @@ -15,7 +19,9 @@ public class LinuxProcessorStat { long idleTimeDiff = current.getCombinedIdleTime() - previous.getCombinedIdleTime(); float utilization = (float) (workTimeDiff - idleTimeDiff) / workTimeDiff; - utilizationPercentage = (utilization * 100); + busy = (utilization * 100); + + // TODO: Calculate user, system, idle and wait diff times into percentage. } @@ -24,13 +30,18 @@ public class LinuxProcessorStat { } + public Float getBusy() { + return busy; + } + + public Map getTags() { + return new HashMap<>(); + } + public Map getFields() { - - HashMap fieldsMap = new HashMap<>(); - fieldsMap.put("utilization", utilizationPercentage); - - return fieldsMap; - + Map fields = new HashMap<>(); + fields.put("busy", busy); + return fields; } diff --git a/plugins/sysmon-linux/src/test/groovy/LinuxProcessorTest.groovy b/plugins/sysmon-linux/src/test/groovy/LinuxProcessorTest.groovy index 6d79718..4627c2c 100644 --- a/plugins/sysmon-linux/src/test/groovy/LinuxProcessorTest.groovy +++ b/plugins/sysmon-linux/src/test/groovy/LinuxProcessorTest.groovy @@ -13,13 +13,13 @@ class LinuxProcessorTest extends Specification { when: LinuxProcessorExtension extension = new LinuxProcessorExtension() - List procLines = extension.processFileOutput(lines) + LinuxProcessorProcLine procLine = extension.processFileOutput(lines) then: - procLines[0].getSystemTime() == 4686l - procLines[0].getUserTime() == 27477l - procLines[0].getIdleTime() == 281276l - procLines[0].getIoWaitTime() == 252l + procLine.getSystemTime() == 4686l + procLine.getUserTime() == 27477l + procLine.getIdleTime() == 281276l + procLine.getIoWaitTime() == 252l } @@ -29,15 +29,14 @@ class LinuxProcessorTest extends Specification { setup: def testFile1 = new File(getClass().getResource('/proc1.txt').toURI()) def testFile2 = new File(getClass().getResource('/proc2.txt').toURI()) - LinuxProcessorProcLine processorProcLine1 = new LinuxProcessorProcLine(testFile1.readLines().get(1)) - LinuxProcessorProcLine processorProcLine2 = new LinuxProcessorProcLine(testFile2.readLines().get(1)) + LinuxProcessorProcLine processorProcLine1 = new LinuxProcessorProcLine(testFile1.readLines().get(0)) + LinuxProcessorProcLine processorProcLine2 = new LinuxProcessorProcLine(testFile2.readLines().get(0)) when: LinuxProcessorStat processorStat = new LinuxProcessorStat(processorProcLine1, processorProcLine2) then: - processorStat.getName() == "cpu0" - processorStat.getFields().get("utilization") == 42.13362f + processorStat.getBusy() == 38.001614f } diff --git a/server/build.gradle b/server/build.gradle index 9c4d46e..52858c4 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -3,7 +3,7 @@ plugins { id "com.github.johnrengelman.shadow" version "6.1.0" id "net.nemerosa.versioning" version "2.14.0" - id "nebula.ospackage" version "8.4.1" + id "nebula.ospackage" version "8.5.6" } dependencies { @@ -64,10 +64,15 @@ buildRpm { os = "LINUX" } + buildDeb { dependsOn startShadowScripts } +task aixRpm(type: Rpm) { + os "AIX" +} + jar { manifest { attributes( diff --git a/server/src/main/java/org/sysmon/server/MetricResultToPointProcessor.java b/server/src/main/java/org/sysmon/server/MetricResultToPointProcessor.java index 3a0743d..1853cd9 100644 --- a/server/src/main/java/org/sysmon/server/MetricResultToPointProcessor.java +++ b/server/src/main/java/org/sysmon/server/MetricResultToPointProcessor.java @@ -2,6 +2,7 @@ package org.sysmon.server; import org.apache.camel.Exchange; import org.apache.camel.Processor; +import org.influxdb.dto.BatchPoints; import org.influxdb.dto.Point; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -20,37 +21,32 @@ public class MetricResultToPointProcessor implements Processor { public void process(Exchange exchange) throws Exception { MetricResult metricResult = exchange.getIn().getBody(MetricResult.class); + Measurement measurement = metricResult.getMeasurement(); Point.Builder builder = Point.measurement(metricResult.getName()) .time(metricResult.getTimestamp(), TimeUnit.MILLISECONDS) .tag("hostname", metricResult.getHostname()); - List measurements = metricResult.getMeasurements(); - for(Measurement measurement : measurements) { - - for (Map.Entry entry : measurement.getTags().entrySet()) { - log.debug("process() - tag: " + entry.getKey() + "=" + entry.getValue()); - builder.tag(entry.getKey(), entry.getValue()); - } - - - for (Map.Entry entry : measurement.getFields().entrySet()) { - log.debug("process() - field: " + entry.getKey() + "=" + entry.getValue()); - if(entry.getValue() instanceof Number) { - Number num = (Number) entry.getValue(); - builder.addField(entry.getKey(), num); - } else if(entry.getValue() instanceof Boolean) { - Boolean bol = (Boolean) entry.getValue(); - builder.addField(entry.getKey(), bol); - } else { - String str = (String) entry.getValue(); - builder.addField(entry.getKey(), str); - } - - } - + for (Map.Entry entry : measurement.getTags().entrySet()) { + log.debug("process() - tag: " + entry.getKey() + "=" + entry.getValue()); + builder.tag(entry.getKey(), entry.getValue()); } + for (Map.Entry entry : measurement.getFields().entrySet()) { + log.debug("process() - field: " + entry.getKey() + "=" + entry.getValue()); + if(entry.getValue() instanceof Number) { + Number num = (Number) entry.getValue(); + builder.addField(entry.getKey(), num); + } else if(entry.getValue() instanceof Boolean) { + Boolean bol = (Boolean) entry.getValue(); + builder.addField(entry.getKey(), bol); + } else { + String str = (String) entry.getValue(); + builder.addField(entry.getKey(), str); + } + } + + exchange.getIn().setBody(builder.build()); } diff --git a/server/src/main/java/org/sysmon/server/ServerRouteBuilder.java b/server/src/main/java/org/sysmon/server/ServerRouteBuilder.java index 23c43fa..39e3a46 100644 --- a/server/src/main/java/org/sysmon/server/ServerRouteBuilder.java +++ b/server/src/main/java/org/sysmon/server/ServerRouteBuilder.java @@ -36,10 +36,9 @@ public class ServerRouteBuilder extends RouteBuilder { //from("seda:inbound").log("Got metric from: ${header.component}").to("mock:sink"); from("seda:inbound") - .log(">>> metric: ${header.hostname} - ${header.metric}") + .log(">>> metric: ${header.hostname} - ${body}") .doTry() .process(new MetricResultToPointProcessor()) - .log("${body}") .to("influxdb://myInfluxConnection?databaseName=sysmon&retentionPolicy=autogen") .doCatch(Exception.class) .log("Error storing metric to InfluxDB: ${exception}") diff --git a/shared/src/main/java/org/sysmon/shared/Measurement.java b/shared/src/main/java/org/sysmon/shared/Measurement.java index f2955f3..0e749d8 100644 --- a/shared/src/main/java/org/sysmon/shared/Measurement.java +++ b/shared/src/main/java/org/sysmon/shared/Measurement.java @@ -2,6 +2,7 @@ package org.sysmon.shared; import java.util.HashMap; import java.util.Map; +import java.util.Objects; public class Measurement { @@ -13,8 +14,8 @@ public class Measurement { } public Measurement(Map tags, Map fields) { - this.tags = tags; - this.fields = fields; + this.tags = Objects.requireNonNull(tags); + this.fields = Objects.requireNonNull(fields); } public Map getTags() { @@ -26,10 +27,12 @@ public class Measurement { } public void setTags(Map tags) { + Objects.requireNonNull(tags); this.tags = tags; } public void setFields(Map fields) { + Objects.requireNonNull(fields); this.fields = fields; } diff --git a/shared/src/main/java/org/sysmon/shared/MetricResult.java b/shared/src/main/java/org/sysmon/shared/MetricResult.java index f6366f5..028c1ab 100644 --- a/shared/src/main/java/org/sysmon/shared/MetricResult.java +++ b/shared/src/main/java/org/sysmon/shared/MetricResult.java @@ -2,8 +2,6 @@ package org.sysmon.shared; import java.io.Serializable; import java.time.Instant; -import java.util.ArrayList; -import java.util.List; import java.util.Map; public class MetricResult implements Serializable { @@ -11,9 +9,9 @@ public class MetricResult implements Serializable { private static final long serialVersionUID = 1L; private String name; - private Long timestamp; // epoch milli private String hostname; - private List measurements = new ArrayList<>(); + private Long timestamp; // epoch milli + private Measurement measurement; public MetricResult() { } @@ -23,12 +21,14 @@ public class MetricResult implements Serializable { this.timestamp = Instant.now().toEpochMilli(); } - public void addMeasurements(List measurementList) { - this.measurements = measurementList; + public MetricResult(String name, Measurement measurement) { + this.name = name; + this.timestamp = Instant.now().toEpochMilli(); + this.measurement = measurement; } - public void addMeasurement(Measurement measurement) { - measurements.add(measurement); + public void setMeasurement(Measurement measurement) { + this.measurement = measurement; } public void setHostname(String hostname) { @@ -55,24 +55,24 @@ public class MetricResult implements Serializable { return hostname; } - public List getMeasurements() { - return measurements; + public Measurement getMeasurement() { + return measurement; } public String toString() { - StringBuilder sb = new StringBuilder(String.format("%s - %s\n", timestamp.toString(), name)); - for(Measurement m : measurements) { + StringBuilder sb = new StringBuilder(String.format("%s - %s {", timestamp.toString(), name)); - for (Map.Entry entry : m.getTags().entrySet()) - sb.append(entry.getKey() + " : " + entry.getValue()); - - for (Map.Entry entry : m.getFields().entrySet()) - sb.append(entry.getKey() + " : " + entry.getValue()); - - sb.append(m.toString()).append("\n"); + if(measurement != null && measurement.getTags() != null) { + for (Map.Entry entry : measurement.getTags().entrySet()) + sb.append(" [").append(entry.getKey()).append(": ").append(entry.getValue()).append("]"); } - return sb.toString(); + if(measurement != null && measurement.getFields() != null) { + for (Map.Entry entry : measurement.getFields().entrySet()) + sb.append(" [").append(entry.getKey()).append(": ").append(entry.getValue()).append("]"); + } + + return sb.append(" }").toString(); } } diff --git a/shared/src/main/java/org/sysmon/shared/PluginHelper.java b/shared/src/main/java/org/sysmon/shared/PluginHelper.java index c7c1bd8..a11e77c 100644 --- a/shared/src/main/java/org/sysmon/shared/PluginHelper.java +++ b/shared/src/main/java/org/sysmon/shared/PluginHelper.java @@ -1,5 +1,8 @@ package org.sysmon.shared; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.io.BufferedReader; import java.io.File; import java.io.IOException; @@ -9,6 +12,8 @@ import java.util.List; public class PluginHelper { + private static final Logger log = LoggerFactory.getLogger(PluginHelper.class); + final static boolean isWindows = System.getProperty("os.name") .toLowerCase().startsWith("windows"); @@ -29,11 +34,9 @@ public class PluginHelper { } builder.directory(new File(System.getProperty("user.home"))); - try { Process process = builder.start(); - BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream())); @@ -43,7 +46,9 @@ public class PluginHelper { } int exitCode = process.waitFor(); - System.out.println("\nExited with error code : " + exitCode); + if(exitCode > 0) { + log.warn("executeCommand() - exit code: " + exitCode); + } } catch (IOException | InterruptedException e) { e.printStackTrace();