From 0361331268e8b8bc41db20d0184ddc3ded7a49b5 Mon Sep 17 00:00:00 2001 From: Mark Nellemann Date: Fri, 10 Sep 2021 12:15:33 +0200 Subject: [PATCH 1/2] Support for configuring extensions. --- build.gradle | 1 + client/build.gradle | 3 +- client/doc/sysmon-client.toml | 10 +++ .../main/java/sysmon/client/Application.java | 23 +++++++ .../sysmon/client/ClientRouteBuilder.java | 18 ++++-- .../java/sysmon/client/Configuration.java | 64 +++++++++++++++++++ ...mon - Process Explorer-1631263229100.json} | 61 ++++++++---------- .../plugins/os_aix/AixNetstatExtension.java | 29 +++++++-- .../plugins/os_aix/AixProcessorExtension.java | 30 +++++++-- plugins/os-base/README.md | 29 ++++++++- .../plugins/os_base/BaseDiskExtension.java | 28 ++++++-- .../plugins/os_base/BaseMemoryExtension.java | 30 +++++++-- .../plugins/os_base/BaseNetworkExtension.java | 29 +++++++-- .../plugins/os_base/BaseProcessExtension.java | 46 ++++++++----- .../os_base/BaseProcessorExtension.java | 34 ++++++---- .../sysmon/plugins/os_ibmi/TestExtension.java | 28 +++++++- .../os_linux/LinuxNetstatExtension.java | 29 +++++++-- .../os_linux/LinuxSockstatExtension.java | 29 +++++++-- server/build.gradle | 2 +- .../sysmon/server/ServerRouteBuilder.java | 2 +- .../java/sysmon/shared/MetricExtension.java | 4 ++ 21 files changed, 426 insertions(+), 103 deletions(-) create mode 100644 client/doc/sysmon-client.toml create mode 100644 client/src/main/java/sysmon/client/Configuration.java rename doc/{Sysmon - Process Explorer-1631013513804.json => Sysmon - Process Explorer-1631263229100.json} (95%) diff --git a/build.gradle b/build.gradle index 00ac202..3175af8 100644 --- a/build.gradle +++ b/build.gradle @@ -19,6 +19,7 @@ subprojects { implementation "org.slf4j:slf4j-api:${slf4jVersion}" implementation "org.slf4j:slf4j-simple:${slf4jVersion}" + implementation 'org.tomlj:tomlj:1.0.0' } diff --git a/client/build.gradle b/client/build.gradle index 325859e..56ba734 100644 --- a/client/build.gradle +++ b/client/build.gradle @@ -37,11 +37,12 @@ def projectName = "sysmon-client" application { // Define the main class for the application. mainClass.set('sysmon.client.Application') - applicationDefaultJvmArgs = [ "-server", "-XX:+UseG1GC", "-Xmx32m" ] + applicationDefaultJvmArgs = [ "-server", "-Xms16m", "-Xmx32m", "-XX:+UseG1GC" ] } run { systemProperty 'pf4j.pluginsDir', '../plugins/output/' + systemProperty 'sysmon.cfgFile', 'doc/sysmon-client.toml' } tasks.named('test') { diff --git a/client/doc/sysmon-client.toml b/client/doc/sysmon-client.toml new file mode 100644 index 0000000..70cf8de --- /dev/null +++ b/client/doc/sysmon-client.toml @@ -0,0 +1,10 @@ +# Configuration for sysmon-client + +[extension.base_disk] +enabled = false + +[extension.base_process] +enabled = true +include = [ "java", "influxd", "grafana-server" ] + + diff --git a/client/src/main/java/sysmon/client/Application.java b/client/src/main/java/sysmon/client/Application.java index d134d6e..07b90fd 100644 --- a/client/src/main/java/sysmon/client/Application.java +++ b/client/src/main/java/sysmon/client/Application.java @@ -8,6 +8,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import picocli.CommandLine; +import java.io.File; import java.io.IOException; import java.net.InetAddress; import java.net.URL; @@ -28,6 +29,9 @@ public class Application implements Callable { @CommandLine.Option(names = { "-p", "--plugin-dir" }, description = "Plugin jar path (default: ${DEFAULT-VALUE}).", paramLabel = "", defaultValue = "/opt/sysmon/plugins") private String pluginPath; + @CommandLine.Option(names = { "-c", "--conf" }, description = "Configuration file [default: '/etc/sysmon-client.toml'].", paramLabel = "", defaultValue = "/etc/sysmon-client.toml") + private File configurationFile; + public static void main(String... args) { int exitCode = new CommandLine(new Application()).execute(args); System.exit(exitCode); @@ -37,6 +41,8 @@ public class Application implements Callable { @Override public Integer call() throws IOException { + Configuration configuration = new Configuration(); + if(hostname == null || hostname.isEmpty()) { try { hostname = InetAddress.getLocalHost().getHostName(); @@ -51,10 +57,26 @@ public class Application implements Callable { pluginPath = pf4jPluginsDir; } + String sysmonCfgFile = System.getProperty("sysmon.cfgFile"); + if(sysmonCfgFile != null) { + configurationFile = new File(sysmonCfgFile); + } + + + if(configurationFile.exists()) { + try { + configuration.parse(configurationFile.toPath()); + } catch (Exception e) { + System.err.println(e.getMessage()); + return 1; + } + } + Main main = new Main(); main.bind("pluginPath", pluginPath); main.bind("myServerUrl", serverUrl.toString()); main.bind("myHostname", hostname); + main.bind("configuration", configuration); main.configure().addRoutesBuilder(ClientRouteBuilder.class); // now keep the application running until the JVM is terminated (ctrl + c or sigterm) @@ -62,6 +84,7 @@ public class Application implements Callable { main.run(); } catch (Exception e) { System.err.println(e.getMessage()); + return 1; } return 0; diff --git a/client/src/main/java/sysmon/client/ClientRouteBuilder.java b/client/src/main/java/sysmon/client/ClientRouteBuilder.java index d5f15dd..7b08153 100644 --- a/client/src/main/java/sysmon/client/ClientRouteBuilder.java +++ b/client/src/main/java/sysmon/client/ClientRouteBuilder.java @@ -24,6 +24,7 @@ public class ClientRouteBuilder extends RouteBuilder { public void configure() { Registry registry = getContext().getRegistry(); + Configuration configuration = (Configuration) registry.lookupByName("configuration"); Path[] pluginpaths = { Paths.get(registry.lookupByNameAndType("pluginPath", String.class)) }; PluginManager pluginManager = new JarPluginManager(pluginpaths); @@ -34,9 +35,18 @@ public class ClientRouteBuilder extends RouteBuilder { List metricExtensions = pluginManager.getExtensions(MetricExtension.class); for (MetricExtension ext : metricExtensions) { - if(ext.isSupported()) { + final String name = ext.getName(); + final String provides = ext.getProvides(); - String provides = ext.getProvides(); + // Load configuration if available + if(configuration.isForExtension(name)) { + log.info(">>> Loading configuring for extension: " + ext.getDescription()); + ext.setConfiguration(configuration.getForExtension(name)); + } + + if(ext.isSupported() && ext.isEnabled()) { + + // Check that another extension has not already been loaded - TODO: Is this required ? if(providers.contains(provides)) { log.warn("Skipping extension (already provided): " + ext.getName()); continue; @@ -46,7 +56,7 @@ public class ClientRouteBuilder extends RouteBuilder { providers.add(provides); - // TODO: Make timer thread configurable + // TODO: Make timer thread configurable ? // Setup Camel route for this extension // a unique timer name gives the timer it's own thread, otherwise it's a shared thread for other timers with same name. @@ -62,7 +72,7 @@ public class ClientRouteBuilder extends RouteBuilder { .otherwise() .to("seda:metrics?discardWhenFull=true"); } else { - log.info(">>> Skipping extension (not supported here): " + ext.getDescription()); + log.info(">>> Skipping extension (not supported or disabled): " + ext.getDescription()); } } diff --git a/client/src/main/java/sysmon/client/Configuration.java b/client/src/main/java/sysmon/client/Configuration.java new file mode 100644 index 0000000..f9a4dfe --- /dev/null +++ b/client/src/main/java/sysmon/client/Configuration.java @@ -0,0 +1,64 @@ +package sysmon.client; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.tomlj.Toml; +import org.tomlj.TomlParseResult; +import org.tomlj.TomlTable; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; + +public final class Configuration { + + private final static Logger log = LoggerFactory.getLogger(Configuration.class); + + private TomlParseResult result; + + void parse(Path configurationFile) throws IOException { + log.info("Parsing configuration file: " + configurationFile); + result = Toml.parse(configurationFile); + result.errors().forEach(error -> log.error(error.toString())); + } + + + boolean isForExtension(String extName) { + if(result == null) { + return false; + } + String key = String.format("extension.%s", extName); + return result.contains(key); + } + + + Map getForExtension(String extName) { + + if(result == null) { + log.debug("No configuration file loaded ..."); + return null; + } + + Map map = new HashMap<>(); + + String key = String.format("extension.%s", extName); + TomlTable table = result.getTableOrEmpty(key); + table.keySet().forEach( k -> { + if(table.isString(k)) { + map.put(k, table.getString(k)); + } else if(table.isBoolean(k)) { + map.put(k, table.getBoolean(k)); + } else if(table.isDouble(k)) { + map.put(k, table.getDouble(k)); + } else if(table.isArray(k)) { + map.put(k, Objects.requireNonNull(table.getArray(k)).toList()); + } + + }); + + return map; + } + +} \ No newline at end of file diff --git a/doc/Sysmon - Process Explorer-1631013513804.json b/doc/Sysmon - Process Explorer-1631263229100.json similarity index 95% rename from doc/Sysmon - Process Explorer-1631013513804.json rename to doc/Sysmon - Process Explorer-1631263229100.json index 3e2d159..80e64a4 100644 --- a/doc/Sysmon - Process Explorer-1631013513804.json +++ b/doc/Sysmon - Process Explorer-1631263229100.json @@ -10,17 +10,11 @@ } ], "__requires": [ - { - "type": "panel", - "id": "gauge", - "name": "Gauge", - "version": "" - }, { "type": "grafana", "id": "grafana", "name": "Grafana", - "version": "8.0.6" + "version": "8.1.2" }, { "type": "datasource", @@ -56,6 +50,12 @@ "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, "type": "dashboard" } ] @@ -65,7 +65,7 @@ "gnetId": null, "graphTooltip": 0, "id": null, - "iteration": 1631013505736, + "iteration": 1631256755587, "links": [], "panels": [ { @@ -120,7 +120,7 @@ "showHeader": true, "sortBy": [] }, - "pluginVersion": "8.0.6", + "pluginVersion": "8.1.2", "targets": [ { "groupBy": [ @@ -286,7 +286,7 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "8.0.6", + "pluginVersion": "8.1.2", "targets": [ { "groupBy": [ @@ -377,7 +377,7 @@ "fieldConfig": { "defaults": { "color": { - "mode": "continuous-GrYlRd" + "mode": "thresholds" }, "mappings": [], "thresholds": { @@ -426,6 +426,10 @@ }, "id": 26, "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" @@ -433,11 +437,10 @@ "fields": "", "values": false }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "text": {} + "text": {}, + "textMode": "auto" }, - "pluginVersion": "8.0.6", + "pluginVersion": "8.1.2", "targets": [ { "groupBy": [ @@ -471,10 +474,6 @@ "params": [], "type": "sum" }, - { - "params": [], - "type": "non_negative_difference" - }, { "params": [ "read" @@ -493,10 +492,6 @@ "params": [], "type": "sum" }, - { - "params": [], - "type": "non_negative_difference" - }, { "params": [ "write" @@ -529,7 +524,7 @@ "timeFrom": null, "timeShift": null, "title": "Disk Metrics", - "type": "gauge" + "type": "stat" }, { "datasource": "${DS_INFLUXDB-SYSMON}", @@ -890,7 +885,7 @@ "refresh": 2, "regex": "", "skipUrlSync": false, - "sort": 0, + "sort": 5, "tagValuesQuery": "", "tagsQuery": "", "type": "query", @@ -913,14 +908,14 @@ "refresh": 1, "regex": "", "skipUrlSync": false, - "sort": 0, + "sort": 5, "type": "query" }, { - "allValue": "", + "allValue": null, "current": {}, "datasource": "${DS_INFLUXDB-SYSMON}", - "definition": "SHOW TAG VALUES FROM \"base_process\" WITH KEY = \"pid\" WHERE hostname =~ /$hostname/AND \"name\" =~ /$process/ AND time > now() - 60m", + "definition": "SELECT DISTINCT(\"pid\") FROM (SELECT * FROM \"base_process\" WHERE time > now() - 60m AND \"hostname\" =~ /$hostname/ AND \"name\" =~ /$process/)", "description": null, "error": null, "hide": 0, @@ -929,17 +924,17 @@ "multi": false, "name": "pid", "options": [], - "query": "SHOW TAG VALUES FROM \"base_process\" WITH KEY = \"pid\" WHERE hostname =~ /$hostname/AND \"name\" =~ /$process/ AND time > now() - 60m", - "refresh": 1, + "query": "SELECT DISTINCT(\"pid\") FROM (SELECT * FROM \"base_process\" WHERE time > now() - 60m AND \"hostname\" =~ /$hostname/ AND \"name\" =~ /$process/)", + "refresh": 2, "regex": "", "skipUrlSync": false, - "sort": 0, + "sort": 3, "type": "query" } ] }, "time": { - "from": "now-3h", + "from": "now-6h", "to": "now-30s" }, "timepicker": { @@ -958,5 +953,5 @@ "timezone": "", "title": "Sysmon - Process Explorer", "uid": "Vjut5mS7k", - "version": 11 + "version": 15 } \ No newline at end of file diff --git a/plugins/os-aix/src/main/java/sysmon/plugins/os_aix/AixNetstatExtension.java b/plugins/os-aix/src/main/java/sysmon/plugins/os_aix/AixNetstatExtension.java index ae93a31..fc7e383 100644 --- a/plugins/os-aix/src/main/java/sysmon/plugins/os_aix/AixNetstatExtension.java +++ b/plugins/os-aix/src/main/java/sysmon/plugins/os_aix/AixNetstatExtension.java @@ -19,6 +19,20 @@ public class AixNetstatExtension implements MetricExtension { private static final Logger log = LoggerFactory.getLogger(AixNetstatExtension.class); + // Extension details + private final String name = "aix_network_netstat"; + private final String provides = "network_netstat"; + private final String description = "AIX Netstat Metrics"; + + // Configuration / Options + private boolean enabled = true; + + + @Override + public boolean isEnabled() { + return enabled; + } + @Override public boolean isSupported() { @@ -37,17 +51,24 @@ public class AixNetstatExtension implements MetricExtension { @Override public String getName() { - return "aix_network_netstat"; + return name; } @Override public String getProvides() { - return "network_netstat"; + return provides; } @Override public String getDescription() { - return "AIX Netstat Metrics"; + return description; + } + + @Override + public void setConfiguration(Map map) { + if (map.containsKey("enabled")) { + enabled = (boolean) map.get("enabled"); + } } @Override @@ -63,7 +84,7 @@ public class AixNetstatExtension implements MetricExtension { } log.debug(fieldsMap.toString()); - return new MetricResult(getName(), new Measurement(tagsMap, fieldsMap)); + return new MetricResult(name, new Measurement(tagsMap, fieldsMap)); } diff --git a/plugins/os-aix/src/main/java/sysmon/plugins/os_aix/AixProcessorExtension.java b/plugins/os-aix/src/main/java/sysmon/plugins/os_aix/AixProcessorExtension.java index 5ee57ed..91855d4 100644 --- a/plugins/os-aix/src/main/java/sysmon/plugins/os_aix/AixProcessorExtension.java +++ b/plugins/os-aix/src/main/java/sysmon/plugins/os_aix/AixProcessorExtension.java @@ -11,7 +11,6 @@ import sysmon.shared.PluginHelper; import java.io.IOException; import java.io.InputStream; import java.util.HashMap; -import java.util.List; import java.util.Map; @Extension @@ -19,6 +18,20 @@ public class AixProcessorExtension implements MetricExtension { private static final Logger log = LoggerFactory.getLogger(AixProcessorExtension.class); + // Extension details + private final String name = "aix_processor"; + private final String provides = "lpar_processor"; + private final String description = "AIX Processor Metrics"; + + // Configuration / Options + private boolean enabled = true; + + + @Override + public boolean isEnabled() { + return enabled; + } + @Override public boolean isSupported() { @@ -38,17 +51,24 @@ public class AixProcessorExtension implements MetricExtension { @Override public String getName() { - return "aix_processor"; + return name; } @Override public String getProvides() { - return "processor_lpar"; + return provides; } @Override public String getDescription() { - return "AIX Processor Metrics"; + return description; + } + + @Override + public void setConfiguration(Map map) { + if (map.containsKey("enabled")) { + enabled = (boolean) map.get("enabled"); + } } @Override @@ -63,7 +83,7 @@ public class AixProcessorExtension implements MetricExtension { fieldsMap = processorStat.getFields(); } - return new MetricResult(getName(), new Measurement(tagsMap, fieldsMap)); + return new MetricResult(name, new Measurement(tagsMap, fieldsMap)); } diff --git a/plugins/os-base/README.md b/plugins/os-base/README.md index 966eab0..3a7e139 100644 --- a/plugins/os-base/README.md +++ b/plugins/os-base/README.md @@ -35,4 +35,31 @@ Metrics reported are: - **reads** - The total number of bytes read. - **writes** - The total number of bytes written. - **iotime** - Time spent on IO in milliseconds. -- **queue** - Lenght of IO queue. \ No newline at end of file +- **queue** - Lenght of IO queue. + +## Process Extension + +Reports metrics on one or more running processes. + +- **mem_rss** - Resident set memory in bytes. +- **mem_vsz** - Virtual memory in bytes. +- **kernel_time** - Time spent (in milliseconds) in kernel space. +- **user_time** - Time used (in milliseconds) in user space. +- **read_bytes** - Bytes read by process. +- **write_bytes** - Bytes written by process. +- **files** - Files currently open by process. +- **threads** - Running threads. +- **user** - User running the process. +- **group** - Group running the process +- **prio** - Process priority. + + +### Configuration + +The **include** option let's you specify what processes to report for. + +```toml +[extension.base_process] +enabled = true # true or false +include = [ "java", "influxd", "grafana-server" ] +``` \ No newline at end of file diff --git a/plugins/os-base/src/main/java/sysmon/plugins/os_base/BaseDiskExtension.java b/plugins/os-base/src/main/java/sysmon/plugins/os_base/BaseDiskExtension.java index 9443593..3e78281 100644 --- a/plugins/os-base/src/main/java/sysmon/plugins/os_base/BaseDiskExtension.java +++ b/plugins/os-base/src/main/java/sysmon/plugins/os_base/BaseDiskExtension.java @@ -3,7 +3,6 @@ package sysmon.plugins.os_base; import org.pf4j.Extension; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import oshi.SystemInfo; import oshi.hardware.HWDiskStore; import oshi.hardware.HardwareAbstractionLayer; import sysmon.shared.Measurement; @@ -19,9 +18,22 @@ public class BaseDiskExtension implements MetricExtension { private static final Logger log = LoggerFactory.getLogger(BaseDiskExtension.class); + // Extension details + private final String name = "base_disk"; + private final String provides = "disk"; + private final String description = "Base Disk Metrics"; + + // Configuration / Options + private boolean enabled = true; + private HardwareAbstractionLayer hardwareAbstractionLayer; + @Override + public boolean isEnabled() { + return enabled; + } + @Override public boolean isSupported() { hardwareAbstractionLayer = BasePlugin.getHardwareAbstractionLayer(); @@ -30,19 +42,25 @@ public class BaseDiskExtension implements MetricExtension { @Override public String getName() { - return "base_disk"; + return name; } @Override public String getProvides() { - return "disk"; + return provides; } @Override public String getDescription() { - return "Base Disk Metrics"; + return description; } + @Override + public void setConfiguration(Map map) { + if (map.containsKey("enabled")) { + enabled = (boolean) map.get("enabled"); + } + } @Override public MetricResult getMetrics() { @@ -73,7 +91,7 @@ public class BaseDiskExtension implements MetricExtension { fieldsMap.put("queue", queueLength); log.debug(fieldsMap.toString()); - return new MetricResult(getName(), new Measurement(tagsMap, fieldsMap)); + return new MetricResult(name, new Measurement(tagsMap, fieldsMap)); } } \ No newline at end of file diff --git a/plugins/os-base/src/main/java/sysmon/plugins/os_base/BaseMemoryExtension.java b/plugins/os-base/src/main/java/sysmon/plugins/os_base/BaseMemoryExtension.java index 5058faa..b960099 100644 --- a/plugins/os-base/src/main/java/sysmon/plugins/os_base/BaseMemoryExtension.java +++ b/plugins/os-base/src/main/java/sysmon/plugins/os_base/BaseMemoryExtension.java @@ -3,7 +3,6 @@ package sysmon.plugins.os_base; import org.pf4j.Extension; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import oshi.SystemInfo; import oshi.hardware.HardwareAbstractionLayer; import sysmon.shared.Measurement; import sysmon.shared.MetricExtension; @@ -17,8 +16,22 @@ public class BaseMemoryExtension implements MetricExtension { private static final Logger log = LoggerFactory.getLogger(BaseMemoryExtension.class); + // Extension details + private final String name = "base_memory"; + private final String provides = "memory"; + private final String description = "Base Memory Metrics"; + + // Configuration / Options + private boolean enabled = true; + private HardwareAbstractionLayer hardwareAbstractionLayer; + + @Override + public boolean isEnabled() { + return enabled; + } + @Override public boolean isSupported() { hardwareAbstractionLayer = BasePlugin.getHardwareAbstractionLayer(); @@ -27,17 +40,24 @@ public class BaseMemoryExtension implements MetricExtension { @Override public String getName() { - return "base_memory"; + return name; } @Override public String getProvides() { - return "memory"; + return provides; } @Override public String getDescription() { - return "Base Memory Metrics"; + return description; + } + + @Override + public void setConfiguration(Map map) { + if (map.containsKey("enabled")) { + enabled = (boolean) map.get("enabled"); + } } @Override @@ -57,7 +77,7 @@ public class BaseMemoryExtension implements MetricExtension { fieldsMap.put("virtual", hardwareAbstractionLayer.getMemory().getVirtualMemory().getVirtualInUse()); log.debug(fieldsMap.toString()); - return new MetricResult(getName(), new Measurement(tagsMap, fieldsMap)); + return new MetricResult(name, new Measurement(tagsMap, fieldsMap)); } diff --git a/plugins/os-base/src/main/java/sysmon/plugins/os_base/BaseNetworkExtension.java b/plugins/os-base/src/main/java/sysmon/plugins/os_base/BaseNetworkExtension.java index 1c961f5..744f459 100644 --- a/plugins/os-base/src/main/java/sysmon/plugins/os_base/BaseNetworkExtension.java +++ b/plugins/os-base/src/main/java/sysmon/plugins/os_base/BaseNetworkExtension.java @@ -19,8 +19,22 @@ public class BaseNetworkExtension implements MetricExtension { private static final Logger log = LoggerFactory.getLogger(BaseNetworkExtension.class); + // Extension details + private final String name = "base_network"; + private final String provides = "network"; + private final String description = "Base Network Metrics"; + + // Configuration / Options + private boolean enabled = true; + + private HardwareAbstractionLayer hardwareAbstractionLayer; + @Override + public boolean isEnabled() { + return enabled; + } + @Override public boolean isSupported() { hardwareAbstractionLayer = BasePlugin.getHardwareAbstractionLayer(); @@ -29,17 +43,24 @@ public class BaseNetworkExtension implements MetricExtension { @Override public String getName() { - return "base_network"; + return name; } @Override public String getProvides() { - return "network"; + return provides; } @Override public String getDescription() { - return "Base Network Metrics"; + return description; + } + + @Override + public void setConfiguration(Map map) { + if (map.containsKey("enabled")) { + enabled = (boolean) map.get("enabled"); + } } @Override @@ -75,7 +96,7 @@ public class BaseNetworkExtension implements MetricExtension { fieldsMap.put("txErrors", txErrs); log.debug(fieldsMap.toString()); - return new MetricResult(getName(), new Measurement(tagsMap, fieldsMap)); + return new MetricResult(name, new Measurement(tagsMap, fieldsMap)); } } diff --git a/plugins/os-base/src/main/java/sysmon/plugins/os_base/BaseProcessExtension.java b/plugins/os-base/src/main/java/sysmon/plugins/os_base/BaseProcessExtension.java index 6178053..bdff0e4 100644 --- a/plugins/os-base/src/main/java/sysmon/plugins/os_base/BaseProcessExtension.java +++ b/plugins/os-base/src/main/java/sysmon/plugins/os_base/BaseProcessExtension.java @@ -9,29 +9,31 @@ import sysmon.shared.Measurement; import sysmon.shared.MetricExtension; import sysmon.shared.MetricResult; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; +import java.util.*; @Extension public class BaseProcessExtension implements MetricExtension { private static final Logger log = LoggerFactory.getLogger(BaseProcessorExtension.class); - // TODO: configurable include-list and/or exclude-list of process names - private final List includeList = new ArrayList() {{ + // Extension details + private final String name = "base_process"; + private final String provides = "process"; + private final String description = "Base Process Metrics"; + + // Configuration / Options + private boolean enabled = true; + private List includeList = new ArrayList() {{ add("java"); - add("nginx"); - add("influxd"); - add("dockerd"); - add("containerd"); - add("mysqld"); - add("postgres"); - add("grafana-server"); }}; private SystemInfo systemInfo; + @Override + public boolean isEnabled() { + return enabled; + } + @Override public boolean isSupported() { systemInfo = BasePlugin.getSystemInfo(); @@ -40,19 +42,29 @@ public class BaseProcessExtension implements MetricExtension { @Override public String getName() { - return "base_process"; + return name; } @Override public String getProvides() { - return "process"; + return provides; } @Override public String getDescription() { - return "Base Process Metrics"; + return description; } + @Override + public void setConfiguration(Map map) { + if(map.containsKey("enabled")) { + enabled = (boolean) map.get("enabled"); + } + if(map.containsKey("include")) { + includeList = (List) map.get("include"); + } + log.info(includeList.toString()); + } @Override public MetricResult getMetrics() { @@ -71,7 +83,7 @@ public class BaseProcessExtension implements MetricExtension { if(!includeList.contains(name)) { continue; } - log.debug("pid: " + p.getProcessID() + ", name: " + name + ", virt: " + p.getVirtualSize() + " rss: " + p.getResidentSetSize() + " cmd: " + p.getCommandLine()); + log.debug("pid: " + p.getProcessID() + ", name: " + name + ", virt: " + p.getVirtualSize() + " rss: " + p.getResidentSetSize()); HashMap tagsMap = new HashMap<>(); HashMap fieldsMap = new HashMap<>(); @@ -95,7 +107,7 @@ public class BaseProcessExtension implements MetricExtension { } //log.info("Size of measurements: " + measurementList.size()); - return new MetricResult(getName(), measurementList); + return new MetricResult(name, measurementList); } } diff --git a/plugins/os-base/src/main/java/sysmon/plugins/os_base/BaseProcessorExtension.java b/plugins/os-base/src/main/java/sysmon/plugins/os_base/BaseProcessorExtension.java index 76cf809..3127c5f 100644 --- a/plugins/os-base/src/main/java/sysmon/plugins/os_base/BaseProcessorExtension.java +++ b/plugins/os-base/src/main/java/sysmon/plugins/os_base/BaseProcessorExtension.java @@ -3,20 +3,13 @@ package sysmon.plugins.os_base; import org.pf4j.Extension; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import oshi.SystemInfo; import oshi.hardware.CentralProcessor; import oshi.hardware.HardwareAbstractionLayer; import sysmon.shared.Measurement; import sysmon.shared.MetricExtension; import sysmon.shared.MetricResult; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.util.ArrayList; import java.util.HashMap; -import java.util.List; import java.util.Map; @Extension @@ -24,9 +17,22 @@ public class BaseProcessorExtension implements MetricExtension { private static final Logger log = LoggerFactory.getLogger(BaseProcessorExtension.class); + // Extension details + private final String name = "base_processor"; + private final String provides = "processor"; + private final String description = "Base Processor Metrics"; + + // Configuration / Options + private boolean enabled = true; + private HardwareAbstractionLayer hardwareAbstractionLayer; private long[] oldTicks; + @Override + public boolean isEnabled() { + return enabled; + } + @Override public boolean isSupported() { hardwareAbstractionLayer = BasePlugin.getHardwareAbstractionLayer(); @@ -35,19 +41,25 @@ public class BaseProcessorExtension implements MetricExtension { @Override public String getName() { - return "base_processor"; + return name; } @Override public String getProvides() { - return "processor"; + return provides; } @Override public String getDescription() { - return "Base Processor Metrics"; + return description; } + @Override + public void setConfiguration(Map map) { + if (map.containsKey("enabled")) { + enabled = (boolean) map.get("enabled"); + } + } @Override public MetricResult getMetrics() { @@ -86,7 +98,7 @@ public class BaseProcessorExtension implements MetricExtension { oldTicks = ticks; log.debug(fieldsMap.toString()); - return new MetricResult(getName(), new Measurement(tagsMap, fieldsMap)); + return new MetricResult(name, new Measurement(tagsMap, fieldsMap)); } } diff --git a/plugins/os-ibmi/src/main/java/sysmon/plugins/os_ibmi/TestExtension.java b/plugins/os-ibmi/src/main/java/sysmon/plugins/os_ibmi/TestExtension.java index 1a0c39c..6978d92 100644 --- a/plugins/os-ibmi/src/main/java/sysmon/plugins/os_ibmi/TestExtension.java +++ b/plugins/os-ibmi/src/main/java/sysmon/plugins/os_ibmi/TestExtension.java @@ -8,6 +8,7 @@ import sysmon.shared.MetricExtension; import sysmon.shared.MetricResult; import java.io.IOException; +import java.util.Map; // Disable for now... //@Extension @@ -15,9 +16,23 @@ public class TestExtension implements MetricExtension { private static final Logger log = LoggerFactory.getLogger(TestExtension.class); + // Extension details + private final String name = "ibmi_test"; + private final String provides = "ibmi_test"; + private final String description = "IBM i Test Metrics"; + + // Configuration / Options + private boolean enabled = true; + private AS400 as400; private SystemStatus systemStatus; + + @Override + public boolean isEnabled() { + return enabled; + } + @Override public boolean isSupported() { @@ -47,17 +62,24 @@ public class TestExtension implements MetricExtension { @Override public String getName() { - return "ibmi-test"; + return name; } @Override public String getProvides() { - return "test"; + return provides; } @Override public String getDescription() { - return "IBM i Test Extension"; + return description; + } + + @Override + public void setConfiguration(Map map) { + if (map.containsKey("enabled")) { + enabled = (boolean) map.get("enabled"); + } } @Override diff --git a/plugins/os-linux/src/main/java/sysmon/plugins/os_linux/LinuxNetstatExtension.java b/plugins/os-linux/src/main/java/sysmon/plugins/os_linux/LinuxNetstatExtension.java index 1b561e1..79b73e2 100644 --- a/plugins/os-linux/src/main/java/sysmon/plugins/os_linux/LinuxNetstatExtension.java +++ b/plugins/os-linux/src/main/java/sysmon/plugins/os_linux/LinuxNetstatExtension.java @@ -19,6 +19,20 @@ public class LinuxNetstatExtension implements MetricExtension { private static final Logger log = LoggerFactory.getLogger(LinuxNetstatExtension.class); + // Extension details + private final String name = "linux_network_netstat"; + private final String provides = "network_netstat"; + private final String description = "Linux Netstat Metrics"; + + // Configuration / Options + private boolean enabled = true; + + + @Override + public boolean isEnabled() { + return enabled; + } + @Override public boolean isSupported() { @@ -37,17 +51,24 @@ public class LinuxNetstatExtension implements MetricExtension { @Override public String getName() { - return "linux_network_netstat"; + return name; } @Override public String getProvides() { - return "network_netstat"; + return provides; } @Override public String getDescription() { - return "Linux Netstat Metrics"; + return description; + } + + @Override + public void setConfiguration(Map map) { + if (map.containsKey("enabled")) { + enabled = (boolean) map.get("enabled"); + } } @Override @@ -63,7 +84,7 @@ public class LinuxNetstatExtension implements MetricExtension { } log.debug(fieldsMap.toString()); - return new MetricResult(getName(), new Measurement(tagsMap, fieldsMap)); + return new MetricResult(name, new Measurement(tagsMap, fieldsMap)); } diff --git a/plugins/os-linux/src/main/java/sysmon/plugins/os_linux/LinuxSockstatExtension.java b/plugins/os-linux/src/main/java/sysmon/plugins/os_linux/LinuxSockstatExtension.java index 46e6dc9..adfc8dc 100644 --- a/plugins/os-linux/src/main/java/sysmon/plugins/os_linux/LinuxSockstatExtension.java +++ b/plugins/os-linux/src/main/java/sysmon/plugins/os_linux/LinuxSockstatExtension.java @@ -17,6 +17,20 @@ public class LinuxSockstatExtension implements MetricExtension { private static final Logger log = LoggerFactory.getLogger(LinuxSockstatExtension.class); + // Extension details + private final String name = "linux_network_sockets"; + private final String provides = "network_sockets"; + private final String description = "Linux Network Socket Metrics"; + + // Configuration / Options + private boolean enabled = true; + + + @Override + public boolean isEnabled() { + return enabled; + } + @Override public boolean isSupported() { @@ -30,17 +44,24 @@ public class LinuxSockstatExtension implements MetricExtension { @Override public String getName() { - return "linux_network_sockets"; + return name; } @Override public String getProvides() { - return "network_sockets"; + return provides; } @Override public String getDescription() { - return "Linux Network Socket Metrics"; + return description; + } + + @Override + public void setConfiguration(Map map) { + if (map.containsKey("enabled")) { + enabled = (boolean) map.get("enabled"); + } } @Override @@ -52,7 +73,7 @@ public class LinuxSockstatExtension implements MetricExtension { HashMap fieldsMap = sockStat.getFields(); log.debug(fieldsMap.toString()); - return new MetricResult(getName(), new Measurement(tagsMap, fieldsMap)); + return new MetricResult(name, new Measurement(tagsMap, fieldsMap)); } diff --git a/server/build.gradle b/server/build.gradle index 032e695..3816f9e 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -28,7 +28,7 @@ def projectName = "sysmon-server" application { // Define the main class for the application. mainClass.set('sysmon.server.Application') - applicationDefaultJvmArgs = [ "-server", "-XX:+UseG1GC", "-Xmx128m" ] + applicationDefaultJvmArgs = [ "-server", "-Xms64m", "-Xmx128m", "-XX:+UseG1GC" ] } tasks.named('test') { diff --git a/server/src/main/java/sysmon/server/ServerRouteBuilder.java b/server/src/main/java/sysmon/server/ServerRouteBuilder.java index a51e849..a14084a 100644 --- a/server/src/main/java/sysmon/server/ServerRouteBuilder.java +++ b/server/src/main/java/sysmon/server/ServerRouteBuilder.java @@ -39,7 +39,7 @@ public class ServerRouteBuilder extends RouteBuilder { .route() .setHeader(Exchange.HTTP_RESPONSE_CODE, constant(202)) .setHeader("Content-Type", constant("application/x-www-form-urlencoded")) - .to("seda:inbound") + .to("seda:inbound?discardWhenFull=true") .endRest(); fromF("seda:inbound?concurrentConsumers=%s", threads) diff --git a/shared/src/main/java/sysmon/shared/MetricExtension.java b/shared/src/main/java/sysmon/shared/MetricExtension.java index 8e7f37f..3249cf1 100644 --- a/shared/src/main/java/sysmon/shared/MetricExtension.java +++ b/shared/src/main/java/sysmon/shared/MetricExtension.java @@ -2,14 +2,18 @@ package sysmon.shared; import org.pf4j.ExtensionPoint; +import java.util.Map; public interface MetricExtension extends ExtensionPoint { + boolean isEnabled(); boolean isSupported(); String getName(); String getProvides(); String getDescription(); + void setConfiguration(Map map); + MetricResult getMetrics() throws Exception; } From 203c5daf3e396b82124d7b68d23569f455e671a7 Mon Sep 17 00:00:00 2001 From: Mark Nellemann Date: Fri, 10 Sep 2021 21:58:14 +0200 Subject: [PATCH 2/2] Some cleanup. --- client/doc/AIX.md | 2 +- client/doc/sysmon-client.service | 4 ++-- client/src/main/java/sysmon/client/Application.java | 4 ---- gradle.properties | 2 +- .../main/java/sysmon/plugins/os_aix/AixNetstatParser.java | 1 - .../main/java/sysmon/plugins/os_aix/AixProcessorStat.java | 5 ++--- .../main/java/sysmon/plugins/os_base/BaseDiskExtension.java | 2 +- .../java/sysmon/plugins/os_base/BaseNetworkExtension.java | 3 +-- .../java/sysmon/plugins/os_linux/LinuxNetstatParser.java | 1 - .../java/sysmon/plugins/os_linux/LinuxNetworkSockStat.java | 1 - server/doc/sysmon-server.service | 6 ++++-- server/src/main/java/sysmon/server/Application.java | 2 -- server/src/main/java/sysmon/server/ServerRouteBuilder.java | 2 -- shared/src/main/java/sysmon/shared/MetricResult.java | 1 - 14 files changed, 12 insertions(+), 24 deletions(-) diff --git a/client/doc/AIX.md b/client/doc/AIX.md index 2bd3c90..cea5076 100644 --- a/client/doc/AIX.md +++ b/client/doc/AIX.md @@ -4,7 +4,7 @@ Works on IBM Power VIO (Virtual IO) servers, as well as regular IBM Power AIX in ## Installation -We require Java 8, which should already be installed. +We require Java 8, which should already be installed on AIX, or is available to install. The RPM packages are *"noarch"* Java bytecode, so we can use the **--ignoreos** option to install: ```shell diff --git a/client/doc/sysmon-client.service b/client/doc/sysmon-client.service index 33d62ce..cee5c52 100644 --- a/client/doc/sysmon-client.service +++ b/client/doc/sysmon-client.service @@ -2,8 +2,8 @@ Description=Sysmon Client Service [Service] -TimeoutStartSec=0 -Restart=always +TimeoutSec=20 +Restart=on-failure ExecStart=/opt/sysmon/client/bin/client -s http://10.20.30.40:9925/metrics [Install] diff --git a/client/src/main/java/sysmon/client/Application.java b/client/src/main/java/sysmon/client/Application.java index 07b90fd..4fcad29 100644 --- a/client/src/main/java/sysmon/client/Application.java +++ b/client/src/main/java/sysmon/client/Application.java @@ -4,8 +4,6 @@ package sysmon.client; import org.apache.camel.main.Main; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import picocli.CommandLine; import java.io.File; @@ -18,8 +16,6 @@ import java.util.concurrent.Callable; @CommandLine.Command(name = "sysmon-client", mixinStandardHelpOptions = true) public class Application implements Callable { - private static final Logger log = LoggerFactory.getLogger(Application.class); - @CommandLine.Option(names = { "-s", "--server-url" }, description = "Server URL (default: ${DEFAULT-VALUE}).", defaultValue = "http://127.0.0.1:9925/metrics", paramLabel = "") private URL serverUrl; diff --git a/gradle.properties b/gradle.properties index 084a2b6..91b92b6 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,4 +1,4 @@ -version=0.0.9 +version=0.0.10 pf4jVersion=3.6.0 slf4jVersion=1.7.32 camelVersion=3.11.1 diff --git a/plugins/os-aix/src/main/java/sysmon/plugins/os_aix/AixNetstatParser.java b/plugins/os-aix/src/main/java/sysmon/plugins/os_aix/AixNetstatParser.java index 21c5ef3..06d5dfc 100644 --- a/plugins/os-aix/src/main/java/sysmon/plugins/os_aix/AixNetstatParser.java +++ b/plugins/os-aix/src/main/java/sysmon/plugins/os_aix/AixNetstatParser.java @@ -8,7 +8,6 @@ import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.util.HashMap; -import java.util.Map; public class AixNetstatParser { diff --git a/plugins/os-aix/src/main/java/sysmon/plugins/os_aix/AixProcessorStat.java b/plugins/os-aix/src/main/java/sysmon/plugins/os_aix/AixProcessorStat.java index c42d801..085bb9f 100644 --- a/plugins/os-aix/src/main/java/sysmon/plugins/os_aix/AixProcessorStat.java +++ b/plugins/os-aix/src/main/java/sysmon/plugins/os_aix/AixProcessorStat.java @@ -8,8 +8,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.Objects; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -86,7 +85,7 @@ public class AixProcessorStat { } //String lparstat = lines.get(lines.size() -1); - String[] splitStr = lastLine.trim().split("\\s+"); + String[] splitStr = Objects.requireNonNull(lastLine).trim().split("\\s+"); if(type.equalsIgnoreCase("shared") && splitStr.length < 9 || type.equalsIgnoreCase("dedicated") && splitStr.length < 8) { throw new UnsupportedOperationException("lparstat string error: " + lastLine); diff --git a/plugins/os-base/src/main/java/sysmon/plugins/os_base/BaseDiskExtension.java b/plugins/os-base/src/main/java/sysmon/plugins/os_base/BaseDiskExtension.java index 3e78281..1fde4b9 100644 --- a/plugins/os-base/src/main/java/sysmon/plugins/os_base/BaseDiskExtension.java +++ b/plugins/os-base/src/main/java/sysmon/plugins/os_base/BaseDiskExtension.java @@ -76,7 +76,7 @@ public class BaseDiskExtension implements MetricExtension { List diskStores = hardwareAbstractionLayer.getDiskStores(); for(HWDiskStore store : diskStores) { String name = store.getName(); - if (name.matches("hdisk[0-9]+") || name.matches("/dev/x?[sv]d[a-z]{1}") || name.matches("/dev/nvme[0-9]n[0-9]")) { + if (name.matches("hdisk[0-9]+") || name.matches("/dev/x?[sv]d[a-z]") || name.matches("/dev/nvme[0-9]n[0-9]")) { log.debug("Using device: " + name); writeBytes += store.getWriteBytes(); readBytes += store.getReadBytes(); diff --git a/plugins/os-base/src/main/java/sysmon/plugins/os_base/BaseNetworkExtension.java b/plugins/os-base/src/main/java/sysmon/plugins/os_base/BaseNetworkExtension.java index 744f459..a8780b6 100644 --- a/plugins/os-base/src/main/java/sysmon/plugins/os_base/BaseNetworkExtension.java +++ b/plugins/os-base/src/main/java/sysmon/plugins/os_base/BaseNetworkExtension.java @@ -3,7 +3,6 @@ package sysmon.plugins.os_base; import org.pf4j.Extension; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import oshi.SystemInfo; import oshi.hardware.HardwareAbstractionLayer; import oshi.hardware.NetworkIF; import sysmon.shared.Measurement; @@ -27,9 +26,9 @@ public class BaseNetworkExtension implements MetricExtension { // Configuration / Options private boolean enabled = true; - private HardwareAbstractionLayer hardwareAbstractionLayer; + @Override public boolean isEnabled() { return enabled; diff --git a/plugins/os-linux/src/main/java/sysmon/plugins/os_linux/LinuxNetstatParser.java b/plugins/os-linux/src/main/java/sysmon/plugins/os_linux/LinuxNetstatParser.java index 876c105..35bf550 100644 --- a/plugins/os-linux/src/main/java/sysmon/plugins/os_linux/LinuxNetstatParser.java +++ b/plugins/os-linux/src/main/java/sysmon/plugins/os_linux/LinuxNetstatParser.java @@ -8,7 +8,6 @@ import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.util.HashMap; -import java.util.Map; public class LinuxNetstatParser { diff --git a/plugins/os-linux/src/main/java/sysmon/plugins/os_linux/LinuxNetworkSockStat.java b/plugins/os-linux/src/main/java/sysmon/plugins/os_linux/LinuxNetworkSockStat.java index ae1f77e..8f4c595 100644 --- a/plugins/os-linux/src/main/java/sysmon/plugins/os_linux/LinuxNetworkSockStat.java +++ b/plugins/os-linux/src/main/java/sysmon/plugins/os_linux/LinuxNetworkSockStat.java @@ -5,7 +5,6 @@ import org.slf4j.LoggerFactory; import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; diff --git a/server/doc/sysmon-server.service b/server/doc/sysmon-server.service index 54e3c47..805a0a6 100644 --- a/server/doc/sysmon-server.service +++ b/server/doc/sysmon-server.service @@ -2,8 +2,10 @@ Description=Sysmon Server Service [Service] -TimeoutStartSec=0 -Restart=always +#User=nobody +#Group=nobody +TimeoutSec=20 +Restart=on-failure ExecStart=/opt/sysmon/server/bin/server [Install] diff --git a/server/src/main/java/sysmon/server/Application.java b/server/src/main/java/sysmon/server/Application.java index 2328896..450dd76 100644 --- a/server/src/main/java/sysmon/server/Application.java +++ b/server/src/main/java/sysmon/server/Application.java @@ -2,9 +2,7 @@ package sysmon.server; import org.apache.camel.main.Main; import org.influxdb.InfluxDB; -import org.influxdb.InfluxDBException; import org.influxdb.InfluxDBFactory; -import org.influxdb.dto.Query; import picocli.CommandLine; import java.io.IOException; diff --git a/server/src/main/java/sysmon/server/ServerRouteBuilder.java b/server/src/main/java/sysmon/server/ServerRouteBuilder.java index a14084a..985f33d 100644 --- a/server/src/main/java/sysmon/server/ServerRouteBuilder.java +++ b/server/src/main/java/sysmon/server/ServerRouteBuilder.java @@ -2,8 +2,6 @@ package sysmon.server; import org.apache.camel.Exchange; import org.apache.camel.builder.RouteBuilder; -import org.apache.camel.component.influxdb.InfluxDbConstants; -import org.apache.camel.component.jackson.JacksonDataFormat; import org.apache.camel.model.rest.RestBindingMode; import org.apache.camel.spi.Registry; import sysmon.shared.MetricResult; diff --git a/shared/src/main/java/sysmon/shared/MetricResult.java b/shared/src/main/java/sysmon/shared/MetricResult.java index a4c3b35..eb94c1a 100644 --- a/shared/src/main/java/sysmon/shared/MetricResult.java +++ b/shared/src/main/java/sysmon/shared/MetricResult.java @@ -3,7 +3,6 @@ package sysmon.shared; import java.io.Serializable; import java.time.Instant; import java.util.ArrayList; -import java.util.List; import java.util.Map; public class MetricResult implements Serializable {