Initial work in Linux network metrics.

This commit is contained in:
Mark Nellemann 2021-05-24 22:47:40 +02:00
parent 0124b21692
commit 9f8191d0e5
18 changed files with 431 additions and 29 deletions

View file

@ -44,6 +44,26 @@ tasks.named('test') {
useJUnitPlatform()
}
jar {
manifest {
attributes(
'Created-By' : "Gradle ${gradle.gradleVersion}",
'Build-OS' : "${System.properties['os.name']} ${System.properties['os.arch']} ${System.properties['os.version']}",
'Build-Jdk' : "${System.properties['java.version']} (${System.properties['java.vendor']} ${System.properties['java.vm.version']})",
'Build-User' : System.properties['user.name'],
'Build-Version' : versioning.info.tag ?: (versioning.info.branch + "-" + versioning.info.build),
'Build-Revision' : versioning.info.commit,
'Build-Timestamp': new Date().format("yyyy-MM-dd'T'HH:mm:ss.SSSZ").toString(),
)
}
}
shadowJar {
archiveBaseName.set(projectName)
archiveClassifier.set('')
archiveVersion.set('')
mergeServiceFiles() // Tell plugin to merge duplicate service files
}
apply plugin: 'nebula.ospackage'
ospackage {
@ -86,24 +106,3 @@ task buildRpmAix(type: Rpm) {
packageName = "${projectName}-AIX"
os = Os.AIX
}
jar {
manifest {
attributes(
'Created-By' : "Gradle ${gradle.gradleVersion}",
'Build-OS' : "${System.properties['os.name']} ${System.properties['os.arch']} ${System.properties['os.version']}",
'Build-Jdk' : "${System.properties['java.version']} (${System.properties['java.vendor']} ${System.properties['java.vm.version']})",
'Build-User' : System.properties['user.name'],
'Build-Version' : versioning.info.tag ?: (versioning.info.branch + "-" + versioning.info.build),
'Build-Revision' : versioning.info.commit,
'Build-Timestamp': new Date().format("yyyy-MM-dd'T'HH:mm:ss.SSSZ").toString(),
)
}
}
shadowJar {
archiveBaseName.set(projectName)
archiveClassifier.set('')
archiveVersion.set('')
mergeServiceFiles() // Tell plugin to merge duplicate service files
}

View file

@ -70,7 +70,7 @@ public class LinuxDiskProcLine {
//private Long timeSpentFlushing = 0L; // ms
LinuxDiskProcLine(List<String> procLines) {
public LinuxDiskProcLine(List<String> procLines) {
for(String procLine : procLines) {

View file

@ -13,7 +13,7 @@ public class LinuxMemoryStat {
Mem: 16069172 5896832 4597860 639780 5574480 9192992
Swap: 3985404 0 3985404
*/
private final Pattern pattern = Pattern.compile("^Mem:\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)");
private static final Pattern pattern = Pattern.compile("^Mem:\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)");
private long total;
private long used;

View file

@ -0,0 +1,90 @@
package sysmon.plugins.os_linux;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class LinuxNetworkDevProcLine {
private static final Logger log = LoggerFactory.getLogger(LinuxNetworkDevProcLine.class);
private static final Pattern pattern1 = Pattern.compile("^\\s+([a-z]{2,}[0-9]+):.*");
private static final Pattern pattern2 = Pattern.compile("^\\s+([a-z]{2,}[0-9]+):\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)");
private long rxBytes;
private long rxPackets;
private long rxErrs;
private long txBytes;
private long txPackets;
private long txErrs;
/*
Inter-| Receive | Transmit
face |bytes packets errs drop fifo frame compressed multicast|bytes packets errs drop fifo colls carrier compressed
env2: 657010764 483686 0 0 0 0 0 0 55416850 431020 0 0 0 0 0 0
env3: 6900272 41836 0 0 0 0 0 0 7667444 41849 0 0 0 0 0 0
lo: 3098805 14393 0 0 0 0 0 0 3098805 14393 0 0 0 0 0 0
*/
public LinuxNetworkDevProcLine(List<String> procLines) {
Matcher matcher1;
Matcher matcher2;
for(String procLine : procLines) {
matcher1 = pattern1.matcher(procLine);
if(matcher1.matches()) {
if(matcher1.group(1).equals("lo")) {
continue;
}
matcher2 = pattern2.matcher(procLine);
if(matcher2.matches() && matcher2.groupCount() == 17) {
rxBytes += Long.parseLong(matcher2.group(2));
rxPackets += Long.parseLong(matcher2.group(3));
rxErrs += Long.parseLong(matcher2.group(4));
txBytes += Long.parseLong(matcher2.group(10));
txPackets += Long.parseLong(matcher2.group(11));
txErrs += Long.parseLong(matcher2.group(12));
}
}
}
}
public long getRxBytes() {
return rxBytes;
}
public long getRxPackets() {
return rxPackets;
}
public long getRxErrs() {
return rxErrs;
}
public long getTxBytes() {
return txBytes;
}
public long getTxPackets() {
return txPackets;
}
public long getTxErrs() {
return txErrs;
}
}

View file

@ -0,0 +1,36 @@
package sysmon.plugins.os_linux;
import java.util.HashMap;
import java.util.Map;
public class LinuxNetworkDevStat {
private final long rxBytes;
private final long rxPackets;
private final long txBytes;
private final long txPackets;
public LinuxNetworkDevStat(LinuxNetworkDevProcLine previous, LinuxNetworkDevProcLine current) {
rxBytes = current.getRxBytes() - previous.getRxBytes();
rxPackets = current.getRxPackets() - previous.getRxPackets();
txBytes = current.getTxBytes() - previous.getTxBytes();
txPackets = current.getTxPackets() - previous.getTxPackets();
}
public Map<String, String> getTags() {
return new HashMap<>();
}
public Map<String, Object> getFields() {
Map<String, Object> fields = new HashMap<>();
fields.put("rxBytes", rxBytes);
fields.put("rxPackets", rxPackets);
fields.put("txBytes", txBytes);
fields.put("txPackets", txPackets);
return fields;
}
}

View file

@ -0,0 +1,78 @@
package sysmon.plugins.os_linux;
import org.pf4j.Extension;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import sysmon.shared.Measurement;
import sysmon.shared.MetricExtension;
import sysmon.shared.MetricResult;
import sysmon.shared.PluginHelper;
import java.util.List;
import java.util.Map;
@Extension
public class LinuxNetworkExtension implements MetricExtension {
private static final Logger log = LoggerFactory.getLogger(LinuxNetworkExtension.class);
@Override
public boolean isSupported() {
if(!System.getProperty("os.name").toLowerCase().contains("linux")) {
log.warn("Requires Linux.");
return false;
}
return true;
}
@Override
public String getName() {
return "linux-network";
}
@Override
public String getProvides() {
return "network";
}
@Override
public String getDescription() {
return "Linux Network Metrics";
}
@Override
public MetricResult getMetrics() {
// LinuxNetworkDevStat = 2 x reading from /proc/net/dev ?
LinuxNetworkDevProcLine proc1 = processDevOutput(PluginHelper.readFile("/proc/net/dev"));
try {
Thread.sleep(1 * 1000); // TODO: Configure sample collect time
} catch (InterruptedException e) {
log.warn("getMetrics() - sleep interrupted");
return null;
}
LinuxNetworkDevProcLine proc2 = processDevOutput(PluginHelper.readFile("/proc/net/dev"));
// LinuxNetworkSockStat = 1 x reading from /proc/net/sockstats
LinuxNetworkSockStat stat = processSockOutput(PluginHelper.readFile("/proc/net/sockstat"));
Map<String, String> tagsMap = stat.getTags();
Map<String, Object> fieldsMap = stat.getFields();
return new MetricResult("network", new Measurement(tagsMap, fieldsMap));
}
protected LinuxNetworkSockStat processSockOutput(List<String> inputLines) {
return new LinuxNetworkSockStat(inputLines);
}
protected LinuxNetworkDevProcLine processDevOutput(List<String> inputLines) {
return new LinuxNetworkDevProcLine(inputLines);
}
}

View file

@ -0,0 +1,97 @@
package sysmon.plugins.os_linux;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class LinuxNetworkSockStat {
private static final Logger log = LoggerFactory.getLogger(LinuxNetworkSockStat.class);
private static final Pattern pattern1 = Pattern.compile("^sockets: used (\\d+)");
private static final Pattern pattern2 = Pattern.compile("^TCP: inuse (\\d+) orphan (\\d+) tw (\\d+) alloc (\\d+) mem (\\d+)");
private static final Pattern pattern3 = Pattern.compile("^UDP: inuse (\\d+) mem (\\d+)");
private long sockets;
private long tcp_inuse;
private long tcp_orphan;
private long tcp_tw;
private long tcp_alloc;
private long tcp_mem;
private long udp_inuse;
private long udp_mem;
/*
sockets: used 1238
TCP: inuse 52 orphan 0 tw 18 alloc 55 mem 7
UDP: inuse 11 mem 10
UDPLITE: inuse 0
RAW: inuse 0
FRAG: inuse 0 memory 0
*/
LinuxNetworkSockStat(List<String> lines) {
Matcher matcher;
for(String line : lines) {
String proto = line.substring(0, line.indexOf(':'));
switch (proto) {
case "sockets":
matcher = pattern1.matcher(line);
if (matcher.matches() && matcher.groupCount() == 1) {
sockets = Long.parseLong(matcher.group(1));
}
break;
case "TCP":
matcher = pattern2.matcher(line);
if (matcher.matches() && matcher.groupCount() == 5) {
tcp_inuse = Long.parseLong(matcher.group(1));
tcp_orphan = Long.parseLong(matcher.group(2));
tcp_tw = Long.parseLong(matcher.group(3));
tcp_alloc = Long.parseLong(matcher.group(4));
tcp_mem = Long.parseLong(matcher.group(5));
}
break;
case "UDP":
matcher = pattern3.matcher(line);
if (matcher.matches() && matcher.groupCount() == 2) {
udp_inuse = Long.parseLong(matcher.group(1));
udp_mem = Long.parseLong(matcher.group(2));
}
break;
}
}
}
public Map<String, String> getTags() {
return new HashMap<>();
}
public Map<String, Object> getFields() {
Map<String, Object> fields = new HashMap<>();
fields.put("sockets", sockets);
fields.put("tcp_inuse", tcp_inuse);
fields.put("tcp_alloc", tcp_alloc);
fields.put("tcp_orphan", tcp_orphan);
fields.put("tcp_mem", tcp_mem);
fields.put("tcp_tw", tcp_tw);
fields.put("udp_inuse", udp_inuse);
fields.put("udp_mem", udp_mem);
return fields;
}
}

View file

@ -8,7 +8,7 @@ class LinuxDiskTest extends Specification {
void "test proc file processing"() {
setup:
def testFile = new File(getClass().getResource('/diskstats1.txt').toURI())
def testFile = new File(getClass().getResource('/proc_diskstats1.txt').toURI())
List<String> lines = testFile.readLines("UTF-8")
when:
@ -23,8 +23,8 @@ class LinuxDiskTest extends Specification {
void "test disk utilization"() {
setup:
def testFile1 = new File(getClass().getResource('/diskstats1.txt').toURI())
def testFile2 = new File(getClass().getResource('/diskstats2.txt').toURI())
def testFile1 = new File(getClass().getResource('/proc_diskstats1.txt').toURI())
def testFile2 = new File(getClass().getResource('/proc_diskstats2.txt').toURI())
LinuxDiskExtension extension = new LinuxDiskExtension()
LinuxDiskProcLine procLine1 = extension.processFileOutput(testFile1.readLines())
LinuxDiskProcLine procLine2 = extension.processFileOutput(testFile2.readLines())

View file

@ -0,0 +1,74 @@
import spock.lang.Specification
import sysmon.plugins.os_linux.LinuxDiskExtension
import sysmon.plugins.os_linux.LinuxDiskProcLine
import sysmon.plugins.os_linux.LinuxDiskStat
import sysmon.plugins.os_linux.LinuxNetworkDevProcLine
import sysmon.plugins.os_linux.LinuxNetworkDevStat
import sysmon.plugins.os_linux.LinuxNetworkExtension
import sysmon.plugins.os_linux.LinuxNetworkSockStat
class LinuxNetworkTest extends Specification {
void "test /proc/net/sockstat parsing"() {
setup:
def testFile = new File(getClass().getResource('/proc_net_sockstat.txt').toURI())
List<String> lines = testFile.readLines("UTF-8")
when:
LinuxNetworkExtension extension = new LinuxNetworkExtension()
LinuxNetworkSockStat stats = extension.processSockOutput(lines)
then:
stats.getFields().get("sockets") == 1238L
stats.getFields().get("tcp_inuse") == 52L
stats.getFields().get("tcp_orphan") == 0L
stats.getFields().get("tcp_alloc") == 55L
stats.getFields().get("tcp_mem") == 7l
stats.getFields().get("tcp_tw") == 18L
stats.getFields().get("udp_inuse") == 11L
stats.getFields().get("udp_mem") == 10L
}
void "test /proc/net/dev parsing"() {
setup:
def testFile = new File(getClass().getResource('/proc_net_dev1.txt').toURI())
List<String> lines = testFile.readLines("UTF-8")
when:
LinuxNetworkExtension extension = new LinuxNetworkExtension()
LinuxNetworkDevProcLine procLine = extension.processDevOutput(lines)
then:
procLine.getRxBytes() == 663911036L
procLine.getRxPackets() == 525522L
procLine.getRxErrs() == 0L
procLine.getTxBytes() == 63084294L
procLine.getTxPackets() == 472869L
procLine.getTxErrs() == 0L
}
void "test dev utilization"() {
setup:
def testFile1 = new File(getClass().getResource('/proc_net_dev1.txt').toURI())
def testFile2 = new File(getClass().getResource('/proc_net_dev2.txt').toURI())
LinuxNetworkExtension extension = new LinuxNetworkExtension()
LinuxNetworkDevProcLine procLine1 = extension.processDevOutput(testFile1.readLines())
LinuxNetworkDevProcLine procLine2 = extension.processDevOutput(testFile2.readLines())
when:
LinuxNetworkDevStat networkDevStat = new LinuxNetworkDevStat(procLine1, procLine2)
then:
networkDevStat.getFields().get("rxPackets") == 223L
networkDevStat.getFields().get("rxBytes") == 31501L
networkDevStat.getFields().get("txBytes") == 46460L
networkDevStat.getFields().get("txPackets") == 341L
}
}

View file

@ -8,7 +8,7 @@ class LinuxProcessorTest extends Specification {
void "test proc file processing"() {
setup:
def testFile = new File(getClass().getResource('/proc1.txt').toURI())
def testFile = new File(getClass().getResource('/proc_stats1.txt').toURI())
List<String> lines = testFile.readLines("UTF-8")
when:
@ -27,8 +27,8 @@ class LinuxProcessorTest extends Specification {
void "test processor utilization"() {
setup:
def testFile1 = new File(getClass().getResource('/proc1.txt').toURI())
def testFile2 = new File(getClass().getResource('/proc2.txt').toURI())
def testFile1 = new File(getClass().getResource('/proc_stats1.txt').toURI())
def testFile2 = new File(getClass().getResource('/proc_stats2.txt').toURI())
LinuxProcessorProcLine processorProcLine1 = new LinuxProcessorProcLine(testFile1.readLines().get(0))
LinuxProcessorProcLine processorProcLine2 = new LinuxProcessorProcLine(testFile2.readLines().get(0))

View file

@ -0,0 +1,5 @@
Inter-| Receive | Transmit
face |bytes packets errs drop fifo frame compressed multicast|bytes packets errs drop fifo colls carrier compressed
env2: 657010764 483686 0 0 0 0 0 0 55416850 431020 0 0 0 0 0 0
env3: 6900272 41836 0 0 0 0 0 0 7667444 41849 0 0 0 0 0 0
lo: 3098805 14393 0 0 0 0 0 0 3098805 14393 0 0 0 0 0 0

View file

@ -0,0 +1,5 @@
Inter-| Receive | Transmit
face |bytes packets errs drop fifo frame compressed multicast|bytes packets errs drop fifo colls carrier compressed
env2: 657034787 483864 0 0 0 0 0 0 55454936 431316 0 0 0 0 0 0
env3: 6907750 41881 0 0 0 0 0 0 7675818 41894 0 0 0 0 0 0
lo: 3098805 14393 0 0 0 0 0 0 3098805 14393 0 0 0 0 0 0

View file

@ -0,0 +1,6 @@
sockets: used 1238
TCP: inuse 52 orphan 0 tw 18 alloc 55 mem 7
UDP: inuse 11 mem 10
UDPLITE: inuse 0
RAW: inuse 0
FRAG: inuse 0 memory 0

View file

@ -7,6 +7,7 @@ import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.ArrayList;
@ -68,4 +69,15 @@ public class PluginHelper {
.anyMatch(path -> Files.exists(path.resolve(cmd)));
}
public static List<String> readFile(String filename) {
List<String> allLines = new ArrayList<>();
try {
allLines = Files.readAllLines(Paths.get(filename), StandardCharsets.UTF_8);
} catch (IOException e) {
log.error(e.getMessage());
}
return allLines;
}
}