More work on multiple samples.

This commit is contained in:
Mark Nellemann 2022-12-07 16:12:29 +01:00
parent a0cfff18ef
commit fb5bfd532b
9 changed files with 452 additions and 541 deletions

View File

@ -3,7 +3,8 @@
All notable changes to this project will be documented in this file.
## [1.4.1] - 2011-12-xx
- Retrieve multiple PCM samples and keep track of processed samples.
- Retrieve multiple PCM samples and keep track of processing.
- Rename VIOS metric vFC to Virtual.
## [1.4.0] - 2011-12-01
- Rewrite of toml+xml+json de-serialization code (uses jackson now).

View File

@ -39,7 +39,7 @@ dependencies {
application {
mainClass.set('biz.nellemann.hmci.Application')
applicationDefaultJvmArgs = [ "-server", "-Xms64m", "-Xmx64m", "-XX:+UseG1GC", "-XX:+ExitOnOutOfMemoryError", "-XX:+AlwaysPreTouch" ]
applicationDefaultJvmArgs = [ "-server", "-Xms64m", "-Xmx256m", "-XX:+UseG1GC", "-XX:+ExitOnOutOfMemoryError", "-XX:+AlwaysPreTouch" ]
}
java {

View File

@ -1650,7 +1650,7 @@
}
],
"hide": false,
"measurement": "vios_storage_vFC",
"measurement": "vios_storage_virtual",
"orderByTime": "ASC",
"policy": "default",
"refId": "B",
@ -1690,7 +1690,7 @@
]
}
],
"title": "Virtual Fiber Channel Adapters - $ServerName - $ViosName",
"title": "Virtual Adapters - $ServerName - $ViosName",
"type": "timeseries"
}
],

View File

@ -66,6 +66,7 @@ public final class InfluxClient {
influxDB.enableBatch(
BatchOptions.DEFAULTS
.flushDuration(5000)
.threadFactory(runnable -> {
Thread thread = new Thread(runnable);
thread.setDaemon(true);
@ -97,10 +98,11 @@ public final class InfluxClient {
}
/*
public void write(List<Measurement> measurements, Instant timestamp, String name) {
log.debug("write() - measurement: {} {}", name, measurements.size());
processMeasurementMap(measurements, timestamp, name).forEach( (point) -> { influxDB.write(point); });
}
}*/
public void write(List<Measurement> measurements, String name) {
@ -109,6 +111,7 @@ public final class InfluxClient {
}
/*
private List<Point> processMeasurementMap(List<Measurement> measurements, Instant timestamp, String name) {
List<Point> listOfPoints = new ArrayList<>();
measurements.forEach( (m) -> {
@ -122,7 +125,7 @@ public final class InfluxClient {
});
return listOfPoints;
}
}*/
private List<Point> processMeasurementMap(List<Measurement> measurements, String name) {

View File

@ -19,7 +19,6 @@ import biz.nellemann.hmci.dto.xml.Link;
import biz.nellemann.hmci.dto.xml.LogicalPartitionEntry;
import biz.nellemann.hmci.dto.xml.XmlEntry;
import biz.nellemann.hmci.dto.xml.XmlFeed;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.dataformat.xml.XmlMapper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -28,8 +27,6 @@ import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.*;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeoutException;
class LogicalPartition extends Resource {
@ -104,7 +101,7 @@ class LogicalPartition extends Resource {
log.debug("refresh() - {}", name);
try {
String xml = restClient.getRequest(String.format("/rest/api/pcm/ManagedSystem/%s/LogicalPartition/%s/ProcessedMetrics?NoOfSamples=%d", managedSystem.id, id, currentNumberOfSamples));
String xml = restClient.getRequest(String.format("/rest/api/pcm/ManagedSystem/%s/LogicalPartition/%s/ProcessedMetrics?NoOfSamples=%d", managedSystem.id, id, noOfSamples));
// Do not try to parse empty response
if(xml == null || xml.length() <= 1) {
@ -138,7 +135,7 @@ class LogicalPartition extends Resource {
@Override
public void process(int sample) {
public void process(int sample) throws NullPointerException {
log.debug("process() - {} - sample: {}", name, sample);
influxClient.write(getDetails(sample),"lpar_details");
@ -152,11 +149,10 @@ class LogicalPartition extends Resource {
// LPAR Details
List<Measurement> getDetails(int sample) {
List<Measurement> getDetails(int sample) throws NullPointerException {
List<Measurement> list = new ArrayList<>();
try {
Map<String, String> tagsMap = new HashMap<>();
TreeMap<String, Object> fieldsMap = new TreeMap<>();
@ -172,20 +168,16 @@ class LogicalPartition extends Resource {
log.trace("getDetails() - fields: " + fieldsMap);
list.add(new Measurement(getTimestamp(sample), tagsMap, fieldsMap));
} catch (Exception e) {
log.warn("getDetails() - error: {}", e.getMessage());
}
return list;
}
// LPAR Memory
List<Measurement> getMemoryMetrics(int sample) {
List<Measurement> getMemoryMetrics(int sample) throws NullPointerException {
List<Measurement> list = new ArrayList<>();
try {
Map<String, String> tagsMap = new HashMap<>();
TreeMap<String, Object> fieldsMap = new TreeMap<>();
@ -198,19 +190,16 @@ class LogicalPartition extends Resource {
log.trace("getMemoryMetrics() - fields: " + fieldsMap);
list.add(new Measurement(getTimestamp(sample), tagsMap, fieldsMap));
} catch (Exception e) {
log.warn("getMemoryMetrics() - error: {}", e.getMessage());
}
return list;
}
// LPAR Processor
List<Measurement> getProcessorMetrics(int sample) {
List<Measurement> getProcessorMetrics(int sample) throws NullPointerException {
List<Measurement> list = new ArrayList<>();
try {
HashMap<String, String> tagsMap = new HashMap<>();
HashMap<String, Object> fieldsMap = new HashMap<>();
@ -235,20 +224,16 @@ class LogicalPartition extends Resource {
log.trace("getProcessorMetrics() - fields: " + fieldsMap);
list.add(new Measurement(getTimestamp(sample), tagsMap, fieldsMap));
} catch (Exception e) {
log.warn("getProcessorMetrics() - error: {}", e.getMessage());
}
return list;
}
// LPAR Network - Virtual
List<Measurement> getVirtualEthernetAdapterMetrics(int sample) {
List<Measurement> getVirtualEthernetAdapterMetrics(int sample) throws NullPointerException {
List<Measurement> list = new ArrayList<>();
try {
metric.getSample(sample).lparsUtil.network.virtualEthernetAdapters.forEach(adapter -> {
HashMap<String, String> tagsMap = new HashMap<>();
@ -280,20 +265,16 @@ class LogicalPartition extends Resource {
list.add(new Measurement(getTimestamp(sample), tagsMap, fieldsMap));
});
} catch (Exception e) {
log.warn("getVirtualEthernetAdapterMetrics() - error: {}", e.getMessage());
}
return list;
}
// LPAR Storage - Virtual Generic
List<Measurement> getVirtualGenericAdapterMetrics(int sample) {
List<Measurement> getVirtualGenericAdapterMetrics(int sample) throws NullPointerException {
List<Measurement> list = new ArrayList<>();
try {
metric.getSample(sample).lparsUtil.storage.genericVirtualAdapters.forEach(adapter -> {
HashMap<String, String> tagsMap = new HashMap<>();
@ -315,20 +296,16 @@ class LogicalPartition extends Resource {
list.add(new Measurement(getTimestamp(sample), tagsMap, fieldsMap));
});
} catch (Exception e) {
log.warn("getVirtualGenericAdapterMetrics() - error: {}", e.getMessage());
}
return list;
}
// LPAR Storage - Virtual FC
List<Measurement> getVirtualFibreChannelAdapterMetrics(int sample) {
List<Measurement> getVirtualFibreChannelAdapterMetrics(int sample) throws NullPointerException {
List<Measurement> list = new ArrayList<>();
try {
metric.getSample(sample).lparsUtil.storage.virtualFiberChannelAdapters.forEach(adapter -> {
HashMap<String, String> tagsMap = new HashMap<>();
@ -350,20 +327,16 @@ class LogicalPartition extends Resource {
list.add(new Measurement(getTimestamp(sample), tagsMap, fieldsMap));
});
} catch (Exception e) {
log.warn("getVirtualFibreChannelAdapterMetrics() - error: {}", e.getMessage());
}
return list;
}
// LPAR Network - SR-IOV Logical Ports
List<Measurement> getSriovLogicalPorts(int sample) {
List<Measurement> getSriovLogicalPorts(int sample) throws NullPointerException {
List<Measurement> list = new ArrayList<>();
try {
metric.getSample(sample).lparsUtil.network.sriovLogicalPorts.forEach(port -> {
HashMap<String, String> tagsMap = new HashMap<>();
@ -387,9 +360,6 @@ class LogicalPartition extends Resource {
list.add(new Measurement(getTimestamp(sample), tagsMap, fieldsMap));
});
} catch (Exception e) {
log.warn("getSriovLogicalPorts() - error: {}", e.getMessage());
}
return list;
}

View File

@ -156,7 +156,7 @@ class ManagedSystem extends Resource {
log.debug("refresh() - {}", name);
try {
String xml = restClient.getRequest(String.format("/rest/api/pcm/ManagedSystem/%s/ProcessedMetrics?NoOfSamples=%d", id, currentNumberOfSamples));
String xml = restClient.getRequest(String.format("/rest/api/pcm/ManagedSystem/%s/ProcessedMetrics?NoOfSamples=%d", id, noOfSamples));
// Do not try to parse empty response
if(xml == null || xml.length() <= 1) {
@ -200,7 +200,7 @@ class ManagedSystem extends Resource {
@Override
public void process(int sample) {
public void process(int sample) throws NullPointerException {
log.debug("process() - {} - sample: {}", name, sample);
@ -222,7 +222,7 @@ class ManagedSystem extends Resource {
influxClient.write(getVioNetworkGenericAdapters(sample),"vios_network_generic");
influxClient.write(getVioStorageLpars(sample),"vios_storage_lpars");
influxClient.write(getVioStorageFiberChannelAdapters(sample),"vios_storage_FC");
influxClient.write(getVioStorageVirtualAdapters(sample),"vios_storage_vFC");
influxClient.write(getVioStorageVirtualAdapters(sample),"vios_storage_virtual");
influxClient.write(getVioStoragePhysicalAdapters(sample),"vios_storage_physical");
// Missing: vios_storage_SSP
@ -290,10 +290,9 @@ class ManagedSystem extends Resource {
// System details
List<Measurement> getDetails(int sample) {
List<Measurement> getDetails(int sample) throws NullPointerException {
List<Measurement> list = new ArrayList<>();
try {
Map<String, String> tagsMap = new TreeMap<>();
Map<String, Object> fieldsMap = new TreeMap<>();
@ -314,21 +313,16 @@ class ManagedSystem extends Resource {
fieldsMap.put("assignedMem", metric.getSample(sample).systemFirmwareUtil.assignedMem);
log.trace("getDetails() - fields: " + fieldsMap);
list.add(new Measurement(getTimestamp(sample), tagsMap, fieldsMap));
} catch (Exception e) {
log.warn("getDetails() - error: {}", e.getMessage());
}
return list;
}
// System Memory
List<Measurement> getMemoryMetrics(int sample) {
List<Measurement> getMemoryMetrics(int sample) throws NullPointerException {
List<Measurement> list = new ArrayList<>();
try {
HashMap<String, String> tagsMap = new HashMap<>();
Map<String, Object> fieldsMap = new HashMap<>();
@ -343,19 +337,15 @@ class ManagedSystem extends Resource {
log.trace("getMemoryMetrics() - fields: " + fieldsMap);
list.add(new Measurement(getTimestamp(sample), tagsMap, fieldsMap));
} catch (Exception e) {
log.warn("getMemoryMetrics() - error: {}", e.getMessage());
}
return list;
}
// System Processor
List<Measurement> getProcessorMetrics(int sample) {
List<Measurement> getProcessorMetrics(int sample) throws NullPointerException {
List<Measurement> list = new ArrayList<>();
try {
HashMap<String, String> tagsMap = new HashMap<>();
HashMap<String, Object> fieldsMap = new HashMap<>();
@ -369,18 +359,14 @@ class ManagedSystem extends Resource {
log.trace("getProcessorMetrics() - fields: " + fieldsMap);
list.add(new Measurement(getTimestamp(sample), tagsMap, fieldsMap));
} catch (Exception e) {
log.warn("getProcessorMetrics() - error: {}", e.getMessage());
}
return list;
}
// Sytem Shared ProcessorPools
List<Measurement> getSharedProcessorPools(int sample) {
List<Measurement> getSharedProcessorPools(int sample) throws NullPointerException {
List<Measurement> list = new ArrayList<>();
try {
metric.getSample(sample).serverUtil.sharedProcessorPool.forEach(sharedProcessorPool -> {
HashMap<String, String> tagsMap = new HashMap<>();
HashMap<String, Object> fieldsMap = new HashMap<>();
@ -399,19 +385,15 @@ class ManagedSystem extends Resource {
list.add(new Measurement(getTimestamp(sample), tagsMap, fieldsMap));
});
} catch (Exception e) {
log.warn("getSharedProcessorPools() - error: {}", e.getMessage());
}
return list;
}
// System Physical ProcessorPool
List<Measurement> getPhysicalProcessorPool(int sample) {
List<Measurement> getPhysicalProcessorPool(int sample) throws NullPointerException {
List<Measurement> list = new ArrayList<>();
try {
HashMap<String, String> tagsMap = new HashMap<>();
HashMap<String, Object> fieldsMap = new HashMap<>();
@ -426,9 +408,6 @@ class ManagedSystem extends Resource {
log.trace("getPhysicalProcessorPool() - fields: " + fieldsMap);
list.add(new Measurement(getTimestamp(sample), tagsMap, fieldsMap));
} catch (Exception e) {
log.warn("getPhysicalProcessorPool() - error: {}", e.getMessage());
}
return list;
}
@ -440,10 +419,9 @@ class ManagedSystem extends Resource {
// VIO Details
List<Measurement> getVioDetails(int sample) {
List<Measurement> getVioDetails(int sample) throws NullPointerException {
List<Measurement> list = new ArrayList<>();
try {
metric.getSample(sample).viosUtil.forEach(vio -> {
HashMap<String, String> tagsMap = new HashMap<>();
@ -461,19 +439,15 @@ class ManagedSystem extends Resource {
list.add(new Measurement(getTimestamp(sample), tagsMap, fieldsMap));
});
} catch (Exception e) {
log.warn("getVioDetails() - error: {}", e.getMessage());
}
return list;
}
// VIO Memory
List<Measurement> getVioMemoryMetrics(int sample) {
List<Measurement> getVioMemoryMetrics(int sample) throws NullPointerException {
List<Measurement> list = new ArrayList<>();
try {
metric.getSample(sample).viosUtil.forEach(vio -> {
HashMap<String, String> tagsMap = new HashMap<>();
@ -493,19 +467,15 @@ class ManagedSystem extends Resource {
list.add(new Measurement(getTimestamp(sample), tagsMap, fieldsMap));
});
} catch (Exception e) {
log.warn("getVioMemoryMetrics() - error: {}", e.getMessage());
}
return list;
}
// VIO Processor
List<Measurement> getVioProcessorMetrics(int sample) {
List<Measurement> getVioProcessorMetrics(int sample) throws NullPointerException {
List<Measurement> list = new ArrayList<>();
try {
metric.getSample(sample).viosUtil.forEach(vio -> {
HashMap<String, String> tagsMap = new HashMap<>();
@ -532,19 +502,15 @@ class ManagedSystem extends Resource {
list.add(new Measurement(getTimestamp(sample), tagsMap, fieldsMap));
});
} catch (Exception e) {
log.warn("getVioProcessorMetrics() - error: {}", e.getMessage());
}
return list;
}
// VIOs - Network
List<Measurement> getVioNetworkLpars(int sample) {
List<Measurement> getVioNetworkLpars(int sample) throws NullPointerException {
List<Measurement> list = new ArrayList<>();
try {
metric.getSample(sample).viosUtil.forEach(vio -> {
HashMap<String, String> tagsMap = new HashMap<>();
@ -560,19 +526,14 @@ class ManagedSystem extends Resource {
list.add(new Measurement(getTimestamp(sample), tagsMap, fieldsMap));
});
} catch (Exception e) {
log.warn("getVioNetworkLpars() - error: {}", e.getMessage());
}
return list;
}
// VIO Network - Shared
List<Measurement> getVioNetworkSharedAdapters(int sample) {
List<Measurement> getVioNetworkSharedAdapters(int sample) throws NullPointerException {
List<Measurement> list = new ArrayList<>();
try {
metric.getSample(sample).viosUtil.forEach(vio -> {
vio.network.sharedAdapters.forEach(adapter -> {
HashMap<String, String> tagsMap = new HashMap<>();
@ -597,19 +558,15 @@ class ManagedSystem extends Resource {
list.add(new Measurement(getTimestamp(sample), tagsMap, fieldsMap));
});
});
} catch (Exception e) {
log.warn("getVioNetworkSharedAdapters() - error: {}", e.getMessage());
}
return list;
}
// VIO Network - Virtual
List<Measurement> getVioNetworkVirtualAdapters(int sample) {
List<Measurement> getVioNetworkVirtualAdapters(int sample) throws NullPointerException {
List<Measurement> list = new ArrayList<>();
try {
metric.getSample(sample).viosUtil.forEach( vio -> {
vio.network.virtualEthernetAdapters.forEach( adapter -> {
@ -641,19 +598,15 @@ class ManagedSystem extends Resource {
list.add(new Measurement(getTimestamp(sample), tagsMap, fieldsMap));
});
});
} catch (Exception e) {
log.warn("getVioNetworkVirtualAdapters() - error: {}", e.getMessage());
}
return list;
}
// VIO Network - Generic
List<Measurement> getVioNetworkGenericAdapters(int sample) {
List<Measurement> getVioNetworkGenericAdapters(int sample) throws NullPointerException {
List<Measurement> list = new ArrayList<>();
try {
metric.getSample(sample).viosUtil.forEach( vio -> {
vio.network.genericAdapters.forEach( adapter -> {
@ -677,18 +630,15 @@ class ManagedSystem extends Resource {
list.add(new Measurement(getTimestamp(sample), tagsMap, fieldsMap));
});
});
} catch (Exception e) {
log.warn("getVioNetworkGenericAdapters() - error: {}", e.getMessage());
}
return list;
}
// VIOs - Storage
List<Measurement> getVioStorageLpars(int sample) {
List<Measurement> getVioStorageLpars(int sample) throws NullPointerException {
List<Measurement> list = new ArrayList<>();
try {
metric.getSample(sample).viosUtil.forEach(vio -> {
HashMap<String, String> tagsMap = new HashMap<>();
@ -703,18 +653,15 @@ class ManagedSystem extends Resource {
list.add(new Measurement(getTimestamp(sample), tagsMap, fieldsMap));
});
} catch (Exception e) {
log.warn("getVioStorageLpars() - error: {}", e.getMessage());
}
return list;
}
// VIO Storage FC
List<Measurement> getVioStorageFiberChannelAdapters(int sample) {
List<Measurement> getVioStorageFiberChannelAdapters(int sample) throws NullPointerException {
List<Measurement> list = new ArrayList<>();
try {
metric.getSample(sample).viosUtil.forEach( vio -> {
log.trace("getVioStorageFiberChannelAdapters() - VIO: " + vio.name);
@ -741,19 +688,14 @@ class ManagedSystem extends Resource {
});
} catch (Exception e) {
log.warn("getVioStorageFiberChannelAdapters() - error: {}", e.getMessage());
}
return list;
}
// VIO Storage - Physical
List<Measurement> getVioStoragePhysicalAdapters(int sample) {
List<Measurement> getVioStoragePhysicalAdapters(int sample) throws NullPointerException {
List<Measurement> list = new ArrayList<>();
try {
metric.getSample(sample).viosUtil.forEach( vio -> {
log.trace("getVioStoragePhysicalAdapters() - VIO: " + vio.name);
@ -779,19 +721,15 @@ class ManagedSystem extends Resource {
list.add(new Measurement(getTimestamp(sample), tagsMap, fieldsMap));
});
});
} catch (Exception e) {
log.warn("getVioStoragePhysicalAdapters() - error: {}", e.getMessage());
}
return list;
}
// VIO Storage - Virtual
List<Measurement> getVioStorageVirtualAdapters(int sample) {
List<Measurement> getVioStorageVirtualAdapters(int sample) throws NullPointerException {
List<Measurement> list = new ArrayList<>();
try {
metric.getSample(sample).viosUtil.forEach( (vio) -> {
vio.storage.genericVirtualAdapters.forEach( (adapter) -> {
HashMap<String, String> tagsMap = new HashMap<>();
@ -814,9 +752,6 @@ class ManagedSystem extends Resource {
list.add(new Measurement(getTimestamp(sample), tagsMap, fieldsMap));
});
});
} catch (Exception e) {
log.warn("getVioStorageVirtualAdapters() - error: {}", e.getMessage());
}
return list;
}
@ -824,7 +759,7 @@ class ManagedSystem extends Resource {
/*
// VIO Storage SSP TODO
List<Measurement> getViosStorageSharedStoragePools(int sample) {
List<Measurement> getViosStorageSharedStoragePools(int sample) throws NullPointerException {
List<Measurement> list = new ArrayList<>();
metrics.systemUtil.getSample(sample).viosUtil.forEach( vios -> {

View File

@ -21,9 +21,9 @@ public abstract class Resource {
private final ArrayList<String> sampleHistory = new ArrayList<>();
protected SystemUtil metric;
protected final int maxNumberOfSamples = 120;
protected final int maxNumberOfSamples = 60;
protected final int minNumberOfSamples = 5;
protected int currentNumberOfSamples = 15;
protected int noOfSamples = maxNumberOfSamples;
@ -97,9 +97,10 @@ public abstract class Resource {
return;
}
int samples = metric.samples.size();
//log.info("process() - Samples to process: {}, Samples in History: {}, Current Counter: {}", samples, sampleHistory.size(), currentNumberOfSamples);
for(int i = 0; i<samples; i++) {
int processed = 0;
int sampleSize = metric.samples.size();
log.debug("process() - Samples Returned: {}, Samples in History: {}, Fetch Next Counter: {}", sampleSize, sampleHistory.size(), noOfSamples);
for(int i = 0; i<sampleSize; i++) {
UtilSample sample = metric.getSample(i);
String timestamp = sample.getInfo().timestamp;
@ -108,27 +109,28 @@ public abstract class Resource {
continue; // Already processed
}
// Process
//log.info("process() - Sample: {}", timestamp);
try {
process(i);
// Add to end of history
sampleHistory.add(timestamp);
processed++;
sampleHistory.add(timestamp); // Add to processed history
} catch (NullPointerException e) {
log.warn("process() - error: {}", e.getMessage());
}
}
// Remove old elements from history
for(int n = currentNumberOfSamples; n < sampleHistory.size(); n++) {
for(int n = noOfSamples; n < sampleHistory.size(); n++) {
//log.info("process() - Removing element no. {} from sampleHistory: {}", n, sampleHistory.get(0));
sampleHistory.remove(0);
}
// Slowly decrease until we reach minSamples
if(currentNumberOfSamples > minNumberOfSamples) {
currentNumberOfSamples--;
// Decrease until we reach minSamples
if(noOfSamples > minNumberOfSamples) {
noOfSamples = Math.min( (noOfSamples - 1), Math.max( (noOfSamples - processed), minNumberOfSamples));
}
}
public abstract void process(int sample);
public abstract void process(int sample) throws NullPointerException;
}

View File

@ -34,7 +34,7 @@ class SystemEnergy extends Resource {
log.debug("refresh()");
try {
String xml = restClient.getRequest(String.format("/rest/api/pcm/ManagedSystem/%s/ProcessedMetrics?Type=Energy&NoOfSamples=%d", managedSystem.id, currentNumberOfSamples));
String xml = restClient.getRequest(String.format("/rest/api/pcm/ManagedSystem/%s/ProcessedMetrics?Type=Energy&NoOfSamples=%d", managedSystem.id, noOfSamples));
// Do not try to parse empty response
if(xml == null || xml.length() <= 1) {

View File

@ -13,7 +13,7 @@ public class HmcConfiguration {
public String username;
public String password;
public Integer refresh = 60;
public Integer refresh = 30;
public Integer discover = 120;
public String trace;