Update dashboard and add more metrics.

This commit is contained in:
Mark Nellemann 2022-11-29 17:20:47 +01:00
parent 73478f41e1
commit 7ff360944d
22 changed files with 4348 additions and 458 deletions

View file

@ -33,6 +33,7 @@ dependencies {
implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-toml:2.14.1'
testImplementation 'junit:junit:4.13.2'
testImplementation 'org.slf4j:slf4j-simple:2.0.4'
testImplementation 'org.spockframework:spock-core:2.3-groovy-3.0'
testImplementation "org.mock-server:mockserver-netty-no-dependencies:5.14.0"
}

File diff suppressed because it is too large Load diff

View file

@ -12,7 +12,6 @@ database = "svci"
url = "https://10.10.10.12:7443"
username = "superuser"
password = "password"
refresh = 10
discover = 120
refresh = 30
trust = true # Ignore SSL cert. errors

View file

@ -1,3 +1,3 @@
projectId = svci
projectGroup = biz.nellemann.svci
projectVersion = 0.0.1
projectVersion = 0.0.2

View file

@ -0,0 +1,49 @@
package biz.nellemann.svci;
import com.fasterxml.jackson.databind.util.StdConverter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* Jackson Converter
* Converts a "storage capacity" presented as a String with a unit (eg. MB) to a Double value in TB.
*/
public class CapacityToDoubleConverter extends StdConverter<String, Double> {
private final static Logger log = LoggerFactory.getLogger(CapacityToDoubleConverter.class);
final private Pattern p = Pattern.compile("(^\\d*\\.?\\d*)\\s?(\\D{2})$");
@Override
public Double convert(String value) {
Matcher m = p.matcher(value);
if(!m.matches()) {
return null;
}
double input = Double.parseDouble(m.group(1));
String unit = m.group(2);
log.debug("Input: {} {}", input, unit);
double output = input;
if(unit.equals("PB")) {
output = input * 1000;
} else if(unit.equals("TB")) {
output = input;
} else if(unit.equals("GB")) {
output = input / 1000;
} else if(unit.equals("MB")) {
output = input / 1_000_000;
} else {
log.warn("convert() - Unit {} not supported.", unit);
}
log.debug("Output: {} TB", output);
return output;
}
}

View file

@ -15,8 +15,7 @@
*/
package biz.nellemann.svci;
import biz.nellemann.svci.dto.json.EnclosureStat;
import biz.nellemann.svci.dto.json.NodeStat;
import biz.nellemann.svci.dto.json.*;
import biz.nellemann.svci.dto.json.System;
import biz.nellemann.svci.dto.toml.SvcConfiguration;
import com.fasterxml.jackson.databind.ObjectMapper;
@ -38,25 +37,17 @@ class VolumeController implements Runnable {
private final static Logger log = LoggerFactory.getLogger(VolumeController.class);
private final ObjectMapper objectMapper = new ObjectMapper();
private final Integer refreshValue;
private final Integer discoverValue;
//private final List<ManagedSystem> managedSystems = new ArrayList<>();
private final RestClient restClient;
private final InfluxClient influxClient;
private final ObjectMapper objectMapper = new ObjectMapper();
private final AtomicBoolean keepRunning = new AtomicBoolean(true);
protected Integer responseErrors = 0;
protected System system;
VolumeController(SvcConfiguration configuration, InfluxClient influxClient) {
this.refreshValue = configuration.refresh;
this.discoverValue = configuration.discover;
this.influxClient = influxClient;
restClient = new RestClient(configuration.url, configuration.username, configuration.password, configuration.trust);
@ -67,9 +58,7 @@ class VolumeController implements Runnable {
public void run() {
log.trace("run()");
restClient.login();
discover();
do {
Instant instantStart = Instant.now();
@ -96,7 +85,7 @@ class VolumeController implements Runnable {
log.error("run() - sleep interrupted", e);
}
} else {
log.warn("run() - possible slow response from this HMC");
log.warn("run() - possible slow response from this SVC");
}
} while (keepRunning.get());
@ -104,16 +93,12 @@ class VolumeController implements Runnable {
}
void discover() {
log.debug("discover()");
influxClient.write(getSystem(), Instant.now(),"system");
}
void refresh() {
log.debug("refresh()");
influxClient.write(getSystem(), Instant.now(),"system");
influxClient.write(getNodeStats(), Instant.now(),"node_stats");
influxClient.write(getEnclosureStats(), Instant.now(),"enclosure_stats");
influxClient.write(getMDiskGroups(), Instant.now(),"m_disk_groups");
}
@ -139,8 +124,13 @@ class VolumeController implements Runnable {
fieldsMap.put("location", system.location);
fieldsMap.put("code_level", system.codeLevel);
fieldsMap.put("product_name", system.productName);
fieldsMap.put("total_free_tb", system.totalFreeTB);
fieldsMap.put("total_used_tb", system.totalUsedTB);
fieldsMap.put("mdisk_total_tb", system.mDiskTotalTB);
fieldsMap.put("vdisk_total_tb", system.vDiskTotalTB);
fieldsMap.put("vdisk_allocated_tb", system.vDiskAllocatedTB);
log.trace("getNodeStats() - fields: " + fieldsMap);
log.trace("getSystem() - fields: " + fieldsMap);
measurementList.add(new Measurement(tagsMap, fieldsMap));
} catch (IOException e) {
@ -158,13 +148,13 @@ class VolumeController implements Runnable {
String response = restClient.postRequest("/rest/v1/lsnodestats");
// Do not try to parse empty response
if(response == null || response.length() <= 1) {
if(system == null || response == null || response.length() <= 1) {
log.warn("getNodeStats() - no data.");
return measurementList;
}
List<NodeStat> pojo = Arrays.asList(objectMapper.readValue(response, NodeStat[].class));
pojo.forEach((stat) -> {
List<NodeStat> list = Arrays.asList(objectMapper.readValue(response, NodeStat[].class));
list.forEach( (stat) -> {
HashMap<String, String> tagsMap = new HashMap<>();
HashMap<String, Object> fieldsMap = new HashMap<>();
@ -188,6 +178,7 @@ class VolumeController implements Runnable {
return measurementList;
}
List<Measurement> getEnclosureStats() {
List<Measurement> measurementList = new ArrayList<>();
@ -195,13 +186,13 @@ class VolumeController implements Runnable {
String response = restClient.postRequest("/rest/v1/lsenclosurestats");
// Do not try to parse empty response
if(response == null || response.length() <= 1) {
if(system == null || response == null || response.length() <= 1) {
log.warn("getEnclosureStats() - no data.");
return measurementList;
}
List<EnclosureStat> pojo = Arrays.asList(objectMapper.readValue(response, EnclosureStat[].class));
pojo.forEach((stat) -> {
List<EnclosureStat> list = Arrays.asList(objectMapper.readValue(response, EnclosureStat[].class));
list.forEach( (stat) -> {
HashMap<String, String> tagsMap = new HashMap<>();
HashMap<String, Object> fieldsMap = new HashMap<>();
@ -223,4 +214,86 @@ class VolumeController implements Runnable {
return measurementList;
}
List<Measurement> getVDisk() {
List<Measurement> measurementList = new ArrayList<>();
try {
String response = restClient.postRequest("/rest/v1/lsvdisk");
// Do not try to parse empty response
if(system == null || response == null || response.length() <= 1) {
log.warn("getVDisk() - no data.");
return measurementList;
}
List<VDisk> list = Arrays.asList(objectMapper.readValue(response, VDisk[].class));
list.forEach( (stat) -> {
HashMap<String, String> tagsMap = new HashMap<>();
HashMap<String, Object> fieldsMap = new HashMap<>();
tagsMap.put("id", stat.id);
tagsMap.put("name", stat.name);
tagsMap.put("type", stat.type);
tagsMap.put("system", system.name);
fieldsMap.put("capacity_tb", stat.capacity);
log.trace("getVDisk() - fields: " + fieldsMap);
measurementList.add(new Measurement(tagsMap, fieldsMap));
//log.info("{}: {} -> {}", stat.nodeName, stat.statName, stat.statCurrent);
});
} catch (IOException e) {
log.error("getVDisk() - error 2: {}", e.getMessage());
}
return measurementList;
}
List<Measurement> getMDiskGroups() {
List<Measurement> measurementList = new ArrayList<>();
try {
String response = restClient.postRequest("/rest/v1/lsmdiskgrp");
// Do not try to parse empty response
if(system == null || response == null || response.length() <= 1) {
log.warn("getMDiskGroups() - no data.");
return measurementList;
}
List<MDiskGroup> list = Arrays.asList(objectMapper.readValue(response, MDiskGroup[].class));
list.forEach( (stat) -> {
HashMap<String, String> tagsMap = new HashMap<>();
HashMap<String, Object> fieldsMap = new HashMap<>();
tagsMap.put("id", stat.id);
tagsMap.put("name", stat.name);
tagsMap.put("system", system.name);
fieldsMap.put("mdisk_count", stat.mDiskCount);
fieldsMap.put("vdisk_count", stat.vDiskCount);
fieldsMap.put("capacity_free_tb", stat.capacityFree);
fieldsMap.put("capacity_real_tb", stat.capacityReal);
fieldsMap.put("capacity_used_tb", stat.capacityUsed);
fieldsMap.put("capacity_total_tb", stat.capacityTotal);
fieldsMap.put("capacity_virtual_tb", stat.capacityVirtual);
log.trace("getMDiskGroups() - fields: " + fieldsMap);
measurementList.add(new Measurement(tagsMap, fieldsMap));
//log.info("{}: {} -> {}", stat.nodeName, stat.statName, stat.statCurrent);
});
} catch (IOException e) {
log.error("getMDiskGroups() - error 2: {}", e.getMessage());
}
return measurementList;
}
}

View file

@ -0,0 +1,80 @@
package biz.nellemann.svci.dto.json;
import biz.nellemann.svci.CapacityToDoubleConverter;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
@JsonIgnoreProperties(ignoreUnknown = true)
public class MDiskGroup {
public String id;
public String name;
public String status;
@JsonProperty("mdisk_count")
public Number mDiskCount;
@JsonProperty("vdisk_count")
public Number vDiskCount;
@JsonProperty("capacity")
@JsonDeserialize(converter = CapacityToDoubleConverter.class)
public Number capacityTotal;
@JsonProperty("free_capacity")
@JsonDeserialize(converter = CapacityToDoubleConverter.class)
public Number capacityFree;
@JsonProperty("virtual_capacity")
@JsonDeserialize(converter = CapacityToDoubleConverter.class)
public Number capacityVirtual;
@JsonProperty("used_capacity")
@JsonDeserialize(converter = CapacityToDoubleConverter.class)
public Number capacityUsed;
@JsonProperty("real_capacity")
@JsonDeserialize(converter = CapacityToDoubleConverter.class)
public Number capacityReal;
@JsonProperty("parent_mdisk_grp_id")
public Number parentMDiskGroupId;
@JsonProperty("parent_mdisk_grp_name")
public String parentMDiskGroupName;
/*
{
"extent_size": "1024",
"overallocation": "41",
"warning": "80",
"easy_tier": "auto",
"easy_tier_status": "balanced",
"compression_active": "no",
"compression_virtual_capacity": "0.00MB",
"compression_compressed_capacity": "0.00MB",
"compression_uncompressed_capacity": "0.00MB",
"child_mdisk_grp_count": "0",
"child_mdisk_grp_capacity": "0.00MB",
"type": "parent",
"encrypt": "no",
"owner_type": "none",
"owner_id": "",
"owner_name": "",
"site_id": "",
"site_name": "",
"data_reduction": "no",
"used_capacity_before_reduction": "0.00MB",
"used_capacity_after_reduction": "0.00MB",
"overhead_capacity": "0.00MB",
"deduplication_capacity_saving": "0.00MB",
"reclaimable_capacity": "0.00MB",
"easy_tier_fcm_over_allocation_max": "",
"provisioning_policy_id": "",
"provisioning_policy_name": ""
},
*/
}

View file

@ -1,7 +1,9 @@
package biz.nellemann.svci.dto.json;
import biz.nellemann.svci.CapacityToDoubleConverter;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
@JsonIgnoreProperties(ignoreUnknown = true)
public class System {
@ -10,6 +12,9 @@ public class System {
public String location;
@JsonProperty("time_zone")
public String timeZone;
@JsonProperty("statistics_status")
public String statisticsStatus;
@ -22,21 +27,32 @@ public class System {
@JsonProperty("product_name")
public String productName;
@JsonProperty("total_mdisk_capacity")
@JsonDeserialize(converter = CapacityToDoubleConverter.class)
public Double mDiskTotalTB;
@JsonProperty("space_allocated_to_vdisks")
@JsonDeserialize(converter = CapacityToDoubleConverter.class)
public Double vDiskAllocatedTB;
@JsonProperty("total_vdisk_capacity")
@JsonDeserialize(converter = CapacityToDoubleConverter.class)
public Double vDiskTotalTB;
@JsonProperty("total_free_space")
@JsonDeserialize(converter = CapacityToDoubleConverter.class)
public Double totalFreeTB;
@JsonProperty("total_used_capacity")
@JsonDeserialize(converter = CapacityToDoubleConverter.class)
public Double totalUsedTB;
/**
"id": "000001002100613E",
"name": "V7000_A2U12",
"location": "local",
"partnership": "",
"total_mdisk_capacity": "60.9TB",
"space_in_mdisk_grps": "60.9TB",
"space_allocated_to_vdisks": "2.87TB",
"total_free_space": "58.0TB",
"total_vdiskcopy_capacity": "20.42TB",
"total_used_capacity": "2.60TB",
"total_overallocation": "33",
"total_vdisk_capacity": "20.42TB",
"total_allocated_extent_capacity": "2.92TB",
"statistics_status": "on",
"statistics_frequency": "5",
@ -154,6 +170,6 @@ public class System {
"automatic_vdisk_analysis_enabled": "on",
"callhome_accepted_usage": "no",
"safeguarded_copy_suspended": "no"
*/
}

View file

@ -0,0 +1,64 @@
package biz.nellemann.svci.dto.json;
import biz.nellemann.svci.CapacityToDoubleConverter;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
@JsonIgnoreProperties(ignoreUnknown = true)
public class VDisk {
public String id;
public String name;
public String type;
public String status;
@JsonDeserialize(converter = CapacityToDoubleConverter.class)
public Number capacity;
@JsonProperty("IO_group_id")
public Number ioGroupId;
@JsonProperty("IO_group_name")
public String ioGroupName;
@JsonProperty("mdisk_grp_id")
public Number mDiskGroupId;
@JsonProperty("mdisk_grp_name")
public String mDiskGroupName;
@JsonProperty("parent_mdisk_grp_id")
public Number parentMDiskGroupId;
@JsonProperty("parent_mdisk_grp_name")
public String parentMDiskGroupName;
/*
{
"FC_id": "",
"FC_name": "",
"RC_id": "",
"RC_name": "",
"vdisk_UID": "6005076400840184F80000000000005F",
"fc_map_count": "0",
"copy_count": "1",
"fast_write_state": "empty",
"se_copy_count": "1",
"RC_change": "no",
"compressed_copy_count": "0",
"owner_id": "",
"owner_name": "",
"formatting": "no",
"encrypt": "no",
"volume_id": "72",
"volume_name": "volume-Image_rhcos_4-11_volume_1-e4c39c5a-c8bf",
"function": "",
"protocol": ""
},
*/
}

View file

@ -8,10 +8,7 @@ public class SvcConfiguration {
public String url;
public String username;
public String password;
public Integer refresh = 30;
public Integer discover = 120;
public Boolean trust = true;
}

View file

@ -0,0 +1,56 @@
package biz.nellemann.svci
import spock.lang.Specification
class CapacityToDoubleConverterTest extends Specification {
CapacityToDoubleConverter converter = new CapacityToDoubleConverter()
def "convert from TB String to TB Double"() {
when:
def result = converter.convert("123.45TB")
then:
result == 123.45
}
def "convert from PB String to TB Double"() {
when:
def result = converter.convert("1024.0PB")
then:
result == 1024000.0
}
def "convert from GB String to TB Double"() {
when:
def result = converter.convert("8192.0GB")
then:
result == 8.192
}
def "convert from GB String (with a space) to TB Double"() {
when:
def result = converter.convert("8192.0 GB")
then:
result == 8.192
}
def "convert from MB String to TB Double"() {
when:
def result = converter.convert("4096.0MB")
then:
result == 0.004096
}
}

View file

@ -25,7 +25,7 @@ class DeserializationTest extends Specification {
void "lssystem"() {
when:
Path testConfigurationFile = Paths.get(getClass().getResource('/lssystem.json').toURI())
Path testConfigurationFile = Paths.get(getClass().getResource('/json/lssystem.json').toURI())
System system = mapper.readerFor(System.class).readValue(testConfigurationFile.toFile())
then:
@ -33,13 +33,15 @@ class DeserializationTest extends Specification {
system.location == "local"
system.codeLevel == "8.4.2.0 (build 154.20.2109031944000)"
system.productName == "IBM Storwize V7000"
system.totalUsedTB == 2.6
system.totalFreeTB == 58.0
}
void "lsnodestat"() {
when:
Path testConfigurationFile = Paths.get(getClass().getResource('/lsnodestats.json').toURI())
Path testConfigurationFile = Paths.get(getClass().getResource('/json/lsnodestats.json').toURI())
List<NodeStat> nodeStats = Arrays.asList(mapper.readerFor(NodeStat[].class).readValue(testConfigurationFile.toFile()))
then:
@ -53,7 +55,7 @@ class DeserializationTest extends Specification {
void "lsenclosurestats"() {
when:
Path testConfigurationFile = Paths.get(getClass().getResource('/lsenclosurestats.json').toURI())
Path testConfigurationFile = Paths.get(getClass().getResource('/json/lsenclosurestats.json').toURI())
List<EnclosureStat> enclosureStats = Arrays.asList(mapper.readerFor(EnclosureStat[].class).readValue(testConfigurationFile.toFile()))
then:

View file

@ -64,7 +64,7 @@ class RestClientTest extends Specification {
def "Test SVC Login"() {
setup:
def responseFile = new File(getClass().getResource('/svc-auth-response.json').toURI())
def responseFile = new File(getClass().getResource('/json/svc-auth-response.json').toURI())
def req = HttpRequest.request()
.withHeader("X-Auth-Username", "superuser")
.withHeader("X-Auth-Password", "password")

View file

@ -0,0 +1,42 @@
[
{
"id": "0",
"name": "MDisk1",
"status": "online",
"mode": "array",
"mdisk_grp_id": "0",
"mdisk_grp_name": "Pool0",
"capacity": "49.2TB",
"ctrl_LUN_#": "",
"controller_name": "",
"UID": "",
"tier": "tier1_flash",
"encrypt": "no",
"site_id": "",
"site_name": "",
"distributed": "yes",
"dedupe": "no",
"over_provisioned": "no",
"supports_unmap": "yes"
},
{
"id": "16",
"name": "MDisk2",
"status": "online",
"mode": "array",
"mdisk_grp_id": "1",
"mdisk_grp_name": "Pool1",
"capacity": "11.8TB",
"ctrl_LUN_#": "",
"controller_name": "",
"UID": "",
"tier": "tier_enterprise",
"encrypt": "no",
"site_id": "",
"site_name": "",
"distributed": "yes",
"dedupe": "no",
"over_provisioned": "no",
"supports_unmap": "no"
}
]

View file

@ -0,0 +1,84 @@
[
{
"id": "0",
"name": "Pool0",
"status": "online",
"mdisk_count": "1",
"vdisk_count": "95",
"capacity": "49.17TB",
"extent_size": "1024",
"free_capacity": "46.23TB",
"virtual_capacity": "20.42TB",
"used_capacity": "2.62TB",
"real_capacity": "2.89TB",
"overallocation": "41",
"warning": "80",
"easy_tier": "auto",
"easy_tier_status": "balanced",
"compression_active": "no",
"compression_virtual_capacity": "0.00MB",
"compression_compressed_capacity": "0.00MB",
"compression_uncompressed_capacity": "0.00MB",
"parent_mdisk_grp_id": "0",
"parent_mdisk_grp_name": "Pool0",
"child_mdisk_grp_count": "0",
"child_mdisk_grp_capacity": "0.00MB",
"type": "parent",
"encrypt": "no",
"owner_type": "none",
"owner_id": "",
"owner_name": "",
"site_id": "",
"site_name": "",
"data_reduction": "no",
"used_capacity_before_reduction": "0.00MB",
"used_capacity_after_reduction": "0.00MB",
"overhead_capacity": "0.00MB",
"deduplication_capacity_saving": "0.00MB",
"reclaimable_capacity": "0.00MB",
"easy_tier_fcm_over_allocation_max": "",
"provisioning_policy_id": "",
"provisioning_policy_name": ""
},
{
"id": "1",
"name": "Pool1",
"status": "online",
"mdisk_count": "1",
"vdisk_count": "0",
"capacity": "11.74TB",
"extent_size": "1024",
"free_capacity": "11.74TB",
"virtual_capacity": "0.00MB",
"used_capacity": "0.00MB",
"real_capacity": "0.00MB",
"overallocation": "0",
"warning": "80",
"easy_tier": "auto",
"easy_tier_status": "balanced",
"compression_active": "no",
"compression_virtual_capacity": "0.00MB",
"compression_compressed_capacity": "0.00MB",
"compression_uncompressed_capacity": "0.00MB",
"parent_mdisk_grp_id": "1",
"parent_mdisk_grp_name": "Pool1",
"child_mdisk_grp_count": "0",
"child_mdisk_grp_capacity": "0.00MB",
"type": "parent",
"encrypt": "no",
"owner_type": "none",
"owner_id": "",
"owner_name": "",
"site_id": "",
"site_name": "",
"data_reduction": "no",
"used_capacity_before_reduction": "0.00MB",
"used_capacity_after_reduction": "0.00MB",
"overhead_capacity": "0.00MB",
"deduplication_capacity_saving": "0.00MB",
"reclaimable_capacity": "0.00MB",
"easy_tier_fcm_over_allocation_max": "",
"provisioning_policy_id": "",
"provisioning_policy_name": ""
}
]

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,6 @@
org.slf4j.simpleLogger.logFile=System.err
org.slf4j.simpleLogger.showDateTime=false
org.slf4j.simpleLogger.showShortLogName=true
org.slf4j.simpleLogger.dateTimeFormat=yyyy-MM-dd HH:mm:ss.SSS
org.slf4j.simpleLogger.levelInBrackets=true
org.slf4j.simpleLogger.defaultLogLevel=warn

View file

@ -13,6 +13,5 @@ url = "https://10.10.10.18:7443"
username = "superuser"
password = "password"
refresh = 29
discover = 59
trust = true # Ignore SSL cert. errors