More work on metrics for InfluxDB.

This commit is contained in:
Mark Nellemann 2020-08-13 11:48:00 +02:00
parent decac80ac6
commit 810cfee22b
11 changed files with 3053 additions and 172 deletions

View File

@ -24,7 +24,7 @@ Use the gradle build tool
Start the InfluxDB container
docker run --name=influxdb -d -p 8086:8086 influxdb
docker run --name=influxdb --rm -d -p 8086:8086 influxdb
To use the Influx client from the same container
@ -35,6 +35,6 @@ To use the Influx client from the same container
Start the Grafana container, linking it to the InfluxDB container
docker run --name grafana --link influxdb:influxdb -d -p 3000:3000 grafana/grafana:7.1.3
docker run --name grafana --link influxdb:influxdb --rm -d -p 3000:3000 grafana/grafana:7.1.3
Configure a new InfluxDB datasource on **http://influxdb:8086** to talk to the InfluxDB container. The database must be created beforehand, this can be done by running the hmci tool.

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

34
doc/grafana-metrics.txt Normal file
View File

@ -0,0 +1,34 @@
PartitionProcessor
- UtilizedProcUnits => utilizedProcUnits
- /apped/
- /Time/ => /time/
PartitionMemory
-
PartitionVirtualEthernetAdapters
- /Bytes/
- /PhysicalBytes/
PartitionVirtualFiberChannelAdapters
- Bytes
- /trans/
SystemSharedProcessorPool
- !availableProcUnits
SystemProcessor
- /Units/
SystemMemory
-
SystemFiberChannelAdapters
- /writeByt/
SystemSharedAdapters
- transferredBytes

View File

@ -92,6 +92,15 @@ class InfluxClient {
}
/*
Managed System
*/
void writeManagedSystem(ManagedSystem system) {
if(system.metrics == null) {
@ -107,18 +116,9 @@ class InfluxClient {
BatchPoints batchPoints = BatchPoints
.database(database)
//.retentionPolicy("defaultPolicy")
//.retentionPolicy("defaultPolicy")
.build();
/*
ServerProcessor processor
ServerMemory memory
PhysicalProcessorPool physicalProcessorPool
SharedProcessorPool sharedProcessorPool
+ VIOS
*/
getSystemMemory(system, timestamp).each {
batchPoints.point(it)
}
@ -131,9 +131,46 @@ class InfluxClient {
batchPoints.point(it)
}
getSystemSharedAdapters(system, timestamp).each {
batchPoints.point(it)
}
getSystemFiberChannelAdapters(system, timestamp).each {
batchPoints.point(it)
}
influxDB.write(batchPoints);
}
private static List<Point> getSystemMemory(ManagedSystem system, Instant timestamp) {
List<Map> metrics = system.getMemoryMetrics()
return processMeasurementMap(metrics, timestamp, "SystemMemory")
}
private static List<Point> getSystemProcessor(ManagedSystem system, Instant timestamp) {
List<Map> metrics = system.getProcessorMetrics()
return processMeasurementMap(metrics, timestamp, "SystemProcessor")
}
private static List<Point> getSystemSharedProcessorPools(ManagedSystem system, Instant timestamp) {
List<Map> metrics = system.getSharedProcessorPools()
return processMeasurementMap(metrics, timestamp, "SystemSharedProcessorPool")
}
private static List<Point> getSystemSharedAdapters(ManagedSystem system, Instant timestamp) {
List<Map> metrics = system.getSystemSharedAdapters()
return processMeasurementMap(metrics, timestamp, "SystemSharedAdapters")
}
private static List<Point> getSystemFiberChannelAdapters(ManagedSystem system, Instant timestamp) {
List<Map> metrics = system.getSystemFiberChannelAdapters()
return processMeasurementMap(metrics, timestamp, "SystemFiberChannelAdapters")
}
/*
Logical Partitions
*/
void writeLogicalPartition(LogicalPartition partition) {
@ -164,113 +201,39 @@ class InfluxClient {
batchPoints.point(it)
}
getPartitionVirtualFiberChannelAdapter(partition, timestamp).each {
batchPoints.point(it)
}
influxDB.write(batchPoints);
}
private static List<Point> getSystemMemory(ManagedSystem system, Instant timestamp) {
Map map = system.getMemoryMetrics()
List<Point> pointList = map.collect {fieldName, fieldValue ->
return Point.measurement("SystemMemory")
.time(timestamp.toEpochMilli(), TimeUnit.MILLISECONDS)
.tag("system", system.name)
.tag("name", fieldName.capitalize()) // The dashboard expects it
.addField("value", fieldValue)
.build()
}
return pointList;
}
private static List<Point> getSystemProcessor(ManagedSystem system, Instant timestamp) {
Map map = system.getProcessorMetrics()
List<Point> pointList = map.collect {fieldName, fieldValue ->
return Point.measurement("SystemProcessor")
.time(timestamp.toEpochMilli(), TimeUnit.MILLISECONDS)
.tag("system", system.name)
.tag("name", fieldName.capitalize()) // The dashboard expects it
.addField("value", fieldValue)
.build()
}
return pointList;
}
private static List<Point> getSystemSharedProcessorPools(ManagedSystem system, Instant timestamp) {
List<Point> pointList
system.getSharedProcessorPools().each {name, map ->
//log.debug(name) // Pool name
pointList = map.collect { fieldName, fieldValue ->
return Point.measurement("SystemSharedProcessorPool")
.time(timestamp.toEpochMilli(), TimeUnit.MILLISECONDS)
.tag("system", system.name)
.tag("pool", name)
.tag("name", fieldName)
.addField("value", fieldValue)
.build()
}
}
return pointList;
}
private static List<Point> getPartitionMemory(LogicalPartition partition, Instant timestamp) {
Map map = partition.getMemoryMetrics()
List<Point> pointList = map.collect {fieldName, fieldValue ->
return Point.measurement("PartitionMemory")
.time(timestamp.toEpochMilli(), TimeUnit.MILLISECONDS)
.tag("partition", partition.name)
.tag("system", partition.system.name)
.tag("name", fieldName.capitalize()) // The dashboard expects it
.addField("value", fieldValue)
.build()
}
return pointList;
List<Map> metrics = partition.getMemoryMetrics()
return processMeasurementMap(metrics, timestamp, "PartitionMemory")
}
private static List<Point> getPartitionProcessor(LogicalPartition partition, Instant timestamp) {
Map map = partition.getProcessorMetrics()
List<Point> pointList = map.collect {fieldName, fieldValue ->
return Point.measurement("PartitionProcessor")
.time(timestamp.toEpochMilli(), TimeUnit.MILLISECONDS)
.tag("partition", partition.name)
.tag("system", partition.system.name)
.tag("name", fieldName.capitalize()) // The dashboard expects it
.addField("value", fieldValue)
.build()
}
return pointList;
List<Map> metrics = partition.getProcessorMetrics()
return processMeasurementMap(metrics, timestamp, "PartitionProcessor")
}
private static List<Point> getPartitionVirtualEthernetAdapter(LogicalPartition partition, Instant timestamp) {
List<Map> metrics = partition.getVirtualEthernetAdapterMetrics()
return processMeasurementMap(metrics, timestamp, "PartitionVirtualEthernetAdapters")
}
private static List<Point> getPartitionVirtualFiberChannelAdapter(LogicalPartition partition, Instant timestamp) {
List<Map> metrics = partition.getVirtualFiberChannelAdaptersMetrics()
return processMeasurementMap(metrics, timestamp, "PartitionVirtualFiberChannelAdapters")
}
/*
Shared
*/
private static List<Point> processMeasurementMap(List<Map> listOfMaps, Instant timestamp, String measurement) {
List<Point> list = new ArrayList<>()

View File

@ -22,43 +22,63 @@ class LogicalPartition extends MetaSystem {
}
Map<String,BigDecimal> getMemoryMetrics() {
List<Map> getMemoryMetrics() {
HashMap<String, BigDecimal> map = [
List<Map> list = new ArrayList<>()
Map<String, Map> map = new HashMap<String, Map>()
HashMap<String, String> tagsMap = [
system: system.name,
partition: name,
]
map.put("tags", tagsMap)
log.debug(tagsMap.toString())
HashMap<String, BigDecimal> fieldsMap = [
logicalMem: metrics.systemUtil.utilSamples.first().lparsUtil.first().memory.logicalMem.first(),
backedPhysicalMem: metrics.systemUtil.utilSamples.first().lparsUtil.first().memory.backedPhysicalMem.first(),
]
map.put("fields", fieldsMap)
log.debug(fieldsMap.toString())
return map
list.add(map)
return list
}
Map<String,BigDecimal> getProcessorMetrics() {
List<Map> getProcessorMetrics() {
HashMap<String, BigDecimal> map = [
utilizedProcUnits: metrics.systemUtil.utilSamples.first().lparsUtil.first().processor.utilizedProcUnits.first(),
maxVirtualProcessors: metrics.systemUtil.utilSamples.first().lparsUtil.first().processor.maxVirtualProcessors.first(),
currentVirtualProcessors: metrics.systemUtil.utilSamples.first().lparsUtil.first().processor.currentVirtualProcessors.first(),
donatedProcUnits: metrics.systemUtil.utilSamples.first().lparsUtil.first().processor.donatedProcUnits.first(),
entitledProcUnits: metrics.systemUtil.utilSamples.first().lparsUtil.first().processor.entitledProcUnits.first(),
idleProcUnits: metrics.systemUtil.utilSamples.first().lparsUtil.first().processor.idleProcUnits.first(),
maxProcUnits: metrics.systemUtil.utilSamples.first().lparsUtil.first().processor.maxProcUnits.first(),
utilizedCappedProcUnits: metrics.systemUtil.utilSamples.first().lparsUtil.first().processor.utilizedCappedProcUnits.first(),
utilizedUncappedProcUnits: metrics.systemUtil.utilSamples.first().lparsUtil.first().processor.utilizedUncappedProcUnits.first(),
timePerInstructionExecution: metrics.systemUtil.utilSamples.first().lparsUtil.first().processor.timeSpentWaitingForDispatch.first(),
timeSpentWaitingForDispatch: metrics.systemUtil.utilSamples.first().lparsUtil.first().processor.timePerInstructionExecution.first(),
List<Map> list = new ArrayList<>()
Map<String, Map> map = new HashMap<String, Map>()
HashMap<String, String> tagsMap = [
system: system.name,
partition: name,
]
map.put("tags", tagsMap)
log.debug(tagsMap.toString())
return map
HashMap<String, BigDecimal> fieldsMap = [
utilizedProcUnits: metrics.systemUtil.utilSamples.first().lparsUtil.first().processor.utilizedProcUnits.first(),
//maxVirtualProcessors: metrics.systemUtil.utilSamples.first().lparsUtil.first().processor.maxVirtualProcessors.first(),
//currentVirtualProcessors: metrics.systemUtil.utilSamples.first().lparsUtil.first().processor.currentVirtualProcessors.first(),
//donatedProcUnits: metrics.systemUtil.utilSamples.first().lparsUtil.first().processor.donatedProcUnits.first(),
//entitledProcUnits: metrics.systemUtil.utilSamples.first().lparsUtil.first().processor.entitledProcUnits.first(),
//idleProcUnits: metrics.systemUtil.utilSamples.first().lparsUtil.first().processor.idleProcUnits.first(),
//maxProcUnits: metrics.systemUtil.utilSamples.first().lparsUtil.first().processor.maxProcUnits.first(),
utilizedCappedProcUnits: metrics.systemUtil.utilSamples.first().lparsUtil.first().processor.utilizedCappedProcUnits.first(),
utilizedUncappedProcUnits: metrics.systemUtil.utilSamples.first().lparsUtil.first().processor.utilizedUncappedProcUnits.first(),
timePerInstructionExecution: metrics.systemUtil.utilSamples.first().lparsUtil.first().processor.timeSpentWaitingForDispatch.first(),
timeSpentWaitingForDispatch: metrics.systemUtil.utilSamples.first().lparsUtil.first().processor.timePerInstructionExecution.first(),
]
map.put("fields", fieldsMap)
log.debug(fieldsMap.toString())
list.add(map)
return list
}
// PartitionVSCSIAdapters - VIOS?
// PartitionVirtualEthernetAdapters
// PartitionVirtualFiberChannelAdapters
List<Map> getVirtualEthernetAdapterMetrics() {
List<Map> list = new ArrayList<>()
@ -68,12 +88,13 @@ class LogicalPartition extends MetaSystem {
HashMap<String, String> tagsMap = [
system: system.name,
partition: name,
sea: it.sharedEthernetAdapterId,
viosId: it.viosId,
vlanId: it.vlanId,
vswitchId: it.vswitchId,
sea: it.sharedEthernetAdapterId as String,
viosId: it.viosId as String,
vlanId: it.vlanId as String,
vswitchId: it.vswitchId as String,
]
map.put("tags", tagsMap)
log.debug(tagsMap.toString())
HashMap<String, BigDecimal> fieldsMap = [
receivedPhysicalBytes: it.receivedPhysicalBytes.first(),
@ -81,12 +102,42 @@ class LogicalPartition extends MetaSystem {
receivedBytes: it.receivedBytes.first(),
sentBytes: it.sentBytes.first(),
]
map.put(it.physicalLocation, fieldsMap)
map.put("fields", fieldsMap)
log.debug(fieldsMap.toString())
list.add(map)
}
return list
}
//PartitionVirtualFiberChannelAdapters
List<Map> getVirtualFiberChannelAdaptersMetrics() {
List<Map> list = new ArrayList<>()
Map<String, Map> map = new HashMap<String, Map>()
metrics.systemUtil.utilSamples.first().lparsUtil.first().storage?.virtualFiberChannelAdapters?.each {
HashMap<String, String> tagsMap = [
system: system.name,
partition: name,
viosId: it.viosId as String,
wwpn: it.wwpn,
]
map.put("tags", tagsMap)
log.debug(tagsMap.toString())
HashMap<String, BigDecimal> fieldsMap = [
transmittedBytes: it.transmittedBytes.first(),
writeBytes: it.writeBytes.first(),
]
map.put("fields", fieldsMap)
log.debug(fieldsMap.toString())
list.add(map)
}
return list
}
}

View File

@ -27,73 +27,147 @@ class ManagedSystem extends MetaSystem {
return "[${id}] ${name} (${type}-${model} ${serialNumber})"
}
Object getMetrics(String metric) {
switch (metric) {
case "SystemSharedProcessorPool":
return getSharedProcessorPools()
break
}
}
Map<String,BigDecimal> getMemoryMetrics() {
List<Map> getMemoryMetrics() {
HashMap<String, BigDecimal> map = [
List<Map> list = new ArrayList<>()
Map<String, Map> map = new HashMap<String, Map>()
HashMap<String, String> tagsMap = [
system: name,
]
map.put("tags", tagsMap)
log.debug(tagsMap.toString())
HashMap<String, BigDecimal> fieldsMap = [
totalMem: metrics.systemUtil.utilSamples.first().serverUtil.memory.totalMem.first(),
availableMem: metrics.systemUtil.utilSamples.first().serverUtil.memory.availableMem.first(),
configurableMem: metrics.systemUtil.utilSamples.first().serverUtil.memory.configurableMem.first(),
assignedMemToLpars: metrics.systemUtil.utilSamples.first().serverUtil.memory.assignedMemToLpars.first(),
]
map.put("fields", fieldsMap)
log.debug(fieldsMap.toString())
return map
list.add(map)
return list
}
Map<String,BigDecimal> getProcessorMetrics() {
List<Map> getProcessorMetrics() {
HashMap<String, BigDecimal> map = [
List<Map> list = new ArrayList<>()
Map<String, Map> map = new HashMap<String, Map>()
HashMap<String, String> tagsMap = [
system: name,
]
map.put("tags", tagsMap)
log.debug(tagsMap.toString())
HashMap<String, BigDecimal> fieldsMap = [
availableProcUnits: metrics.systemUtil.utilSamples.first().serverUtil.processor.totalProcUnits.first(),
utilizedProcUnits: metrics.systemUtil.utilSamples.first().serverUtil.processor.utilizedProcUnits.first(),
availableProcUnits: metrics.systemUtil.utilSamples.first().serverUtil.processor.availableProcUnits.first(),
configurableProcUnits: metrics.systemUtil.utilSamples.first().serverUtil.processor.configurableProcUnits.first(),
]
map.put("fields", fieldsMap)
log.debug(fieldsMap.toString())
return map
list.add(map)
return list
}
Map<String, BigDecimal> getPhysicalProcessorPool() {
HashMap<String, BigDecimal> map = [
assignedProcUnits: metrics.systemUtil.utilSamples.first().serverUtil.physicalProcessorPool.assignedProcUnits.first(),
availableProcUnits: metrics.systemUtil.utilSamples.first().serverUtil.physicalProcessorPool.availableProcUnits.first(),
]
return map
}
Map<String, Map<String, BigDecimal>> getSharedProcessorPools() {
List<Map> getSharedProcessorPools() {
List<Map> list = new ArrayList<>()
Map<String, Map> map = new HashMap<String, Map>()
metrics.systemUtil.utilSamples.first().serverUtil.sharedProcessorPool.each {
HashMap<String, BigDecimal> innerMap = [
HashMap<String, String> tagsMap = [
system: name,
pool: it.name,
]
map.put("tags", tagsMap)
log.debug(tagsMap.toString())
HashMap<String, BigDecimal> fieldsMap = [
assignedProcUnits: it.assignedProcUnits.first(),
availableProcUnits: it.availableProcUnits.first(),
]
map.put(it.name, innerMap)
map.put("fields", fieldsMap)
log.debug(fieldsMap.toString())
list.add(map)
}
return map
return list
}
List<Map> getSystemSharedAdapters() {
List<Map> list = new ArrayList<>()
Map<String, Map> map = new HashMap<String, Map>()
metrics.systemUtil.utilSamples.first().viosUtil.each {vios ->
vios.network.sharedAdapters.each {
HashMap<String, String> tagsMap = [
system: name,
type: it.type,
vios: vios.name,
]
map.put("tags", tagsMap)
log.debug(tagsMap.toString())
HashMap<String, BigDecimal> fieldsMap = [
sentBytes: it.sentBytes.first(),
transferredBytes: it.transferredBytes.first(),
]
map.put("fields", fieldsMap)
log.debug(fieldsMap.toString())
list.add(map)
}
}
return list
}
// SystemSharedAdapters
// SystemGenericPhysicalAdapters
// SystemGenericVirtualAdapters
// SystemGenericPhysicalAdapters
// SystemGenericAdapters
// SystemFiberChannelAdapters
List<Map> getSystemFiberChannelAdapters() {
List<Map> list = new ArrayList<>()
Map<String, Map> map = new HashMap<String, Map>()
metrics.systemUtil.utilSamples.first().viosUtil.each { vios ->
vios.storage.fiberChannelAdapters.each {
HashMap<String, String> tagsMap = [
system: name,
wwpn: it.wwpn,
vios: vios.name,
]
map.put("tags", tagsMap)
log.debug(tagsMap.toString())
HashMap<String, BigDecimal> fieldsMap = [
writeBytes: it.writeBytes.first(),
readBytes: it.readBytes.first(),
]
map.put("fields", fieldsMap)
log.debug(fieldsMap.toString())
list.add(map)
}
}
return list
}
}

View File

@ -8,7 +8,7 @@ class Storage {
List<String> clientLpars
List<GenericPhysicalAdapters> genericPhysicalAdapters
List<GenericVirtualAdapter> genericVirtualAdapters
List<VirtualFiberChannelAdapter> virtualFiberChannelAdapters
List<FiberChannelAdapter> fiberChannelAdapters
List<VirtualFiberChannelAdapter> virtualFiberChannelAdapters
}

View File

@ -1,6 +1,5 @@
package biz.nellemann.hmci
import okhttp3.mockwebserver.MockWebServer
import spock.lang.Specification
class InfluxClientTest extends Specification {
@ -17,7 +16,7 @@ class InfluxClientTest extends Specification {
}
void "write some managed system data to influx"() {
void "write ManagedSystem data to influx"() {
setup:
def testFile = new File(getClass().getResource('/pcm-data-managed-system.json').toURI())
@ -34,4 +33,22 @@ class InfluxClientTest extends Specification {
}
void "write LogicalPartition data to influx"() {
setup:
def testFile = new File(getClass().getResource('/pcm-data-logical-partition.json').toURI())
def testJson = testFile.getText('UTF-8')
when:
ManagedSystem system = new ManagedSystem("e09834d1-c930-3883-bdad-405d8e26e166", "TestSystem", "TestType", "TestModel", "Test s/n")
LogicalPartition lpar = new LogicalPartition("2DE05DB6-8AD5-448F-8327-0F488D287E82", "9Flash01", "OS400", system)
lpar.processMetrics(testJson)
influxClient.writeLogicalPartition(lpar)
then:
lpar.metrics.systemUtil.utilSamples.first().sampleInfo.status == 2
}
}

View File

@ -19,8 +19,29 @@ class LogicalPartitionTest extends Specification {
then:
lpar.metrics.systemUtil.utilSamples.first().lparsUtil.first().memory.logicalMem.first() == 8192.000
lpar.metrics.systemUtil.utilSamples.first().lparsUtil.first().processor.utilizedProcUnits.first() == 0.001
lpar.metrics.systemUtil.utilSamples.first().lparsUtil.first().network.virtualEthernetAdapters.first().receivedPackets.first() == 3.867
lpar.metrics.systemUtil.utilSamples.first().lparsUtil.first().network.virtualEthernetAdapters.first().receivedBytes.first() == 276.467
}
void "test getVirtualEthernetAdapterMetrics"() {
setup:
def testFile = new File(getClass().getResource('/pcm-data-logical-partition.json').toURI())
def testJson = testFile.getText('UTF-8')
ManagedSystem system = new ManagedSystem("e09834d1-c930-3883-bdad-405d8e26e166", "Test Name","Test Type", "Test Model", "Test S/N")
LogicalPartition lpar = new LogicalPartition("2DE05DB6-8AD5-448F-8327-0F488D287E82", "9Flash01", "OS400", system)
when:
lpar.processMetrics(testJson)
List<Map> listOfMaps = lpar.getVirtualEthernetAdapterMetrics()
then:
listOfMaps.size() == 1
listOfMaps.first().get("fields")['receivedBytes'] == 276.467
listOfMaps.first().get("tags")['sea'] == 'ent5'
}
}

View File

@ -24,4 +24,72 @@ class ManagedSystemTest extends Specification {
}
void "test getMemoryMetrics"() {
setup:
def testFile = new File(getClass().getResource('/pcm-data-managed-system.json').toURI())
def testJson = testFile.getText('UTF-8')
ManagedSystem system = new ManagedSystem("e09834d1-c930-3883-bdad-405d8e26e166", "Test Name","Test Type", "Test Model", "Test S/N")
when:
system.processMetrics(testJson)
List<Map> listOfMaps = system.getMemoryMetrics()
then:
listOfMaps.size() == 1
listOfMaps.first().get("fields")['totalMem'] == 1048576.000
}
void "test getProcessorMetrics"() {
setup:
def testFile = new File(getClass().getResource('/pcm-data-managed-system.json').toURI())
def testJson = testFile.getText('UTF-8')
ManagedSystem system = new ManagedSystem("e09834d1-c930-3883-bdad-405d8e26e166", "Test Name","Test Type", "Test Model", "Test S/N")
when:
system.processMetrics(testJson)
List<Map> listOfMaps = system.getProcessorMetrics()
then:
listOfMaps.size() == 1
listOfMaps.first().get("fields")['availableProcUnits'] == 16.000
}
void "test getSystemSharedProcessorPools"() {
setup:
def testFile = new File(getClass().getResource('/pcm-data-managed-system.json').toURI())
def testJson = testFile.getText('UTF-8')
ManagedSystem system = new ManagedSystem("e09834d1-c930-3883-bdad-405d8e26e166", "Test Name","Test Type", "Test Model", "Test S/N")
when:
system.processMetrics(testJson)
List<Map> listOfMaps = system.getSharedProcessorPools()
then:
listOfMaps.size() == 1
listOfMaps.first().get("fields")['assignedProcUnits'] == 23.767
}
/*
void "test getSharedAdapterMetrics"() {
setup:
def testFile = new File(getClass().getResource('/pcm-data-logical-partition.json').toURI())
def testJson = testFile.getText('UTF-8')
ManagedSystem system = new ManagedSystem("e09834d1-c930-3883-bdad-405d8e26e166", "Test Name","Test Type", "Test Model", "Test S/N")
LogicalPartition lpar = new LogicalPartition("2DE05DB6-8AD5-448F-8327-0F488D287E82", "9Flash01", "OS400", system)
when:
lpar.processMetrics(testJson)
List<Map> listOfMaps = lpar.getSharedAdapterMetrics()
then:
listOfMaps.size() == 1
listOfMaps.first().get("fields")['receivedBytes'] == 276.467
listOfMaps.first().get("tags")['sea'] == 'ent5'
}*/
}