diff --git a/CHANGELOG.md b/CHANGELOG.md index 2c1a202..53849ca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,11 @@ All notable changes to this project will be documented in this file. +## 1.4.4 - 2023-05-20 +- Support for InfluxDB v2, now requires InfluxDB 1.8 or later +- Increase influx writer buffer limit +- Various dashboard improvements + ## 1.4.3 - 2023-03-21 - Fix and improve processor utilization dashboards. - Minor code cleanup. diff --git a/README.md b/README.md index 3966537..4df8101 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ Metrics includes: - *Managed Systems* - the physical Power servers - *Logical Partitions* - the virtualized servers running AIX, Linux and/or IBM-i (AS/400) - *Virtual I/O Servers* - the i/o partition(s) virtualizing network and storage - - *Energy* - watts and temperatures (needs to be enabled and is not available on multi-chassis systems) + - *Energy* - watts and temperatures (needs to be enabled and is not available on all systems) ![architecture](doc/HMCi.png) @@ -66,6 +66,13 @@ Read the [readme-grafana.md](doc/readme-grafana.md) file for instructions and he This is most likely due to timezone, date and/or NTP not being configured correctly on the HMC and/or host running HMCi. +You can check the timestamp of the most recent data by querying InfluxDB with the ```influx``` CLI client, and take note of the timezone when comparing: + +```sql +use hmci; +precision rfc3339; +SELECT * FROM server_details GROUP BY * ORDER BY DESC LIMIT 1; +``` ### Compatibility with nextract Plus @@ -126,6 +133,7 @@ If you rename a partition, the metrics in InfluxDB will still be available by th DELETE WHERE lparname = 'name'; ``` + ## Development Information You need Java (JDK) version 8 or later to build hmci. @@ -141,7 +149,7 @@ Use the gradle build tool, which will download all required dependencies: ### Local Testing -#### InfluxDB +#### InfluxDB v1.x Start a InfluxDB container: @@ -155,6 +163,18 @@ Create the *hmci* database: docker exec -i influxdb influx -execute "CREATE DATABASE hmci" ``` +#### InfluxDB v2.x + +Start a InfluxDB container: + +```shell +docker pull influxdb:latest +docker run --name=influxdb --rm -d -p 8086:8086 influxdb:latest +``` + +- Then use the Web UI to create an initial user (for the web UI), an organization and bucket: http://localhost:8086/ +- Then create an API token with RW access to your bucket. + #### Grafana @@ -166,4 +186,7 @@ docker run --name grafana --link influxdb:influxdb --rm -d -p 3000:3000 grafana/ Setup Grafana to connect to the InfluxDB container by defining a new datasource on URL *http://influxdb:8086* named *hmci*. +If you are [connecting](https://docs.influxdata.com/influxdb/v2.7/tools/grafana/) to InfluxDB v2.x, then add a custom http header, enter bucket as database and disable authorization. +- Authorization = Token abcdef_random_token_from_nfluxdb== + Import dashboards from the [doc/dashboards/](doc/dashboards/) folder. diff --git a/build.gradle b/build.gradle index 105a475..57476aa 100644 --- a/build.gradle +++ b/build.gradle @@ -1,13 +1,10 @@ plugins { id 'java' + id 'jacoco' id 'groovy' id 'application' - - // Code coverage of tests - id 'jacoco' - id "net.nemerosa.versioning" version "2.15.1" - id "com.netflix.nebula.ospackage" version "10.0.0" + id "com.netflix.nebula.ospackage" version "11.2.0" id "com.github.johnrengelman.shadow" version "7.1.2" } @@ -20,19 +17,18 @@ group = projectGroup version = projectVersion dependencies { - annotationProcessor 'info.picocli:picocli-codegen:4.7.1' - implementation 'info.picocli:picocli:4.7.1' - implementation 'org.influxdb:influxdb-java:2.23' - //implementation 'com.influxdb:influxdb-client-java:6.7.0' - implementation 'org.slf4j:slf4j-api:2.0.6' - implementation 'org.slf4j:slf4j-simple:2.0.6' + annotationProcessor 'info.picocli:picocli-codegen:4.7.3' + implementation 'info.picocli:picocli:4.7.3' + implementation 'org.slf4j:slf4j-api:2.0.7' + implementation 'org.slf4j:slf4j-simple:2.0.7' implementation 'com.squareup.okhttp3:okhttp:4.10.0' // Also used by InfluxDB Client - implementation 'com.fasterxml.jackson.core:jackson-databind:2.14.2' - implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-xml:2.14.2' - implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-toml:2.14.2' + implementation 'com.influxdb:influxdb-client-java:6.8.0' + implementation 'com.fasterxml.jackson.core:jackson-databind:2.14.3' + implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-xml:2.14.3' + implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-toml:2.14.3' testImplementation 'junit:junit:4.13.2' - testImplementation 'org.spockframework:spock-core:2.3-groovy-3.0' + testImplementation 'org.spockframework:spock-core:2.3-groovy-4.0' testImplementation "org.mock-server:mockserver-netty-no-dependencies:5.14.0" } @@ -87,7 +83,7 @@ buildDeb { } jacoco { - toolVersion = "0.8.8" + toolVersion = "0.8.9" } jacocoTestReport { diff --git a/doc/HMCi.drawio b/doc/HMCi.drawio index b9e95a6..c923cf6 100644 --- a/doc/HMCi.drawio +++ b/doc/HMCi.drawio @@ -1 +1 @@ -5VrbcqM4EP0aPybFHfwY27nNJJnUpqZm3qYECKwNSIwQvszXrwTCtrg4OBvKqd08JNBqhNSnz1FLZGLO080tBdnykYQwmRhauJmYi4lh6Lrm8D/Csq0s06lbGWKKQum0N7ygP1AaNWktUAhzxZERkjCUqcaAYAwDptgApWStukUkUd+agRi2DC8BSNrWHyhky3peznTfcAdRvJSv9gw5vxTUznIm+RKEZH1gMq8n5pwSwqqrdDOHiQheHZfquZue1t3AKMRsyAPw64UBgQFW68X3JzyznvBNcWHJsbFtPWEY8vnLW0LZksQEg+R6b51RUuAQil51frf3eSAkk8a/IWNbCSYoGOGmJUsT2Zq/QhYs5Q3E4ZXAid9igmFluUGJcNaEMwOU1R5+QoLX2iidRCftWMjw5KSgATwSgDqnAI0hO+JnVn4iOgcvkJG+hSSFjG65A4UJYGilZg+QSRjv/PY48QsJ1QmwyX5XICnkmyaGk/Dxz3zKr2JWxqRpucdRUmwWHS0rSC95f/rlppUPPG0zcRlsE8Rhp3zSs/USMfiSgTKya058FV+/SpAHf2cAwWtcps23gvFuYJ0IVYboNr+OOJpzkhBavtaMIugEQQk1Ja/woCV0p76mNdOoNwP41BjcHMVMtlqOZKqUKteubtd73tcey0PGayOBbJ+dm++nlTGQVvqnopXRS6sQrVq8qg13j/xt2j3ORU7kdTMfgN98hNuUjnqollESwDwfQLQeXjW4BPXQhm4Xl6aOawKnCftupRIsixMghlJeq9n1AZQz3KlCOa9NOd3o4JwzFufMIRnAS50IYNCHast/AfKlTwAN88GP8L8gFWhXv/mQrhIeVYTjfqEfmF0hCYq0hOxkHW9pdGT0aLTjO3Yrrz4gYWz3TY3uTJjRRNo9u0gPKpDeqrLer/TOQKU3zqX0bUpnPOSi2jEsy+yE7wH4fBujhBwkKMaiDOIBgrQqmRji+4Qr2ZCiMKzQhbysAX7ZnwhtRhBm5fjt2cRedAb7WGq1yLHb7MiXKPuJLtJcaJeWY+oKcWqlGxxu2fuzmM1B1w0+WobaA4minKdFE67dGN9PPGeIUt/PHofrZevpZ7LmUBtaucAP0liV2W/Ia0NNQwC9qFNNncCDfjRGxauCVwvnoZjaHWLqjiWm03OIachX593zn0lZvc+urMdGPZiX52ZhP9uUxOgl67iU1K2OAmcsTqLkV7zKpnD7dPX1Jv29Xd1Fvzv2RH8VmBeyGsGiKr3/ORGy0axWHxAu2mcJJwFzLPxR+SN2JxSEiEe7bpMUHPtoQHe8yzYwhtkBjDUWMIO2Kl/Aiu9TNE5KgQpHjqEUSvDevS4CLEIju86LTNRTHCkeYy4wEaHp5RhE/Uz5sNtP7PKhY+ns2ofY7r/PBquvoPU0z/kc9WxfQE8uXa16RnV96Z1aoZoqUI34f0h9avcBYmqa9h8DxNbUzD+2se4GRDcMpYcL3RkBE+8cpaRSMNbSc1Avamq9GIijNRQoJWO7JAlhBIpSTU8/m3UHlpBn+7pR73EPF7Gec6siTa4CRg6pUdLomeSIISIo4hPGSNrBHUYaiwmpzkvnu6+GY3xQ8PRmMddVNXQVDR9xvnlXLKKc/ZpZtxfBl58Y3KP19/PssE7P286xD90PnS2Zj436/5HhHScIH5Te/Hb/qbxaD/b/cGBe/wM= \ No newline at end of file + \ No newline at end of file diff --git a/doc/HMCi.png b/doc/HMCi.png index ca3b45c..87a271e 100644 Binary files a/doc/HMCi.png and b/doc/HMCi.png differ diff --git a/doc/dashboards/HMCi - Power LPAR Overview.json b/doc/dashboards/HMCi - Power LPAR Overview.json index a6339b8..7c63fe2 100644 --- a/doc/dashboards/HMCi - Power LPAR Overview.json +++ b/doc/dashboards/HMCi - Power LPAR Overview.json @@ -529,6 +529,137 @@ "title": "Processors", "type": "row" }, + { + "datasource": { + "type": "influxdb", + "uid": "${DS_HMCI}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 3, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "decimals": 2, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 12 + }, + "id": 2, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "8.1.4", + "targets": [ + { + "alias": "$tag_lparname", + "datasource": { + "type": "influxdb", + "uid": "${DS_HMCI}" + }, + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "/^$ServerName$/", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT mean(\"utilizedProcUnits\") AS \"usage\" FROM \"lpar_processor\" WHERE (\"servername\" =~ /^$ServerName$/ AND \"lparname\" =~ /^$LPAR$/) AND $timeFilter GROUP BY time($interval), \"lparname\", \"servername\" fill(linear)", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "title": "Processor Units - Utilization Stacked", + "transformations": [], + "type": "timeseries" + }, { "datasource": { "type": "influxdb", @@ -597,11 +728,11 @@ }, "gridPos": { "h": 11, - "w": 24, - "x": 0, + "w": 12, + "x": 12, "y": 12 }, - "id": 2, + "id": 40, "links": [], "options": { "legend": { @@ -662,7 +793,7 @@ "tags": [] } ], - "title": "Processor Units - Utilization / Entitled Percentage", + "title": "Processor Units - Utilization / Entitled", "transformations": [], "type": "timeseries" }, @@ -2510,7 +2641,7 @@ "spanNulls": false, "stacking": { "group": "A", - "mode": "percent" + "mode": "normal" }, "thresholdsStyle": { "mode": "off" @@ -2522,10 +2653,6 @@ "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, @@ -2617,11 +2744,11 @@ ] } ], - "title": "Memory Assigned", + "title": "Memory Assigned - Stacked", "type": "timeseries" } ], - "refresh": false, + "refresh": "30s", "schemaVersion": 37, "style": "dark", "tags": [ @@ -2660,7 +2787,7 @@ "type": "influxdb", "uid": "${DS_HMCI}" }, - "definition": "SHOW TAG VALUES FROM \"lpar_processor\" WITH KEY = \"lparname\" WHERE servername =~ /$ServerName/", + "definition": "SHOW TAG VALUES FROM \"lpar_processor\" WITH KEY = \"lparname\" WHERE servername =~ /$ServerName/ ", "hide": 0, "includeAll": true, "label": "Logical Partition", @@ -2668,7 +2795,7 @@ "multiFormat": "regex values", "name": "LPAR", "options": [], - "query": "SHOW TAG VALUES FROM \"lpar_processor\" WITH KEY = \"lparname\" WHERE servername =~ /$ServerName/", + "query": "SHOW TAG VALUES FROM \"lpar_processor\" WITH KEY = \"lparname\" WHERE servername =~ /$ServerName/ ", "refresh": 1, "refresh_on_load": false, "regex": "", @@ -2711,6 +2838,6 @@ "timezone": "browser", "title": "HMCi - Power LPAR Overview", "uid": "Xl7oHESGz", - "version": 3, + "version": 9, "weekStart": "" } diff --git a/doc/dashboards/HMCi - Power System Energy.json b/doc/dashboards/HMCi - Power System Energy.json index ea7fde9..90a0a83 100644 --- a/doc/dashboards/HMCi - Power System Energy.json +++ b/doc/dashboards/HMCi - Power System Energy.json @@ -107,6 +107,21 @@ "transparent": true, "type": "text" }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 3 + }, + "id": 15, + "panels": [], + "repeat": "ServerName", + "repeatDirection": "h", + "title": "$ServerName", + "type": "row" + }, { "datasource": { "type": "influxdb", @@ -140,7 +155,7 @@ "h": 7, "w": 24, "x": 0, - "y": 3 + "y": 4 }, "id": 7, "options": { @@ -250,7 +265,7 @@ "h": 11, "w": 8, "x": 0, - "y": 10 + "y": 11 }, "id": 4, "options": { @@ -453,7 +468,7 @@ "h": 11, "w": 16, "x": 8, - "y": 10 + "y": 11 }, "id": 12, "options": { @@ -629,7 +644,7 @@ "h": 10, "w": 8, "x": 0, - "y": 21 + "y": 22 }, "id": 13, "options": { @@ -779,7 +794,7 @@ "h": 10, "w": 16, "x": 8, - "y": 21 + "y": 22 }, "id": 5, "options": { @@ -874,13 +889,13 @@ "type": "influxdb", "uid": "${DS_HMCI}" }, - "definition": "SHOW TAG VALUES FROM \"server_processor\" WITH KEY = \"servername\" WHERE time > now() - 24h", + "definition": "SHOW TAG VALUES FROM \"server_energy_power\" WITH KEY = \"servername\" WHERE time > now() - 24h", "hide": 0, - "includeAll": false, - "multi": false, + "includeAll": true, + "multi": true, "name": "ServerName", "options": [], - "query": "SHOW TAG VALUES FROM \"server_processor\" WITH KEY = \"servername\" WHERE time > now() - 24h", + "query": "SHOW TAG VALUES FROM \"server_energy_power\" WITH KEY = \"servername\" WHERE time > now() - 24h", "refresh": 1, "regex": "", "skipUrlSync": false, @@ -912,6 +927,6 @@ "timezone": "", "title": "HMCi - Power System Energy", "uid": "oHcrgD1Mk", - "version": 2, + "version": 7, "weekStart": "" } diff --git a/doc/dashboards/HMCi - Power System Overview.json b/doc/dashboards/HMCi - Power System Overview.json index 2cb4f27..3c2fb1f 100644 --- a/doc/dashboards/HMCi - Power System Overview.json +++ b/doc/dashboards/HMCi - Power System Overview.json @@ -11,12 +11,6 @@ ], "__elements": {}, "__requires": [ - { - "type": "panel", - "id": "bargauge", - "name": "Bar gauge", - "version": "" - }, { "type": "panel", "id": "gauge", @@ -35,12 +29,6 @@ "name": "InfluxDB", "version": "1.0.0" }, - { - "type": "panel", - "id": "stat", - "name": "Stat", - "version": "" - }, { "type": "panel", "id": "text", @@ -133,7 +121,7 @@ "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", - "fillOpacity": 10, + "fillOpacity": 4, "gradientMode": "opacity", "hideFrom": { "legend": false, @@ -159,7 +147,6 @@ "decimals": 2, "links": [], "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ @@ -174,7 +161,7 @@ "overrides": [] }, "gridPos": { - "h": 6, + "h": 7, "w": 12, "x": 0, "y": 3 @@ -186,17 +173,17 @@ "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": true + "showLegend": false }, "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "none" } }, "pluginVersion": "8.1.4", "targets": [ { - "alias": "$col", + "alias": "$col ($tag_servername)", "datasource": { "type": "influxdb", "uid": "${DS_HMCI}" @@ -311,7 +298,7 @@ ] } ], - "title": "System Processor Utilization", + "title": "System Processors", "type": "timeseries" }, { @@ -332,20 +319,23 @@ "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "opacity", + "fillOpacity": 4, + "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, - "showPoints": "never", + "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", @@ -363,10 +353,6 @@ { "color": "green", "value": null - }, - { - "color": "red", - "value": 80 } ] }, @@ -375,7 +361,7 @@ "overrides": [] }, "gridPos": { - "h": 6, + "h": 7, "w": 12, "x": 12, "y": 3 @@ -387,7 +373,7 @@ "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": true + "showLegend": false }, "tooltip": { "mode": "multi", @@ -397,7 +383,7 @@ "pluginVersion": "8.1.4", "targets": [ { - "alias": "$col", + "alias": " $col ($tag_servername)", "datasource": { "type": "influxdb", "uid": "${DS_HMCI}" @@ -476,7 +462,7 @@ ] } ], - "title": "System Memory Utilization", + "title": "System Memory", "type": "timeseries" }, { @@ -497,7 +483,7 @@ "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", - "fillOpacity": 10, + "fillOpacity": 4, "gradientMode": "opacity", "hideFrom": { "legend": false, @@ -506,11 +492,11 @@ }, "lineInterpolation": "linear", "lineWidth": 1, - "pointSize": 5, + "pointSize": 3, "scaleDistribution": { "type": "linear" }, - "showPoints": "never", + "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", @@ -530,10 +516,6 @@ { "color": "green", "value": null - }, - { - "color": "red", - "value": 80 } ] }, @@ -543,9 +525,9 @@ }, "gridPos": { "h": 9, - "w": 15, + "w": 12, "x": 0, - "y": 9 + "y": 10 }, "id": 19, "links": [], @@ -554,17 +536,17 @@ "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": true + "showLegend": false }, "tooltip": { "mode": "multi", - "sort": "desc" + "sort": "none" } }, "pluginVersion": "8.1.4", "targets": [ { - "alias": "$tag_poolname", + "alias": "$tag_poolname ($tag_servername)", "datasource": { "type": "influxdb", "uid": "${DS_HMCI}" @@ -633,9 +615,124 @@ ] } ], - "title": "Shared Processor Pools - Core Utilization", + "title": "Shared Processor Pools", "type": "timeseries" }, + { + "datasource": { + "type": "influxdb", + "uid": "${DS_HMCI}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "#EAB839", + "value": 0.75 + }, + { + "color": "red", + "value": 0.85 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 12, + "y": 10 + }, + "id": 29, + "options": { + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "alias": "$tag_poolname ($tag_servername)", + "datasource": { + "type": "influxdb", + "uid": "${DS_HMCI}" + }, + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "poolname" + ], + "type": "tag" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "server_sharedProcessorPool", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT mean(\"utilizedProcUnits\") / mean(\"assignedProcUnits\") AS \"Utilization\" FROM \"server_sharedProcessorPool\" WHERE (\"servername\" =~ /^$ServerName$/) AND $timeFilter GROUP BY time($__interval), \"poolname\", \"servername\" fill(none)", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "utilizedProcUnits" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "servername", + "operator": "=~", + "value": "/^$ServerName$/" + } + ] + } + ], + "title": "Shared Processor Pools", + "type": "gauge" + }, { "datasource": { "type": "influxdb", @@ -649,116 +746,6 @@ "min": 0, "thresholds": { "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 5, - "x": 15, - "y": 9 - }, - "id": 29, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.1.6", - "targets": [ - { - "alias": "$tag_poolname", - "datasource": { - "type": "influxdb", - "uid": "${DS_HMCI}" - }, - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "poolname" - ], - "type": "tag" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "server_sharedProcessorPool", - "orderByTime": "ASC", - "policy": "default", - "query": "SELECT mean(\"utilizedProcUnits\") / mean(\"assignedProcUnits\") AS \"Utilization\" FROM \"server_sharedProcessorPool\" WHERE (\"servername\" =~ /^$ServerName$/) AND $timeFilter GROUP BY time($__interval), \"poolname\" fill(none)", - "rawQuery": true, - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "utilizedProcUnits" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - } - ] - ], - "tags": [ - { - "key": "servername", - "operator": "=~", - "value": "/^$ServerName$/" - } - ] - } - ], - "title": "Shared Processor Pools - Utilization", - "type": "stat" - }, - { - "datasource": { - "type": "influxdb", - "uid": "${DS_HMCI}" - }, - "fieldConfig": { - "defaults": { - "decimals": 2, - "mappings": [], - "max": 100, - "min": 0, - "thresholds": { - "mode": "percentage", "steps": [ { "color": "green", @@ -766,11 +753,11 @@ }, { "color": "orange", - "value": 70 + "value": 0.75 }, { "color": "red", - "value": 85 + "value": 0.85 } ] }, @@ -782,7 +769,7 @@ "h": 9, "w": 4, "x": 20, - "y": 9 + "y": 10 }, "id": 30, "options": { @@ -800,7 +787,7 @@ "pluginVersion": "9.1.6", "targets": [ { - "alias": "", + "alias": "$tag_servername", "datasource": { "type": "influxdb", "uid": "${DS_HMCI}" @@ -828,7 +815,7 @@ "measurement": "server_sharedProcessorPool", "orderByTime": "ASC", "policy": "default", - "query": "SELECT mean(\"utilizedProcUnits\") / mean(\"totalProcUnits\") AS \"Utilization\" FROM \"server_processor\" WHERE (\"servername\" =~ /^$ServerName$/) AND $timeFilter GROUP BY time($__interval) fill(none)", + "query": "SELECT mean(\"utilizedProcUnits\") / mean(\"totalProcUnits\") AS \"Utilization\" FROM \"server_processor\" WHERE (\"servername\" =~ /^$ServerName$/) AND $timeFilter GROUP BY \"servername\", time($__interval) fill(none)", "rawQuery": true, "refId": "A", "resultFormat": "time_series", @@ -855,7 +842,7 @@ ] } ], - "title": "System Processors - Utilization", + "title": "System Processors", "type": "gauge" }, { @@ -920,7 +907,7 @@ "h": 9, "w": 12, "x": 0, - "y": 18 + "y": 19 }, "id": 27, "links": [], @@ -929,7 +916,7 @@ "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": true + "showLegend": false }, "tooltip": { "mode": "multi", @@ -939,7 +926,7 @@ "pluginVersion": "8.1.4", "targets": [ { - "alias": "$col", + "alias": "$col ($tag_servername)", "datasource": { "type": "influxdb", "uid": "${DS_HMCI}" @@ -1008,24 +995,6 @@ "type": "alias" } ], - [ - { - "params": [ - "availableProcUnits" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - }, - { - "params": [ - "available" - ], - "type": "alias" - } - ], [ { "params": [ @@ -1054,7 +1023,7 @@ ] } ], - "title": "Physical Processor Pool - Core Utilization", + "title": "Physical Processor Pool", "type": "timeseries" }, { @@ -1119,7 +1088,7 @@ "h": 9, "w": 12, "x": 12, - "y": 18 + "y": 19 }, "id": 35, "links": [], @@ -1128,7 +1097,7 @@ "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": true + "showLegend": false }, "tooltip": { "mode": "multi", @@ -1138,7 +1107,7 @@ "pluginVersion": "8.1.4", "targets": [ { - "alias": "$tag_poolname", + "alias": "$tag_poolname ($tag_servername)", "datasource": { "type": "influxdb", "uid": "${DS_HMCI}" @@ -1157,6 +1126,12 @@ ], "type": "tag" }, + { + "params": [ + "servername" + ], + "type": "tag" + }, { "params": [ "none" @@ -1199,7 +1174,7 @@ ] } ], - "title": "Shared Processor Pools - Core Utilization", + "title": "Shared Processor Pools", "type": "timeseries" }, { @@ -1295,7 +1270,7 @@ "h": 9, "w": 12, "x": 0, - "y": 27 + "y": 28 }, "id": 11, "links": [], @@ -1304,7 +1279,7 @@ "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": true + "showLegend": false }, "tooltip": { "mode": "multi", @@ -1314,7 +1289,7 @@ "pluginVersion": "8.1.4", "targets": [ { - "alias": "vlan $tag_vlanId", + "alias": "vlan $tag_vlanId ($tag_servername)", "datasource": { "type": "influxdb", "uid": "${DS_HMCI}" @@ -1339,6 +1314,12 @@ ], "type": "tag" }, + { + "params": [ + "servername" + ], + "type": "tag" + }, { "params": [ "none" @@ -1375,7 +1356,7 @@ ] } ], - "title": "Network Throughput - By VLAN", + "title": "Network Throughput - Transferred By VLAN", "type": "timeseries" }, { @@ -1442,7 +1423,7 @@ "h": 9, "w": 12, "x": 12, - "y": 27 + "y": 28 }, "id": 16, "links": [], @@ -1451,7 +1432,7 @@ "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": true + "showLegend": false }, "tooltip": { "mode": "multi", @@ -1461,7 +1442,7 @@ "pluginVersion": "8.1.4", "targets": [ { - "alias": "$tag_location $col", + "alias": "$tag_location ($tag_servername)", "datasource": { "type": "influxdb", "uid": "${DS_HMCI}" @@ -1486,6 +1467,12 @@ ], "type": "tag" }, + { + "params": [ + "servername" + ], + "type": "tag" + }, { "params": [ "none" @@ -1522,450 +1509,7 @@ ] } ], - "title": "Physical Fiber Channel Adapters", - "type": "timeseries" - }, - { - "datasource": { - "type": "influxdb", - "uid": "${DS_HMCI}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "continuous-BlYlRd" - }, - "decimals": 1, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "celsius" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 36 - }, - "id": 26, - "options": { - "displayMode": "lcd", - "minVizHeight": 10, - "minVizWidth": 0, - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showUnfilled": true - }, - "pluginVersion": "9.1.6", - "targets": [ - { - "alias": "$col", - "datasource": { - "type": "influxdb", - "uid": "${DS_HMCI}" - }, - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "system" - ], - "type": "tag" - }, - { - "params": [ - "none" - ], - "type": "fill" - } - ], - "measurement": "server_energy_thermal", - "orderByTime": "ASC", - "policy": "default", - "query": "SELECT mean(\"cpuTemperature*\") FROM \"server_energy_thermal\" WHERE (\"servername\" =~ /^$ServerName$/) AND $timeFilter GROUP BY time($__interval), \"system\", \"servername\" fill(linear)", - "rawQuery": false, - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "cpuTemperature_1" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - }, - { - "params": [ - "CPU-1" - ], - "type": "alias" - } - ], - [ - { - "params": [ - "cpuTemperature_2" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - }, - { - "params": [ - "CPU-2" - ], - "type": "alias" - } - ], - [ - { - "params": [ - "cpuTemperature_3" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - }, - { - "params": [ - "CPU-3" - ], - "type": "alias" - } - ], - [ - { - "params": [ - "cpuTemperature_4" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - }, - { - "params": [ - "CPU-4" - ], - "type": "alias" - } - ] - ], - "tags": [ - { - "key": "servername", - "operator": "=~", - "value": "/^$ServerName$/" - } - ] - } - ], - "title": "CPU Temperature", - "type": "bargauge" - }, - { - "datasource": { - "type": "influxdb", - "uid": "${DS_HMCI}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "continuous-BlYlRd" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "scheme", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 3, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 6, - "x": 12, - "y": 36 - }, - "id": 34, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "8.1.4", - "targets": [ - { - "alias": "$tag_servername", - "datasource": { - "type": "influxdb", - "uid": "${DS_HMCI}" - }, - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "servername" - ], - "type": "tag" - }, - { - "params": [ - "none" - ], - "type": "fill" - } - ], - "measurement": "server_energy_thermal", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "inletTemperature_1" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - } - ] - ], - "tags": [ - { - "key": "servername", - "operator": "=~", - "value": "/^$ServerName$/" - } - ] - } - ], - "title": "Inlet Temperature", - "type": "timeseries" - }, - { - "datasource": { - "type": "influxdb", - "uid": "${DS_HMCI}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "continuous-GrYlRd" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "scheme", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 3, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 6, - "x": 18, - "y": 36 - }, - "id": 22, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "8.1.4", - "targets": [ - { - "alias": "$tag_servername", - "datasource": { - "type": "influxdb", - "uid": "${DS_HMCI}" - }, - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "servername" - ], - "type": "tag" - }, - { - "params": [ - "none" - ], - "type": "fill" - } - ], - "measurement": "server_energy_power", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "powerReading" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - } - ] - ], - "tags": [ - { - "key": "servername", - "operator": "=~", - "value": "/^$ServerName$/" - } - ] - } - ], - "title": "Power Consumption", + "title": "Physical Fiber Channel Adapters - Transmitted", "type": "timeseries" } ], @@ -1986,9 +1530,9 @@ }, "definition": "SHOW TAG VALUES FROM \"server_processor\" WITH KEY = \"servername\" WHERE time > now() - 24h", "hide": 0, - "includeAll": false, + "includeAll": true, "label": "Server", - "multi": false, + "multi": true, "multiFormat": "regex values", "name": "ServerName", "options": [], @@ -2035,6 +1579,6 @@ "timezone": "browser", "title": "HMCi - Power System Overview", "uid": "ClJhHPIGz", - "version": 3, + "version": 10, "weekStart": "" } diff --git a/doc/dashboards/HMCi - Power VIO Overview.json b/doc/dashboards/HMCi - Power VIO Overview.json index 853214f..75fc203 100644 --- a/doc/dashboards/HMCi - Power VIO Overview.json +++ b/doc/dashboards/HMCi - Power VIO Overview.json @@ -1390,7 +1390,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null } ] }, @@ -1563,7 +1564,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1706,7 +1708,7 @@ }, "definition": "SHOW TAG VALUES FROM \"server_processor\" WITH KEY = \"servername\" WHERE time > now() - 24h", "hide": 0, - "includeAll": false, + "includeAll": true, "label": "Server", "multi": true, "multiFormat": "regex values", @@ -1779,6 +1781,6 @@ "timezone": "browser", "title": "HMCi - Power VIO Overview", "uid": "DDNEv5vGz", - "version": 2, + "version": 3, "weekStart": "" } diff --git a/doc/hmci.toml b/doc/hmci.toml index e798ad2..64cccf2 100644 --- a/doc/hmci.toml +++ b/doc/hmci.toml @@ -3,14 +3,23 @@ ### ### Define one InfluxDB to save metrics into +### There must be only one and it should be named [influx] ### +# InfluxDB v1.x example +#[influx] +#url = "http://localhost:8086" +#username = "root" +#password = "" +#database = "hmci" + + +# InfluxDB v2.x example [influx] url = "http://localhost:8086" -username = "root" -password = "" -database = "hmci" - +org = "myOrg" +token = "rAnd0mT0k3nG3neRaT3dByInF1uxDb==" +bucket = "hmci" ### diff --git a/doc/readme-grafana.md b/doc/readme-grafana.md index 36d3a17..0533f72 100644 --- a/doc/readme-grafana.md +++ b/doc/readme-grafana.md @@ -7,6 +7,7 @@ When installed Grafana listens on [http://localhost:3000](http://localhost:3000) - Configure Grafana to use InfluxDB as a new datasource - Name the datasource **hmci** to make it obvious what it contains. - You would typically use *http://localhost:8086* without any credentials. + - For InfluxDB 2.x add a custom header: Authorization = Token myTokenFromInfluxDB - The name of the database would be *hmci* (or another name you used when creating it) - **NOTE:** set *Min time interval* to *30s* or *1m* depending on your HMCi *refresh* setting. diff --git a/doc/readme-influxdb.md b/doc/readme-influxdb.md new file mode 100644 index 0000000..81d7c4f --- /dev/null +++ b/doc/readme-influxdb.md @@ -0,0 +1,10 @@ +# InfluxDB Notes + + +## Delete data + +To delete *all* data before a specific date, run: + +```sql +DELETE WHERE time < '2023-01-01' +``` diff --git a/gradle.properties b/gradle.properties index a7b2fcb..91253d0 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,3 +1,3 @@ projectId = hmci projectGroup = biz.nellemann.hmci -projectVersion = 1.4.3 +projectVersion = 1.4.4 diff --git a/src/main/java/biz/nellemann/hmci/InfluxClient.java b/src/main/java/biz/nellemann/hmci/InfluxClient.java index 21e5afc..90803b7 100644 --- a/src/main/java/biz/nellemann/hmci/InfluxClient.java +++ b/src/main/java/biz/nellemann/hmci/InfluxClient.java @@ -16,65 +16,82 @@ package biz.nellemann.hmci; import biz.nellemann.hmci.dto.toml.InfluxConfiguration; -import org.influxdb.BatchOptions; -import org.influxdb.InfluxDB; -import org.influxdb.InfluxDBFactory; -import org.influxdb.dto.Point; +import com.influxdb.client.InfluxDBClient; +import com.influxdb.client.InfluxDBClientFactory; +import com.influxdb.client.WriteApi; +import com.influxdb.client.WriteOptions; +import com.influxdb.client.write.Point; +import com.influxdb.client.domain.WritePrecision; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.ArrayList; import java.util.List; -import java.util.concurrent.TimeUnit; import static java.lang.Thread.sleep; + public final class InfluxClient { private final static Logger log = LoggerFactory.getLogger(InfluxClient.class); final private String url; - final private String username; - final private String password; - final private String database; + final private String org; // v2 only + final private String token; + final private String bucket; // Bucket in v2, Database in v1 + + + private InfluxDBClient influxDBClient; + private WriteApi writeApi; - private InfluxDB influxDB; InfluxClient(InfluxConfiguration config) { this.url = config.url; - this.username = config.username; - this.password = config.password; - this.database = config.database; + if(config.org != null) { + this.org = config.org; + } else { + this.org = "hmci"; // In InfluxDB 1.x, there is no concept of organization. + } + if(config.token != null) { + this.token = config.token; + } else { + this.token = config.username + ":" + config.password; + } + if(config.bucket != null) { + this.bucket = config.bucket; + } else { + this.bucket = config.database; + } } synchronized void login() throws RuntimeException, InterruptedException { - if(influxDB != null) { + if(influxDBClient != null) { return; } boolean connected = false; int loginErrors = 0; + do { try { log.debug("Connecting to InfluxDB - {}", url); - influxDB = InfluxDBFactory.connect(url, username, password).setDatabase(database); - influxDB.version(); // This ensures that we actually try to connect to the db + influxDBClient = InfluxDBClientFactory.create(url, token.toCharArray(), org, bucket); + influxDBClient.version(); // This ensures that we actually try to connect to the db + Runtime.getRuntime().addShutdownHook(new Thread(influxDBClient::close)); - influxDB.enableBatch( - BatchOptions.DEFAULTS - .flushDuration(5000) - .threadFactory(runnable -> { - Thread thread = new Thread(runnable); - thread.setDaemon(true); - return thread; - }) - ); - Runtime.getRuntime().addShutdownHook(new Thread(influxDB::close)); + // Todo: Handle events - https://github.com/influxdata/influxdb-client-java/tree/master/client#handle-the-events + //writeApi = influxDBClient.makeWriteApi(); + writeApi = influxDBClient.makeWriteApi( + WriteOptions.builder() + .bufferLimit(20_000) + .flushInterval(5_000) + .build()); connected = true; + } catch(Exception e) { sleep(15 * 1000); if(loginErrors++ > 3) { @@ -90,10 +107,10 @@ public final class InfluxClient { synchronized void logoff() { - if(influxDB != null) { - influxDB.close(); + if(influxDBClient != null) { + influxDBClient.close(); } - influxDB = null; + influxDBClient = null; } @@ -101,7 +118,7 @@ public final class InfluxClient { log.debug("write() - measurement: {} {}", name, measurements.size()); if(!measurements.isEmpty()) { processMeasurementMap(measurements, name).forEach((point) -> { - influxDB.write(point); + writeApi.writePoint(point); }); } } @@ -111,11 +128,11 @@ public final class InfluxClient { List listOfPoints = new ArrayList<>(); measurements.forEach( (m) -> { log.trace("processMeasurementMap() - timestamp: {}, tags: {}, fields: {}", m.timestamp, m.tags, m.fields); - Point.Builder builder = Point.measurement(name) - .time(m.timestamp.getEpochSecond(), TimeUnit.SECONDS) - .tag(m.tags) - .fields(m.fields); - listOfPoints.add(builder.build()); + Point point = new Point(name) + .time(m.timestamp.getEpochSecond(), WritePrecision.S) + .addTags(m.tags) + .addFields(m.fields); + listOfPoints.add(point); }); return listOfPoints; } diff --git a/src/main/java/biz/nellemann/hmci/dto/toml/InfluxConfiguration.java b/src/main/java/biz/nellemann/hmci/dto/toml/InfluxConfiguration.java index a07b85e..e315cfc 100644 --- a/src/main/java/biz/nellemann/hmci/dto/toml/InfluxConfiguration.java +++ b/src/main/java/biz/nellemann/hmci/dto/toml/InfluxConfiguration.java @@ -3,6 +3,10 @@ package biz.nellemann.hmci.dto.toml; public class InfluxConfiguration { public String url; + public String org; + public String token; + public String bucket; + public String username; public String password; public String database;