diff --git a/README.md b/README.md index e129db1..c80fab6 100644 --- a/README.md +++ b/README.md @@ -2,16 +2,24 @@ Small utility to fetch metrics from one or more HMC's and push those to an InfluxDB time-series database. +## Known Problems + +- When running on Windows, the data is collected and written to InfluxDB, but in Grafana there is no data. + + ## Usage Instructions ### Create Configuration -Create a configuration file with setup for HMC's and InfluxDB. +Modify the **/opt/hmci/conf/hmci.groovy** configuration file to suit your environment. ### Run HMCi Tool -Requires Java 8+ runtime. +Requires Java 8+ runtime + + /opt/hmci/bin/hmci + ## Development Information diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml index 27d79e6..44acca8 100644 --- a/bitbucket-pipelines.yml +++ b/bitbucket-pipelines.yml @@ -18,5 +18,5 @@ pipelines: script: - ./gradlew clean build shadowJar startShadowScripts buildDep buildRpm - for file in ${BITBUCKET_CLONE_DIR}/build/libs/*-all.jar ; do curl -X POST --user "${BB_AUTH_STRING}" "https://api.bitbucket.org/2.0/repositories/${BITBUCKET_REPO_OWNER}/${BITBUCKET_REPO_SLUG}/downloads" --form files=@"${file}" ; done - - for file in ${BITBUCKET_CLONE_DIR}/build/distributions/*_all.deb ; do curl -X POST --user "${BB_AUTH_STRING}" "https://api.bitbucket.org/2.0/repositories/${BITBUCKET_REPO_OWNER}/${BITBUCKET_REPO_SLUG}/downloads" --form files=@"${file}" ; done - - for file in ${BITBUCKET_CLONE_DIR}/build/distributions/*.noarch.rpm ; do curl -X POST --user "${BB_AUTH_STRING}" "https://api.bitbucket.org/2.0/repositories/${BITBUCKET_REPO_OWNER}/${BITBUCKET_REPO_SLUG}/downloads" --form files=@"${file}" ; done + - for file in ${BITBUCKET_CLONE_DIR}/build/distributions/*.deb ; do curl -X POST --user "${BB_AUTH_STRING}" "https://api.bitbucket.org/2.0/repositories/${BITBUCKET_REPO_OWNER}/${BITBUCKET_REPO_SLUG}/downloads" --form files=@"${file}" ; done + - for file in ${BITBUCKET_CLONE_DIR}/build/distributions/*.rpm ; do curl -X POST --user "${BB_AUTH_STRING}" "https://api.bitbucket.org/2.0/repositories/${BITBUCKET_REPO_OWNER}/${BITBUCKET_REPO_SLUG}/downloads" --form files=@"${file}" ; done diff --git a/build.gradle b/build.gradle index 182d0df..4d7bbb4 100644 --- a/build.gradle +++ b/build.gradle @@ -1,11 +1,3 @@ -/* - * This file was generated by the Gradle 'init' task. - * - * This generated file contains a sample Groovy project to get you started. - * For more details take a look at the Groovy Quickstart chapter in the Gradle - * User Manual available at https://docs.gradle.org/6.5.1/userguide/tutorial_groovy_projects.html - */ - plugins { // Apply the groovy plugin to add support for Groovy id 'groovy' @@ -18,20 +10,16 @@ plugins { } repositories { - // Use jcenter for resolving dependencies. - // You can declare any Maven/Ivy/file repository here. jcenter() } dependencies { - // Use the latest Groovy version for building this library implementation 'org.codehaus.groovy:groovy-all:3.0.5' implementation 'com.squareup.okhttp3:okhttp:4.8.0' implementation 'org.influxdb:influxdb-java:2.19' implementation 'org.slf4j:slf4j-api:1.7.+' runtimeOnly 'ch.qos.logback:logback-classic:1.+' - // Use the awesome Spock testing and specification framework testImplementation('org.spockframework:spock-core:2.0-M3-groovy-3.0') testImplementation("org.slf4j:slf4j-simple:1.7.+") testImplementation('com.squareup.okhttp3:mockwebserver:4.8.0') @@ -40,7 +28,6 @@ dependencies { } application { - // Define the main class for the application. mainClassName = 'biz.nellemann.hmci.App' } @@ -52,6 +39,9 @@ apply plugin: 'nebula.ospackage' ospackage { packageName = 'hmci' release = '1' + os = LINUX + user = 'root' + packager = "Mark Nellemann " into '/opt/hmci' diff --git a/conf/hmci.groovy b/conf/hmci.groovy index 0443b71..61d6c8f 100644 --- a/conf/hmci.groovy +++ b/conf/hmci.groovy @@ -2,7 +2,7 @@ Configuration for HMCi */ -hmci.refresh = 30 +hmci.refresh = 60 hmci.rescan = 15 // InfluxDB to save metrics diff --git a/hmci.service b/hmci.service deleted file mode 100644 index 897ea65..0000000 --- a/hmci.service +++ /dev/null @@ -1,13 +0,0 @@ -[Unit] -Description=HMC Insights Daemon - -[Install] -WantedBy=multi-user.target - -[Service] -User=nobody -Group=nogroup -TimeoutSec=20 -Restart=always -WorkingDirectory=/opt/hmci -ExecStart=/usr/bin/java -jar lib/hmci-all.jar diff --git a/src/main/groovy/biz/nellemann/hmci/App.groovy b/src/main/groovy/biz/nellemann/hmci/App.groovy index 331d772..a4bfcad 100644 --- a/src/main/groovy/biz/nellemann/hmci/App.groovy +++ b/src/main/groovy/biz/nellemann/hmci/App.groovy @@ -1,6 +1,5 @@ package biz.nellemann.hmci - import groovy.cli.picocli.CliBuilder import groovy.cli.picocli.OptionAccessor import groovy.util.logging.Slf4j @@ -8,28 +7,31 @@ import groovy.util.logging.Slf4j @Slf4j class App implements Runnable { - HmcClient hmc - InfluxClient influx - final ConfigObject configuration final Integer refreshEverySec final Integer rescanHmcEvery - Map discoveredHmc = new HashMap<>() + InfluxClient influxClient + Map hmcClients = new HashMap<>() Map systems = new HashMap() Map partitions = new HashMap() App(ConfigObject configuration) { - log.debug configuration.toString() this.configuration = configuration + log.debug configuration.toString() refreshEverySec = (Integer)configuration.get('hmci.refresh') ?: 60 rescanHmcEvery = (Integer)configuration.get('hmci.rescan') ?: 15 + String influxUrl = configuration.get('influx')['url'] + String influxUsername = configuration.get('influx')['username'] + String influxPassword = configuration.get('influx')['password'] + String influxDatabase = configuration.get('influx')['database'] + try { - influx = new InfluxClient((String) configuration.get('influx')['url'], (String) configuration.get('influx')['username'], (String) configuration.get('influx')['password'], (String) configuration.get('influx')['database']) - influx.login() + influxClient = new InfluxClient(influxUrl, influxUsername, influxPassword, influxDatabase) + influxClient.login() } catch(Exception e) { System.exit(1) } @@ -44,15 +46,21 @@ class App implements Runnable { void discover() { configuration.get('hmc').each { Object key, Object hmc -> - if(!discoveredHmc?.containsKey(key)) { + if(!hmcClients?.containsKey(key)) { log.info("Adding HMC: " + hmc.toString()) - HmcClient hmcClient = new HmcClient(key as String, hmc['url'] as String, hmc['username'] as String, hmc['password'] as String, hmc['unsafe'] as Boolean) - discoveredHmc.put(key as String, hmcClient) + String hmcKey = key + String hmcUrl = hmc['url'] + String hmcUsername = hmc['username'] + String hmcPassword = hmc['password'] + Boolean hmcUnsafe = hmc['unsafe'] + HmcClient hmcClient = new HmcClient(hmcKey, hmcUrl, hmcUsername, hmcPassword, hmcUnsafe) + hmcClients.put(hmcKey, hmcClient) } } - discoveredHmc.each {id, hmcClient -> + hmcClients.each { hmcId, hmcClient -> + log.info("Loggin in to HMC " + hmcId) try { hmcClient.login() hmcClient.getManagedSystems().each { systemId, system -> @@ -68,8 +76,8 @@ class App implements Runnable { } } } catch(Exception e) { - log.error("discover() - " + id + " error: " + e.message) - discoveredHmc.remove(id) + log.error("discover() - " + hmcId + " error: " + e.message) + hmcClients.remove(hmcId) } } @@ -83,7 +91,7 @@ class App implements Runnable { systems.each {systemId, system -> - HmcClient hmcClient = discoveredHmc.get(system.hmcId) + HmcClient hmcClient = hmcClients.get(system.hmcId) // Get and process metrics for this system String tmpJsonString = hmcClient.getPcmDataForManagedSystem(system) @@ -108,7 +116,7 @@ class App implements Runnable { // Get LPAR's for this system partitions.each { partitionId, partition -> - HmcClient hmcClient = discoveredHmc.get(partition.system.hmcId) + HmcClient hmcClient = hmcClients.get(partition.system.hmcId) // Get and process metrics for this partition String tmpJsonString2 = hmcClient.getPcmDataForLogicalPartition(partition) @@ -127,14 +135,14 @@ class App implements Runnable { void writeMetricsForManagedSystems() { systems.each {systemId, system -> - influx.writeManagedSystem(system) + influxClient.writeManagedSystem(system) } } void writeMetricsForLogicalPartitions() { partitions.each {partitionId, partition -> - influx.writeLogicalPartition(partition) + influxClient.writeLogicalPartition(partition) } } @@ -158,11 +166,7 @@ class App implements Runnable { System.exit(1) } - // Read in 'config.groovy' for the development environment. configuration = new ConfigSlurper("development").parse(configurationFile.toURI().toURL()); - - // Flatten configuration for easy access keys with dotted notation. - //configuration = conf.flatten(); } new App(configuration) @@ -173,7 +177,7 @@ class App implements Runnable { @Override void run() { - log.info("In RUN ") + log.info("run()") boolean keepRunning = true int executions = 0 diff --git a/src/main/groovy/biz/nellemann/hmci/HmcClient.groovy b/src/main/groovy/biz/nellemann/hmci/HmcClient.groovy index 635b4ce..3408910 100644 --- a/src/main/groovy/biz/nellemann/hmci/HmcClient.groovy +++ b/src/main/groovy/biz/nellemann/hmci/HmcClient.groovy @@ -7,7 +7,6 @@ import okhttp3.OkHttpClient import okhttp3.Request import okhttp3.RequestBody import okhttp3.Response -import org.influxdb.InfluxDBFactory import javax.net.ssl.HostnameVerifier import javax.net.ssl.SSLContext @@ -30,11 +29,9 @@ class HmcClient { private final String password private final Boolean unsafe - //protected Map managedSystems = new HashMap() protected String authToken private final OkHttpClient client - HmcClient(String hmcId, String baseUrl, String username, String password, Boolean unsafe = false) { this.hmcId = hmcId this.baseUrl = baseUrl diff --git a/src/main/groovy/biz/nellemann/hmci/InfluxClient.groovy b/src/main/groovy/biz/nellemann/hmci/InfluxClient.groovy index e555298..b47d565 100644 --- a/src/main/groovy/biz/nellemann/hmci/InfluxClient.groovy +++ b/src/main/groovy/biz/nellemann/hmci/InfluxClient.groovy @@ -1,18 +1,15 @@ package biz.nellemann.hmci - import groovy.util.logging.Slf4j +import org.influxdb.BatchOptions +import org.influxdb.InfluxDB +import org.influxdb.InfluxDBFactory import org.influxdb.dto.BatchPoints +import org.influxdb.dto.Point +import org.influxdb.dto.Query import java.time.Instant import java.util.concurrent.TimeUnit -import org.influxdb.InfluxDB -import org.influxdb.BatchOptions -import org.influxdb.InfluxDBFactory -import org.influxdb.dto.QueryResult -import org.influxdb.dto.Query -import org.influxdb.dto.Point - @Slf4j class InfluxClient { @@ -53,13 +50,13 @@ class InfluxClient { influxDB.query(new Query("CREATE DATABASE " + database)); influxDB.setDatabase(database); +/* // ... and a retention policy, if necessary. - /* String retentionPolicyName = "HMCI_ONE_YEAR"; influxDB.query(new Query("CREATE RETENTION POLICY " + retentionPolicyName + " ON " + database + " DURATION 365d REPLICATION 1 DEFAULT")); - influxDB.setRetentionPolicy(retentionPolicyName);*/ - + influxDB.setRetentionPolicy(retentionPolicyName); + */ // Enable batch writes to get better performance. influxDB.enableBatch(BatchOptions.DEFAULTS); }