diff --git a/README.md b/README.md index e69de29..adf38d6 100644 --- a/README.md +++ b/README.md @@ -0,0 +1,14 @@ +# Memory Performance Test + + +## Examples + +Test with 8GB memory: +```shell +java -Xms10g -Xmx10g -XX:+AlwaysPreTouch -jar memstress-0.0.1-all.jar -t 8 +``` + +Test with 100GB memory: +```shell +java -Xms128g -Xmx128g -XX:+AlwaysPreTouch -jar memstress-0.0.1-all.jar -t 100 +``` diff --git a/build.gradle b/build.gradle index 192ec8a..5ee067b 100644 --- a/build.gradle +++ b/build.gradle @@ -2,7 +2,6 @@ plugins { id 'groovy' id 'application' id "net.nemerosa.versioning" version "2.15.1" - id "com.netflix.nebula.ospackage" version "11.2.0" id 'com.github.johnrengelman.shadow' version '8.1.1' } @@ -30,7 +29,7 @@ java { } application { - mainClass = 'biz.nellemann.memstress.Application' + mainClass = 'biz.nellemann.jmemperf.Application' } tasks.named('test') { @@ -55,41 +54,3 @@ jar { ) } } - -apply plugin: 'com.netflix.nebula.ospackage' -ospackage { - packageName = 'memstress' - release = '1' - user = 'root' - packager = "Mark Nellemann " - - into '/opt/memstress' - - from(shadowJar.outputs.files) { - into 'lib' - } - - from('build/scriptsShadow') { - into 'bin' - } - - from(['README.md', 'LICENSE']) { - into 'doc' - } - -} - -buildDeb { - dependsOn build, startShadowScripts -} - -buildRpm { - dependsOn build, startShadowScripts - os = org.redline_rpm.header.Os.LINUX -} - -tasks.register("packages") { - group "build" - dependsOn ":buildDeb" - dependsOn ":buildRpm" -} diff --git a/gradle.properties b/gradle.properties index 2eae34a..6bbf328 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,3 +1,3 @@ -projectId = memstress -projectGroup = biz.nellemann.memstress +projectId = jmemperf +projectGroup = biz.nellemann.jmemperf projectVersion = 0.0.1 diff --git a/settings.gradle b/settings.gradle index 8592cea..56a9f21 100644 --- a/settings.gradle +++ b/settings.gradle @@ -12,4 +12,4 @@ plugins { id 'org.gradle.toolchains.foojay-resolver-convention' version '0.4.0' } -rootProject.name = 'memstress' +rootProject.name = 'jmemperf' diff --git a/src/main/java/biz/nellemann/memstress/Application.java b/src/main/java/biz/nellemann/jmemperf/Application.java similarity index 50% rename from src/main/java/biz/nellemann/memstress/Application.java rename to src/main/java/biz/nellemann/jmemperf/Application.java index 64c025c..86ecca7 100644 --- a/src/main/java/biz/nellemann/memstress/Application.java +++ b/src/main/java/biz/nellemann/jmemperf/Application.java @@ -1,13 +1,13 @@ -package biz.nellemann.memstress; +package biz.nellemann.jmemperf; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import picocli.CommandLine; -import java.util.Scanner; +import java.time.Duration; import java.util.concurrent.Callable; -@CommandLine.Command(name = "memstress", mixinStandardHelpOptions = true, versionProvider = VersionProvider.class, description = "Memory performance measurement tool.") +@CommandLine.Command(name = "jmemperf", mixinStandardHelpOptions = true, versionProvider = VersionProvider.class, description = "Memory performance measurement tool.") public class Application implements Callable { final Logger log = LoggerFactory.getLogger(Application.class); @@ -22,21 +22,25 @@ public class Application implements Callable { @CommandLine.Option(names = { "-d", "--data" }, paramLabel = "NUM", description = "Create this much data (MB) pr. row [default: ${DEFAULT-VALUE}]") int maxDataPerRow = 100; + @CommandLine.Option(names = { "-i", "--iterations" }, paramLabel = "NUM", description = "Iterate test his many times [default: ${DEFAULT-VALUE}]") + int iterations = 3; @Override public Integer call() throws Exception { - MyDatabase database = new MyDatabase(maxTables, maxRowsPerTable, maxDataPerRow); - database.build("testDb"); + long writeTimeMillis = 0; + long readTimeMillis = 0; - System.out.println("TODO: How to search / read from data stored in rows?"); + for(int i = 1; i <= iterations; i++) { + log.info("Starting test {} of {}", i, iterations); + MemDatabase database = new MemDatabase(maxTables, maxRowsPerTable, maxDataPerRow); + writeTimeMillis += database.write("testDb"); + readTimeMillis += database.read("testDb"); + database.destroy("testDb"); + } - Scanner scanner = new Scanner(System.in); - System.out.println("Press ENTER to stop"); - String s= scanner.nextLine(); - scanner.close(); - - database.destroy("testDb"); + log.info("Average writing time: {}", Duration.ofMillis(writeTimeMillis / iterations)); + log.info("Average reading time: {}", Duration.ofMillis(readTimeMillis / iterations)); return 0; } diff --git a/src/main/java/biz/nellemann/memstress/MyDatabase.java b/src/main/java/biz/nellemann/jmemperf/MemDatabase.java similarity index 52% rename from src/main/java/biz/nellemann/memstress/MyDatabase.java rename to src/main/java/biz/nellemann/jmemperf/MemDatabase.java index 174dcff..5dffec3 100644 --- a/src/main/java/biz/nellemann/memstress/MyDatabase.java +++ b/src/main/java/biz/nellemann/jmemperf/MemDatabase.java @@ -1,8 +1,8 @@ -package biz.nellemann.memstress; +package biz.nellemann.jmemperf; -import biz.nellemann.memstress.db.Database; -import biz.nellemann.memstress.db.DatabaseManager; -import biz.nellemann.memstress.db.Table; +import biz.nellemann.jmemperf.db.Database; +import biz.nellemann.jmemperf.db.DatabaseManager; +import biz.nellemann.jmemperf.db.Table; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -10,18 +10,18 @@ import java.nio.ByteBuffer; import java.time.Duration; import java.time.Instant; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; -import java.util.Random; +import java.util.concurrent.atomic.AtomicLong; -public class MyDatabase { +public class MemDatabase { - final Logger log = LoggerFactory.getLogger(MyDatabase.class); + final Logger log = LoggerFactory.getLogger(MemDatabase.class); private final int BYTE_SIZE_1MB = 1_000_000; private final int BYTE_SIZE_1GB = 1_000_000_000; private final DatabaseManager databaseManager = new DatabaseManager(); - private final Random random = new Random(); // Use when searching or using later? private final ArrayList tables = new ArrayList
(); @@ -35,7 +35,7 @@ public class MyDatabase { private char[] baseCar; private byte[] byteBase; - public MyDatabase(int tables, int rows, int size) { + public MemDatabase(int tables, int rows, int size) { this.maxTables = tables; this.maxRowsPerTable = rows; this.maxDataPerRow = size; @@ -44,19 +44,19 @@ public class MyDatabase { for (int i = 0; i < 128; i++) { baseCar[i] = 'A'; } - for (int i = 0; i < byteBase.length; i++) { - byteBase[i] = 0; - } + Arrays.fill(byteBase, (byte) 0); } - public Database build(String dbName) { + public long write(String dbName) { Instant instant1 = Instant.now(); Database database = databaseManager.createDatabase(dbName); + + AtomicLong bytesWritten = new AtomicLong(); for (int t = 1; t <= maxTables; t++) { String tableName = String.format("table_%d", t); - log.info("Creating table \"{}\"", tableName); + log.debug("Creating table \"{}\"", tableName); Table table = database.createTable(tableName); @@ -65,6 +65,7 @@ public class MyDatabase { HashMap map = new HashMap(); for (int m = 1; m <= maxDataPerRow; m++) { map.put(randomString(), randomBytes()); + bytesWritten.addAndGet(byteBase.length); } table.insertEntry(rowIdx, map); } @@ -73,22 +74,49 @@ public class MyDatabase { } Instant instant2 = Instant.now(); - log.info("Done building in-memory database \"{}\" in {}", dbName, Duration.between(instant1, instant2)); - return database; + Duration duration = Duration.between(instant1, instant2); + log.info("Done writing {} bytes -> \"{}\" in {}", bytesWritten, dbName, duration); + + return duration.toMillis(); } + + public long read(String dbName) { + Instant instant1 = Instant.now(); + Database database = databaseManager.getDatabase(dbName); + + AtomicLong bytesRead = new AtomicLong(); + for(Table table : tables) { + table.getRows().forEach((idx, row) -> { + HashMap values = row.getColumnValuesMap(); + values.forEach((str, byteBuffer) -> { + byteBuffer.rewind(); + while (byteBuffer.hasRemaining()) { + byte[] tmp = new byte[BYTE_SIZE_1MB]; + byteBuffer.get(tmp); + bytesRead.addAndGet(tmp.length); + } + }); + }); + } + Instant instant2 = Instant.now(); + Duration duration = Duration.between(instant1, instant2); + log.info("Done reading {} bytes <- \"{}\" in {}", bytesRead.get(), dbName, duration); + + return duration.toMillis(); + } + + + String randomString() { baseCar[(idx++) % 128]++; - String s = new String(baseCar); - return s; + return new String(baseCar); } ByteBuffer randomBytes() { byteBase[(idx2++) % byteBase.length]++; byte[] bytes = new byte[byteBase.length]; - for (int i = 0; i < bytes.length; i++) { - bytes[i] = byteBase[i]; - } + System.arraycopy(byteBase, 0, bytes, 0, bytes.length); return ByteBuffer.wrap(bytes); } diff --git a/src/main/java/biz/nellemann/memstress/VersionProvider.java b/src/main/java/biz/nellemann/jmemperf/VersionProvider.java similarity index 93% rename from src/main/java/biz/nellemann/memstress/VersionProvider.java rename to src/main/java/biz/nellemann/jmemperf/VersionProvider.java index fc3265a..baf10e0 100644 --- a/src/main/java/biz/nellemann/memstress/VersionProvider.java +++ b/src/main/java/biz/nellemann/jmemperf/VersionProvider.java @@ -1,4 +1,4 @@ -package biz.nellemann.memstress; +package biz.nellemann.jmemperf; import picocli.CommandLine; diff --git a/src/main/java/biz/nellemann/memstress/db/Database.java b/src/main/java/biz/nellemann/jmemperf/db/Database.java similarity index 97% rename from src/main/java/biz/nellemann/memstress/db/Database.java rename to src/main/java/biz/nellemann/jmemperf/db/Database.java index 7d63efb..b199680 100644 --- a/src/main/java/biz/nellemann/memstress/db/Database.java +++ b/src/main/java/biz/nellemann/jmemperf/db/Database.java @@ -1,7 +1,7 @@ /** * Example taken from: https://medium.com/@mithoonkumar/design-an-in-memory-nosql-database-ood-428d48b68dfa */ -package biz.nellemann.memstress.db; +package biz.nellemann.jmemperf.db; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/main/java/biz/nellemann/memstress/db/DatabaseManager.java b/src/main/java/biz/nellemann/jmemperf/db/DatabaseManager.java similarity index 61% rename from src/main/java/biz/nellemann/memstress/db/DatabaseManager.java rename to src/main/java/biz/nellemann/jmemperf/db/DatabaseManager.java index dd3a420..333a8b0 100644 --- a/src/main/java/biz/nellemann/memstress/db/DatabaseManager.java +++ b/src/main/java/biz/nellemann/jmemperf/db/DatabaseManager.java @@ -1,16 +1,21 @@ /** * Example taken from: https://medium.com/@mithoonkumar/design-an-in-memory-nosql-database-ood-428d48b68dfa */ -package biz.nellemann.memstress.db; +package biz.nellemann.jmemperf.db; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.HashMap; public class DatabaseManager { + final Logger log = LoggerFactory.getLogger(DatabaseManager.class); + private HashMap databaseHashMap; public Database createDatabase(String databaseName) { if (databaseHashMap.containsKey(databaseName)) { - System.out.println("A database already exists with this name"); + log.warn("createDatabase() - A database already exists with this name: {}", databaseName); } else { databaseHashMap.put(databaseName, new Database(databaseName)); } @@ -21,6 +26,17 @@ public class DatabaseManager { databaseHashMap.remove(databaseName); } + + public Database getDatabase(String databaseName) { + if (databaseHashMap.containsKey(databaseName)) { + return databaseHashMap.get(databaseName); + } else { + log.warn("getDatabase() - Database was not found: {}", databaseName); + } + return null; + } + + public DatabaseManager() { this.databaseHashMap = new HashMap<>(); } diff --git a/src/main/java/biz/nellemann/memstress/db/Row.java b/src/main/java/biz/nellemann/jmemperf/db/Row.java similarity index 97% rename from src/main/java/biz/nellemann/memstress/db/Row.java rename to src/main/java/biz/nellemann/jmemperf/db/Row.java index 07fe2e6..aae91ea 100644 --- a/src/main/java/biz/nellemann/memstress/db/Row.java +++ b/src/main/java/biz/nellemann/jmemperf/db/Row.java @@ -1,7 +1,7 @@ /** * Example taken from: https://medium.com/@mithoonkumar/design-an-in-memory-nosql-database-ood-428d48b68dfa */ -package biz.nellemann.memstress.db; +package biz.nellemann.jmemperf.db; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/main/java/biz/nellemann/memstress/db/Table.java b/src/main/java/biz/nellemann/jmemperf/db/Table.java similarity index 98% rename from src/main/java/biz/nellemann/memstress/db/Table.java rename to src/main/java/biz/nellemann/jmemperf/db/Table.java index a464cdb..bc247b2 100644 --- a/src/main/java/biz/nellemann/memstress/db/Table.java +++ b/src/main/java/biz/nellemann/jmemperf/db/Table.java @@ -1,7 +1,7 @@ /** * Example taken from: https://medium.com/@mithoonkumar/design-an-in-memory-nosql-database-ood-428d48b68dfa */ -package biz.nellemann.memstress.db; +package biz.nellemann.jmemperf.db; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/main/resources/logback.xml b/src/main/resources/logback.xml index 1c6f3db..0640d69 100644 --- a/src/main/resources/logback.xml +++ b/src/main/resources/logback.xml @@ -3,7 +3,7 @@ false - %cyan(%d{yyyy-MM-dd HH:mm:ss.SSS}) %gray([%-10thread]) %highlight(%-5level) %magenta(%logger{12}) - %msg%n + %cyan(%d{yyyy-MM-dd HH:mm:ss.SSS}) - %msg%n diff --git a/src/test/groovy/biz/nellemann/memstress/ApplicationTest.groovy b/src/test/groovy/biz/nellemann/jmemperf/ApplicationTest.groovy similarity index 83% rename from src/test/groovy/biz/nellemann/memstress/ApplicationTest.groovy rename to src/test/groovy/biz/nellemann/jmemperf/ApplicationTest.groovy index 91c31b1..52550bf 100644 --- a/src/test/groovy/biz/nellemann/memstress/ApplicationTest.groovy +++ b/src/test/groovy/biz/nellemann/jmemperf/ApplicationTest.groovy @@ -1,7 +1,7 @@ /* * This Spock specification was generated by the Gradle 'init' task. */ -package biz.nellemann.memstress +package biz.nellemann.jmemperf import spock.lang.Specification