Merge pull request 'Development changes.' (#1) from dev into main

Reviewed-on: #1

README.md

@@ -0,0 +1,14 @@
+# Memory Performance Test
+
+
+## Examples
+
+Test with 8GB memory:
+```shell
+java -Xms10g -Xmx10g -XX:+AlwaysPreTouch -jar memstress-0.0.1-all.jar -t 8
+```
+
+Test with 100GB memory:
+```shell
+java -Xms128g -Xmx128g -XX:+AlwaysPreTouch -jar memstress-0.0.1-all.jar -t 100
+```

build.gradle

@@ -2,7 +2,6 @@ plugins {
     id 'groovy'
     id 'application'
     id "net.nemerosa.versioning" version "2.15.1"
-    id "com.netflix.nebula.ospackage" version "11.2.0"
     id 'com.github.johnrengelman.shadow' version '8.1.1'
 }
 
@@ -30,7 +29,7 @@ java {
 }
 
 application {
-    mainClass = 'biz.nellemann.memstress.Application'
+    mainClass = 'biz.nellemann.jmemperf.Application'
 }
 
 tasks.named('test') {
@@ -55,41 +54,3 @@ jar {
         )
     }
 }
-
-apply plugin: 'com.netflix.nebula.ospackage'
-ospackage {
-    packageName = 'memstress'
-    release = '1'
-    user = 'root'
-    packager = "Mark Nellemann <mark.nellemann@gmail.com>"
-
-    into '/opt/memstress'
-
-    from(shadowJar.outputs.files) {
-        into 'lib'
-    }
-
-    from('build/scriptsShadow') {
-        into 'bin'
-    }
-
-    from(['README.md', 'LICENSE']) {
-        into 'doc'
-    }
-
-}
-
-buildDeb {
-    dependsOn build, startShadowScripts
-}
-
-buildRpm {
-    dependsOn build, startShadowScripts
-    os = org.redline_rpm.header.Os.LINUX
-}
-
-tasks.register("packages") {
-    group "build"
-    dependsOn ":buildDeb"
-    dependsOn ":buildRpm"
-}

gradle.properties

@@ -1,3 +1,3 @@
-projectId               = memstress
-projectGroup            = biz.nellemann.memstress
+projectId               = jmemperf
+projectGroup            = biz.nellemann.jmemperf
 projectVersion          = 0.0.1

settings.gradle

@@ -12,4 +12,4 @@ plugins {
     id 'org.gradle.toolchains.foojay-resolver-convention' version '0.4.0'
 }
 
-rootProject.name = 'memstress'
+rootProject.name = 'jmemperf'

src/main/java/biz/nellemann/jmemperf/Application.java

@@ -1,13 +1,13 @@
-package biz.nellemann.memstress;
+package biz.nellemann.jmemperf;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import picocli.CommandLine;
 
-import java.util.Scanner;
+import java.time.Duration;
 import java.util.concurrent.Callable;
 
-@CommandLine.Command(name = "memstress", mixinStandardHelpOptions = true, versionProvider = VersionProvider.class, description = "Memory performance measurement tool.")
+@CommandLine.Command(name = "jmemperf", mixinStandardHelpOptions = true, versionProvider = VersionProvider.class, description = "Memory performance measurement tool.")
 public class Application implements Callable<Integer> {
 
     final Logger log = LoggerFactory.getLogger(Application.class);
@@ -22,21 +22,25 @@ public class Application implements Callable<Integer> {
     @CommandLine.Option(names = { "-d", "--data" }, paramLabel = "NUM", description = "Create this much data (MB) pr. row [default: ${DEFAULT-VALUE}]")
     int maxDataPerRow = 100;
 
+    @CommandLine.Option(names = { "-i", "--iterations" }, paramLabel = "NUM", description = "Iterate test his many times [default: ${DEFAULT-VALUE}]")
+    int iterations = 3;
 
     @Override
     public Integer call() throws Exception {
 
-        MyDatabase database = new MyDatabase(maxTables, maxRowsPerTable, maxDataPerRow);
-        database.build("testDb");
+        long writeTimeMillis = 0;
+        long readTimeMillis = 0;
 
-        System.out.println("TODO: How to search / read from data stored in rows?");
+        for(int i = 1; i <= iterations; i++) {
+            log.info("Starting test {} of {}", i, iterations);
+            MemDatabase database = new MemDatabase(maxTables, maxRowsPerTable, maxDataPerRow);
+            writeTimeMillis += database.write("testDb");
+            readTimeMillis += database.read("testDb");
+            database.destroy("testDb");
+        }
 
-        Scanner scanner = new Scanner(System.in);
-        System.out.println("Press ENTER to stop");
-        String s= scanner.nextLine();
-        scanner.close();
-
-        database.destroy("testDb");
+        log.info("Average writing time: {}", Duration.ofMillis(writeTimeMillis / iterations));
+        log.info("Average reading time: {}", Duration.ofMillis(readTimeMillis / iterations));
 
         return 0;
     }

src/main/java/biz/nellemann/jmemperf/MemDatabase.java

@@ -1,8 +1,8 @@
-package biz.nellemann.memstress;
+package biz.nellemann.jmemperf;
 
-import biz.nellemann.memstress.db.Database;
-import biz.nellemann.memstress.db.DatabaseManager;
-import biz.nellemann.memstress.db.Table;
+import biz.nellemann.jmemperf.db.Database;
+import biz.nellemann.jmemperf.db.DatabaseManager;
+import biz.nellemann.jmemperf.db.Table;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -10,18 +10,18 @@ import java.nio.ByteBuffer;
 import java.time.Duration;
 import java.time.Instant;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.HashMap;
-import java.util.Random;
+import java.util.concurrent.atomic.AtomicLong;
 
-public class MyDatabase {
+public class MemDatabase {
 
-    final Logger log = LoggerFactory.getLogger(MyDatabase.class);
+    final Logger log = LoggerFactory.getLogger(MemDatabase.class);
 
     private final int BYTE_SIZE_1MB = 1_000_000;
     private final int BYTE_SIZE_1GB = 1_000_000_000;
 
     private final DatabaseManager databaseManager = new DatabaseManager();
-    private final Random random = new Random();
 
     // Use when searching or using later?
     private final ArrayList<Table> tables = new ArrayList<Table>();
@@ -35,7 +35,7 @@ public class MyDatabase {
     private char[] baseCar;
     private byte[] byteBase;
 
-    public MyDatabase(int tables, int rows, int size) {
+    public MemDatabase(int tables, int rows, int size) {
         this.maxTables = tables;
         this.maxRowsPerTable = rows;
         this.maxDataPerRow = size;
@@ -44,19 +44,19 @@ public class MyDatabase {
         for (int i = 0; i < 128; i++) {
             baseCar[i] = 'A';
         }
-        for (int i = 0; i < byteBase.length; i++) {
-            byteBase[i] = 0;
-        }
+        Arrays.fill(byteBase, (byte) 0);
     }
 
 
-    public Database build(String dbName) {
+    public long write(String dbName) {
         Instant instant1 = Instant.now();
         Database database = databaseManager.createDatabase(dbName);
+
+        AtomicLong bytesWritten = new AtomicLong();
         for (int t = 1; t <= maxTables; t++) {
 
             String tableName = String.format("table_%d", t);
-            log.info("Creating table \"{}\"", tableName);
+            log.debug("Creating table \"{}\"", tableName);
 
             Table table = database.createTable(tableName);
 
@@ -65,6 +65,7 @@ public class MyDatabase {
                 HashMap<String, ByteBuffer> map = new HashMap<String, ByteBuffer>();
                 for (int m = 1; m <= maxDataPerRow; m++) {
                     map.put(randomString(), randomBytes());
+                    bytesWritten.addAndGet(byteBase.length);
                 }
                 table.insertEntry(rowIdx, map);
             }
@@ -73,22 +74,49 @@ public class MyDatabase {
         }
 
         Instant instant2 = Instant.now();
-        log.info("Done building in-memory database \"{}\" in {}", dbName, Duration.between(instant1, instant2));
-        return database;
+        Duration duration = Duration.between(instant1, instant2);
+        log.info("Done writing {} bytes -> \"{}\" in {}", bytesWritten, dbName, duration);
+
+        return duration.toMillis();
     }
 
+
+    public long read(String dbName) {
+        Instant instant1 = Instant.now();
+        Database database = databaseManager.getDatabase(dbName);
+
+        AtomicLong bytesRead = new AtomicLong();
+        for(Table table : tables) {
+            table.getRows().forEach((idx, row) -> {
+                HashMap<String, ByteBuffer> values = row.getColumnValuesMap();
+                values.forEach((str, byteBuffer) -> {
+                    byteBuffer.rewind();
+                    while (byteBuffer.hasRemaining()) {
+                        byte[] tmp = new byte[BYTE_SIZE_1MB];
+                        byteBuffer.get(tmp);
+                        bytesRead.addAndGet(tmp.length);
+                    }
+                });
+            });
+        }
+        Instant instant2 = Instant.now();
+        Duration duration = Duration.between(instant1, instant2);
+        log.info("Done reading {} bytes <- \"{}\" in {}", bytesRead.get(), dbName, duration);
+
+        return duration.toMillis();
+    }
+
+
+
     String randomString() {
         baseCar[(idx++) % 128]++;
-        String s = new String(baseCar);
-        return s;
+        return new String(baseCar);
     }
 
     ByteBuffer randomBytes() {
         byteBase[(idx2++) % byteBase.length]++;
         byte[] bytes = new byte[byteBase.length];
-        for (int i = 0; i < bytes.length; i++) {
-            bytes[i] = byteBase[i];
-        }
+        System.arraycopy(byteBase, 0, bytes, 0, bytes.length);
         return ByteBuffer.wrap(bytes);
     }
 

src/main/java/biz/nellemann/jmemperf/VersionProvider.java

@@ -1,4 +1,4 @@
-package biz.nellemann.memstress;
+package biz.nellemann.jmemperf;
 
 
 import picocli.CommandLine;

src/main/java/biz/nellemann/jmemperf/db/Database.java

@@ -1,7 +1,7 @@
 /**
  * Example taken from: https://medium.com/@mithoonkumar/design-an-in-memory-nosql-database-ood-428d48b68dfa
  */
-package biz.nellemann.memstress.db;
+package biz.nellemann.jmemperf.db;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;

src/main/java/biz/nellemann/jmemperf/db/DatabaseManager.java

@@ -1,16 +1,21 @@
 /**
  * Example taken from: https://medium.com/@mithoonkumar/design-an-in-memory-nosql-database-ood-428d48b68dfa
  */
-package biz.nellemann.memstress.db;
+package biz.nellemann.jmemperf.db;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.util.HashMap;
 
 public class DatabaseManager {
+    final Logger log = LoggerFactory.getLogger(DatabaseManager.class);
+
     private HashMap<String, Database> databaseHashMap;
 
     public Database createDatabase(String databaseName) {
         if (databaseHashMap.containsKey(databaseName)) {
-            System.out.println("A database already exists with this name");
+            log.warn("createDatabase() - A database already exists with this name: {}", databaseName);
         } else {
             databaseHashMap.put(databaseName, new Database(databaseName));
         }
@@ -21,6 +26,17 @@ public class DatabaseManager {
         databaseHashMap.remove(databaseName);
     }
 
+
+    public Database getDatabase(String databaseName) {
+        if (databaseHashMap.containsKey(databaseName)) {
+            return databaseHashMap.get(databaseName);
+        } else {
+            log.warn("getDatabase() - Database was not found: {}", databaseName);
+        }
+        return  null;
+    }
+
+
     public DatabaseManager() {
         this.databaseHashMap = new HashMap<>();
     }

src/main/java/biz/nellemann/jmemperf/db/Row.java

@@ -1,7 +1,7 @@
 /**
  * Example taken from: https://medium.com/@mithoonkumar/design-an-in-memory-nosql-database-ood-428d48b68dfa
  */
-package biz.nellemann.memstress.db;
+package biz.nellemann.jmemperf.db;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;

src/main/java/biz/nellemann/jmemperf/db/Table.java

@@ -1,7 +1,7 @@
 /**
  * Example taken from: https://medium.com/@mithoonkumar/design-an-in-memory-nosql-database-ood-428d48b68dfa
  */
-package biz.nellemann.memstress.db;
+package biz.nellemann.jmemperf.db;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;

src/main/resources/logback.xml

@@ -3,7 +3,7 @@
     <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
         <withJansi>false</withJansi>
         <encoder>
-            <pattern>%cyan(%d{yyyy-MM-dd HH:mm:ss.SSS}) %gray([%-10thread]) %highlight(%-5level) %magenta(%logger{12}) - %msg%n</pattern>
+            <pattern>%cyan(%d{yyyy-MM-dd HH:mm:ss.SSS}) - %msg%n</pattern>
         </encoder>
     </appender>
 

src/test/groovy/biz/nellemann/jmemperf/ApplicationTest.groovy

@@ -1,7 +1,7 @@
 /*
  * This Spock specification was generated by the Gradle 'init' task.
  */
-package biz.nellemann.memstress
+package biz.nellemann.jmemperf
 
 import spock.lang.Specification