Merge remote-tracking branch 'origin/dev' into dev

README.md

@@ -1,9 +1,14 @@
 # Memory Performance Test
 
+
 ## Examples
 
+Test with 8GB memory:
 ```shell
-java -Xms128g -Xmx128g  -XX:+UseLargePages -XX:+AlwaysPreTouch \
+java -Xms10g -Xmx10g -XX:+AlwaysPreTouch -jar memstress-0.0.1-all.jar -t 8
-  -XX:-UseParallelGC -XX:MaxGCPauseMillis=500 -Xgcthreads3  \
+```
-  -jar memstress-0.0.1-all.jar -t 128
+
+Test with 100GB memory:
+```shell
+java -Xms128g -Xmx128g -XX:+AlwaysPreTouch -jar memstress-0.0.1-all.jar -t 100
 ```

build.gradle

@@ -2,7 +2,6 @@ plugins {
     id 'groovy'
     id 'application'
     id "net.nemerosa.versioning" version "2.15.1"
-    id "com.netflix.nebula.ospackage" version "11.2.0"
     id 'com.github.johnrengelman.shadow' version '8.1.1'
 }
 
@@ -55,41 +54,3 @@ jar {
         )
     }
 }
-
-apply plugin: 'com.netflix.nebula.ospackage'
-ospackage {
-    packageName = 'memstress'
-    release = '1'
-    user = 'root'
-    packager = "Mark Nellemann <mark.nellemann@gmail.com>"
-
-    into '/opt/memstress'
-
-    from(shadowJar.outputs.files) {
-        into 'lib'
-    }
-
-    from('build/scriptsShadow') {
-        into 'bin'
-    }
-
-    from(['README.md', 'LICENSE']) {
-        into 'doc'
-    }
-
-}
-
-buildDeb {
-    dependsOn build, startShadowScripts
-}
-
-buildRpm {
-    dependsOn build, startShadowScripts
-    os = org.redline_rpm.header.Os.LINUX
-}
-
-tasks.register("packages") {
-    group "build"
-    dependsOn ":buildDeb"
-    dependsOn ":buildRpm"
-}

src/main/java/biz/nellemann/memstress/Application.java

@@ -4,7 +4,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import picocli.CommandLine;
 
-import java.util.Scanner;
+import java.time.Duration;
 import java.util.concurrent.Callable;
 
 @CommandLine.Command(name = "memstress", mixinStandardHelpOptions = true, versionProvider = VersionProvider.class, description = "Memory performance measurement tool.")
@@ -22,21 +22,25 @@ public class Application implements Callable<Integer> {
     @CommandLine.Option(names = { "-d", "--data" }, paramLabel = "NUM", description = "Create this much data (MB) pr. row [default: ${DEFAULT-VALUE}]")
     int maxDataPerRow = 100;
 
+    @CommandLine.Option(names = { "-i", "--iterations" }, paramLabel = "NUM", description = "Iterate test his many times [default: ${DEFAULT-VALUE}]")
+    int iterations = 3;
 
     @Override
     public Integer call() throws Exception {
 
-        MyDatabase database = new MyDatabase(maxTables, maxRowsPerTable, maxDataPerRow);
+        long writeTimeMillis = 0;
-        database.build("testDb");
+        long readTimeMillis = 0;
 
-        System.out.println("TODO: How to search / read from data stored in rows?");
+        for(int i = 1; i <= iterations; i++) {
+            log.info("Starting test {} of {}", i, iterations);
+            MemDatabase database = new MemDatabase(maxTables, maxRowsPerTable, maxDataPerRow);
+            writeTimeMillis += database.write("testDb");
+            readTimeMillis += database.read("testDb");
+            database.destroy("testDb");
+        }
 
-        Scanner scanner = new Scanner(System.in);
+        log.info("Average writing time: {}", Duration.ofMillis(writeTimeMillis / iterations));
-        System.out.println("Press ENTER to stop");
+        log.info("Average reading time: {}", Duration.ofMillis(readTimeMillis / iterations));
-        String s= scanner.nextLine();
-        scanner.close();
-
-        database.destroy("testDb");
 
         return 0;
     }

src/main/java/biz/nellemann/memstress/MemDatabase.java

@@ -12,17 +12,16 @@ import java.time.Instant;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
-import java.util.Random;
+import java.util.concurrent.atomic.AtomicLong;
 
-public class MyDatabase {
+public class MemDatabase {
 
-    final Logger log = LoggerFactory.getLogger(MyDatabase.class);
+    final Logger log = LoggerFactory.getLogger(MemDatabase.class);
 
     private final int BYTE_SIZE_1MB = 1_000_000;
     private final int BYTE_SIZE_1GB = 1_000_000_000;
 
     private final DatabaseManager databaseManager = new DatabaseManager();
-    private final Random random = new Random();
 
     // Use when searching or using later?
     private final ArrayList<Table> tables = new ArrayList<Table>();
@@ -36,7 +35,7 @@ public class MyDatabase {
     private char[] baseCar;
     private byte[] byteBase;
 
-    public MyDatabase(int tables, int rows, int size) {
+    public MemDatabase(int tables, int rows, int size) {
         this.maxTables = tables;
         this.maxRowsPerTable = rows;
         this.maxDataPerRow = size;
@@ -49,13 +48,15 @@ public class MyDatabase {
     }
 
 
-    public Database build(String dbName) {
+    public long write(String dbName) {
         Instant instant1 = Instant.now();
         Database database = databaseManager.createDatabase(dbName);
+
+        AtomicLong bytesWritten = new AtomicLong();
         for (int t = 1; t <= maxTables; t++) {
 
             String tableName = String.format("table_%d", t);
-            log.info("Creating table \"{}\"", tableName);
+            log.debug("Creating table \"{}\"", tableName);
 
             Table table = database.createTable(tableName);
 
@@ -64,6 +65,7 @@ public class MyDatabase {
                 HashMap<String, ByteBuffer> map = new HashMap<String, ByteBuffer>();
                 for (int m = 1; m <= maxDataPerRow; m++) {
                     map.put(randomString(), randomBytes());
+                    bytesWritten.addAndGet(byteBase.length);
                 }
                 table.insertEntry(rowIdx, map);
             }
@@ -72,10 +74,40 @@ public class MyDatabase {
         }
 
         Instant instant2 = Instant.now();
-        log.info("Done building in-memory database \"{}\" in {}", dbName, Duration.between(instant1, instant2));
+        Duration duration = Duration.between(instant1, instant2);
-        return database;
+        log.info("Done writing {} bytes -> \"{}\" in {}", bytesWritten, dbName, duration);
+
+        return duration.toMillis();
     }
 
+
+    public long read(String dbName) {
+        Instant instant1 = Instant.now();
+        Database database = databaseManager.getDatabase(dbName);
+
+        AtomicLong bytesRead = new AtomicLong();
+        for(Table table : tables) {
+            table.getRows().forEach((idx, row) -> {
+                HashMap<String, ByteBuffer> values = row.getColumnValuesMap();
+                values.forEach((str, byteBuffer) -> {
+                    byteBuffer.rewind();
+                    while (byteBuffer.hasRemaining()) {
+                        byte[] tmp = new byte[BYTE_SIZE_1MB];
+                        byteBuffer.get(tmp);
+                        bytesRead.addAndGet(tmp.length);
+                    }
+                });
+            });
+        }
+        Instant instant2 = Instant.now();
+        Duration duration = Duration.between(instant1, instant2);
+        log.info("Done reading {} bytes <- \"{}\" in {}", bytesRead.get(), dbName, duration);
+
+        return duration.toMillis();
+    }
+
+
+
     String randomString() {
         baseCar[(idx++) % 128]++;
         return new String(baseCar);

src/main/java/biz/nellemann/memstress/db/DatabaseManager.java

@@ -3,14 +3,19 @@
  */
 package biz.nellemann.memstress.db;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import java.util.HashMap;
 
 public class DatabaseManager {
+    final Logger log = LoggerFactory.getLogger(DatabaseManager.class);
+
     private HashMap<String, Database> databaseHashMap;
 
     public Database createDatabase(String databaseName) {
         if (databaseHashMap.containsKey(databaseName)) {
-            System.out.println("A database already exists with this name");
+            log.warn("createDatabase() - A database already exists with this name: {}", databaseName);
         } else {
             databaseHashMap.put(databaseName, new Database(databaseName));
         }
@@ -21,6 +26,17 @@ public class DatabaseManager {
         databaseHashMap.remove(databaseName);
     }
 
+
+    public Database getDatabase(String databaseName) {
+        if (databaseHashMap.containsKey(databaseName)) {
+            return databaseHashMap.get(databaseName);
+        } else {
+            log.warn("getDatabase() - Database was not found: {}", databaseName);
+        }
+        return  null;
+    }
+
+
     public DatabaseManager() {
         this.databaseHashMap = new HashMap<>();
     }