Merge pull request 'Development changes.' (#1) from dev into main

Reviewed-on: #1
This commit is contained in:
Mark Nellemann 2023-06-29 18:42:01 +00:00
commit 4a2936b221
13 changed files with 107 additions and 84 deletions

View file

@ -0,0 +1,14 @@
# Memory Performance Test
## Examples
Test with 8GB memory:
```shell
java -Xms10g -Xmx10g -XX:+AlwaysPreTouch -jar memstress-0.0.1-all.jar -t 8
```
Test with 100GB memory:
```shell
java -Xms128g -Xmx128g -XX:+AlwaysPreTouch -jar memstress-0.0.1-all.jar -t 100
```

View file

@ -2,7 +2,6 @@ plugins {
id 'groovy' id 'groovy'
id 'application' id 'application'
id "net.nemerosa.versioning" version "2.15.1" id "net.nemerosa.versioning" version "2.15.1"
id "com.netflix.nebula.ospackage" version "11.2.0"
id 'com.github.johnrengelman.shadow' version '8.1.1' id 'com.github.johnrengelman.shadow' version '8.1.1'
} }
@ -30,7 +29,7 @@ java {
} }
application { application {
mainClass = 'biz.nellemann.memstress.Application' mainClass = 'biz.nellemann.jmemperf.Application'
} }
tasks.named('test') { tasks.named('test') {
@ -55,41 +54,3 @@ jar {
) )
} }
} }
apply plugin: 'com.netflix.nebula.ospackage'
ospackage {
packageName = 'memstress'
release = '1'
user = 'root'
packager = "Mark Nellemann <mark.nellemann@gmail.com>"
into '/opt/memstress'
from(shadowJar.outputs.files) {
into 'lib'
}
from('build/scriptsShadow') {
into 'bin'
}
from(['README.md', 'LICENSE']) {
into 'doc'
}
}
buildDeb {
dependsOn build, startShadowScripts
}
buildRpm {
dependsOn build, startShadowScripts
os = org.redline_rpm.header.Os.LINUX
}
tasks.register("packages") {
group "build"
dependsOn ":buildDeb"
dependsOn ":buildRpm"
}

View file

@ -1,3 +1,3 @@
projectId = memstress projectId = jmemperf
projectGroup = biz.nellemann.memstress projectGroup = biz.nellemann.jmemperf
projectVersion = 0.0.1 projectVersion = 0.0.1

View file

@ -12,4 +12,4 @@ plugins {
id 'org.gradle.toolchains.foojay-resolver-convention' version '0.4.0' id 'org.gradle.toolchains.foojay-resolver-convention' version '0.4.0'
} }
rootProject.name = 'memstress' rootProject.name = 'jmemperf'

View file

@ -1,13 +1,13 @@
package biz.nellemann.memstress; package biz.nellemann.jmemperf;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import picocli.CommandLine; import picocli.CommandLine;
import java.util.Scanner; import java.time.Duration;
import java.util.concurrent.Callable; import java.util.concurrent.Callable;
@CommandLine.Command(name = "memstress", mixinStandardHelpOptions = true, versionProvider = VersionProvider.class, description = "Memory performance measurement tool.") @CommandLine.Command(name = "jmemperf", mixinStandardHelpOptions = true, versionProvider = VersionProvider.class, description = "Memory performance measurement tool.")
public class Application implements Callable<Integer> { public class Application implements Callable<Integer> {
final Logger log = LoggerFactory.getLogger(Application.class); final Logger log = LoggerFactory.getLogger(Application.class);
@ -22,21 +22,25 @@ public class Application implements Callable<Integer> {
@CommandLine.Option(names = { "-d", "--data" }, paramLabel = "NUM", description = "Create this much data (MB) pr. row [default: ${DEFAULT-VALUE}]") @CommandLine.Option(names = { "-d", "--data" }, paramLabel = "NUM", description = "Create this much data (MB) pr. row [default: ${DEFAULT-VALUE}]")
int maxDataPerRow = 100; int maxDataPerRow = 100;
@CommandLine.Option(names = { "-i", "--iterations" }, paramLabel = "NUM", description = "Iterate test his many times [default: ${DEFAULT-VALUE}]")
int iterations = 3;
@Override @Override
public Integer call() throws Exception { public Integer call() throws Exception {
MyDatabase database = new MyDatabase(maxTables, maxRowsPerTable, maxDataPerRow); long writeTimeMillis = 0;
database.build("testDb"); long readTimeMillis = 0;
System.out.println("TODO: How to search / read from data stored in rows?");
Scanner scanner = new Scanner(System.in);
System.out.println("Press ENTER to stop");
String s= scanner.nextLine();
scanner.close();
for(int i = 1; i <= iterations; i++) {
log.info("Starting test {} of {}", i, iterations);
MemDatabase database = new MemDatabase(maxTables, maxRowsPerTable, maxDataPerRow);
writeTimeMillis += database.write("testDb");
readTimeMillis += database.read("testDb");
database.destroy("testDb"); database.destroy("testDb");
}
log.info("Average writing time: {}", Duration.ofMillis(writeTimeMillis / iterations));
log.info("Average reading time: {}", Duration.ofMillis(readTimeMillis / iterations));
return 0; return 0;
} }

View file

@ -1,8 +1,8 @@
package biz.nellemann.memstress; package biz.nellemann.jmemperf;
import biz.nellemann.memstress.db.Database; import biz.nellemann.jmemperf.db.Database;
import biz.nellemann.memstress.db.DatabaseManager; import biz.nellemann.jmemperf.db.DatabaseManager;
import biz.nellemann.memstress.db.Table; import biz.nellemann.jmemperf.db.Table;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@ -10,18 +10,18 @@ import java.nio.ByteBuffer;
import java.time.Duration; import java.time.Duration;
import java.time.Instant; import java.time.Instant;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap; import java.util.HashMap;
import java.util.Random; import java.util.concurrent.atomic.AtomicLong;
public class MyDatabase { public class MemDatabase {
final Logger log = LoggerFactory.getLogger(MyDatabase.class); final Logger log = LoggerFactory.getLogger(MemDatabase.class);
private final int BYTE_SIZE_1MB = 1_000_000; private final int BYTE_SIZE_1MB = 1_000_000;
private final int BYTE_SIZE_1GB = 1_000_000_000; private final int BYTE_SIZE_1GB = 1_000_000_000;
private final DatabaseManager databaseManager = new DatabaseManager(); private final DatabaseManager databaseManager = new DatabaseManager();
private final Random random = new Random();
// Use when searching or using later? // Use when searching or using later?
private final ArrayList<Table> tables = new ArrayList<Table>(); private final ArrayList<Table> tables = new ArrayList<Table>();
@ -35,7 +35,7 @@ public class MyDatabase {
private char[] baseCar; private char[] baseCar;
private byte[] byteBase; private byte[] byteBase;
public MyDatabase(int tables, int rows, int size) { public MemDatabase(int tables, int rows, int size) {
this.maxTables = tables; this.maxTables = tables;
this.maxRowsPerTable = rows; this.maxRowsPerTable = rows;
this.maxDataPerRow = size; this.maxDataPerRow = size;
@ -44,19 +44,19 @@ public class MyDatabase {
for (int i = 0; i < 128; i++) { for (int i = 0; i < 128; i++) {
baseCar[i] = 'A'; baseCar[i] = 'A';
} }
for (int i = 0; i < byteBase.length; i++) { Arrays.fill(byteBase, (byte) 0);
byteBase[i] = 0;
}
} }
public Database build(String dbName) { public long write(String dbName) {
Instant instant1 = Instant.now(); Instant instant1 = Instant.now();
Database database = databaseManager.createDatabase(dbName); Database database = databaseManager.createDatabase(dbName);
AtomicLong bytesWritten = new AtomicLong();
for (int t = 1; t <= maxTables; t++) { for (int t = 1; t <= maxTables; t++) {
String tableName = String.format("table_%d", t); String tableName = String.format("table_%d", t);
log.info("Creating table \"{}\"", tableName); log.debug("Creating table \"{}\"", tableName);
Table table = database.createTable(tableName); Table table = database.createTable(tableName);
@ -65,6 +65,7 @@ public class MyDatabase {
HashMap<String, ByteBuffer> map = new HashMap<String, ByteBuffer>(); HashMap<String, ByteBuffer> map = new HashMap<String, ByteBuffer>();
for (int m = 1; m <= maxDataPerRow; m++) { for (int m = 1; m <= maxDataPerRow; m++) {
map.put(randomString(), randomBytes()); map.put(randomString(), randomBytes());
bytesWritten.addAndGet(byteBase.length);
} }
table.insertEntry(rowIdx, map); table.insertEntry(rowIdx, map);
} }
@ -73,22 +74,49 @@ public class MyDatabase {
} }
Instant instant2 = Instant.now(); Instant instant2 = Instant.now();
log.info("Done building in-memory database \"{}\" in {}", dbName, Duration.between(instant1, instant2)); Duration duration = Duration.between(instant1, instant2);
return database; log.info("Done writing {} bytes -> \"{}\" in {}", bytesWritten, dbName, duration);
return duration.toMillis();
} }
public long read(String dbName) {
Instant instant1 = Instant.now();
Database database = databaseManager.getDatabase(dbName);
AtomicLong bytesRead = new AtomicLong();
for(Table table : tables) {
table.getRows().forEach((idx, row) -> {
HashMap<String, ByteBuffer> values = row.getColumnValuesMap();
values.forEach((str, byteBuffer) -> {
byteBuffer.rewind();
while (byteBuffer.hasRemaining()) {
byte[] tmp = new byte[BYTE_SIZE_1MB];
byteBuffer.get(tmp);
bytesRead.addAndGet(tmp.length);
}
});
});
}
Instant instant2 = Instant.now();
Duration duration = Duration.between(instant1, instant2);
log.info("Done reading {} bytes <- \"{}\" in {}", bytesRead.get(), dbName, duration);
return duration.toMillis();
}
String randomString() { String randomString() {
baseCar[(idx++) % 128]++; baseCar[(idx++) % 128]++;
String s = new String(baseCar); return new String(baseCar);
return s;
} }
ByteBuffer randomBytes() { ByteBuffer randomBytes() {
byteBase[(idx2++) % byteBase.length]++; byteBase[(idx2++) % byteBase.length]++;
byte[] bytes = new byte[byteBase.length]; byte[] bytes = new byte[byteBase.length];
for (int i = 0; i < bytes.length; i++) { System.arraycopy(byteBase, 0, bytes, 0, bytes.length);
bytes[i] = byteBase[i];
}
return ByteBuffer.wrap(bytes); return ByteBuffer.wrap(bytes);
} }

View file

@ -1,4 +1,4 @@
package biz.nellemann.memstress; package biz.nellemann.jmemperf;
import picocli.CommandLine; import picocli.CommandLine;

View file

@ -1,7 +1,7 @@
/** /**
* Example taken from: https://medium.com/@mithoonkumar/design-an-in-memory-nosql-database-ood-428d48b68dfa * Example taken from: https://medium.com/@mithoonkumar/design-an-in-memory-nosql-database-ood-428d48b68dfa
*/ */
package biz.nellemann.memstress.db; package biz.nellemann.jmemperf.db;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;

View file

@ -1,16 +1,21 @@
/** /**
* Example taken from: https://medium.com/@mithoonkumar/design-an-in-memory-nosql-database-ood-428d48b68dfa * Example taken from: https://medium.com/@mithoonkumar/design-an-in-memory-nosql-database-ood-428d48b68dfa
*/ */
package biz.nellemann.memstress.db; package biz.nellemann.jmemperf.db;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.HashMap; import java.util.HashMap;
public class DatabaseManager { public class DatabaseManager {
final Logger log = LoggerFactory.getLogger(DatabaseManager.class);
private HashMap<String, Database> databaseHashMap; private HashMap<String, Database> databaseHashMap;
public Database createDatabase(String databaseName) { public Database createDatabase(String databaseName) {
if (databaseHashMap.containsKey(databaseName)) { if (databaseHashMap.containsKey(databaseName)) {
System.out.println("A database already exists with this name"); log.warn("createDatabase() - A database already exists with this name: {}", databaseName);
} else { } else {
databaseHashMap.put(databaseName, new Database(databaseName)); databaseHashMap.put(databaseName, new Database(databaseName));
} }
@ -21,6 +26,17 @@ public class DatabaseManager {
databaseHashMap.remove(databaseName); databaseHashMap.remove(databaseName);
} }
public Database getDatabase(String databaseName) {
if (databaseHashMap.containsKey(databaseName)) {
return databaseHashMap.get(databaseName);
} else {
log.warn("getDatabase() - Database was not found: {}", databaseName);
}
return null;
}
public DatabaseManager() { public DatabaseManager() {
this.databaseHashMap = new HashMap<>(); this.databaseHashMap = new HashMap<>();
} }

View file

@ -1,7 +1,7 @@
/** /**
* Example taken from: https://medium.com/@mithoonkumar/design-an-in-memory-nosql-database-ood-428d48b68dfa * Example taken from: https://medium.com/@mithoonkumar/design-an-in-memory-nosql-database-ood-428d48b68dfa
*/ */
package biz.nellemann.memstress.db; package biz.nellemann.jmemperf.db;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;

View file

@ -1,7 +1,7 @@
/** /**
* Example taken from: https://medium.com/@mithoonkumar/design-an-in-memory-nosql-database-ood-428d48b68dfa * Example taken from: https://medium.com/@mithoonkumar/design-an-in-memory-nosql-database-ood-428d48b68dfa
*/ */
package biz.nellemann.memstress.db; package biz.nellemann.jmemperf.db;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;

View file

@ -3,7 +3,7 @@
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender"> <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<withJansi>false</withJansi> <withJansi>false</withJansi>
<encoder> <encoder>
<pattern>%cyan(%d{yyyy-MM-dd HH:mm:ss.SSS}) %gray([%-10thread]) %highlight(%-5level) %magenta(%logger{12}) - %msg%n</pattern> <pattern>%cyan(%d{yyyy-MM-dd HH:mm:ss.SSS}) - %msg%n</pattern>
</encoder> </encoder>
</appender> </appender>

View file

@ -1,7 +1,7 @@
/* /*
* This Spock specification was generated by the Gradle 'init' task. * This Spock specification was generated by the Gradle 'init' task.
*/ */
package biz.nellemann.memstress package biz.nellemann.jmemperf
import spock.lang.Specification import spock.lang.Specification