Merge remote-tracking branch 'origin/dev' into dev

This commit is contained in:
Mark Nellemann 2023-06-29 18:52:40 +02:00
commit 5b14a7679f
5 changed files with 80 additions and 62 deletions

View file

@ -1,9 +1,14 @@
# Memory Performance Test # Memory Performance Test
## Examples ## Examples
Test with 8GB memory:
```shell ```shell
java -Xms128g -Xmx128g -XX:+UseLargePages -XX:+AlwaysPreTouch \ java -Xms10g -Xmx10g -XX:+AlwaysPreTouch -jar memstress-0.0.1-all.jar -t 8
-XX:-UseParallelGC -XX:MaxGCPauseMillis=500 -Xgcthreads3 \ ```
-jar memstress-0.0.1-all.jar -t 128
Test with 100GB memory:
```shell
java -Xms128g -Xmx128g -XX:+AlwaysPreTouch -jar memstress-0.0.1-all.jar -t 100
``` ```

View file

@ -2,7 +2,6 @@ plugins {
id 'groovy' id 'groovy'
id 'application' id 'application'
id "net.nemerosa.versioning" version "2.15.1" id "net.nemerosa.versioning" version "2.15.1"
id "com.netflix.nebula.ospackage" version "11.2.0"
id 'com.github.johnrengelman.shadow' version '8.1.1' id 'com.github.johnrengelman.shadow' version '8.1.1'
} }
@ -55,41 +54,3 @@ jar {
) )
} }
} }
apply plugin: 'com.netflix.nebula.ospackage'
ospackage {
packageName = 'memstress'
release = '1'
user = 'root'
packager = "Mark Nellemann <mark.nellemann@gmail.com>"
into '/opt/memstress'
from(shadowJar.outputs.files) {
into 'lib'
}
from('build/scriptsShadow') {
into 'bin'
}
from(['README.md', 'LICENSE']) {
into 'doc'
}
}
buildDeb {
dependsOn build, startShadowScripts
}
buildRpm {
dependsOn build, startShadowScripts
os = org.redline_rpm.header.Os.LINUX
}
tasks.register("packages") {
group "build"
dependsOn ":buildDeb"
dependsOn ":buildRpm"
}

View file

@ -4,7 +4,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import picocli.CommandLine; import picocli.CommandLine;
import java.util.Scanner; import java.time.Duration;
import java.util.concurrent.Callable; import java.util.concurrent.Callable;
@CommandLine.Command(name = "memstress", mixinStandardHelpOptions = true, versionProvider = VersionProvider.class, description = "Memory performance measurement tool.") @CommandLine.Command(name = "memstress", mixinStandardHelpOptions = true, versionProvider = VersionProvider.class, description = "Memory performance measurement tool.")
@ -22,21 +22,25 @@ public class Application implements Callable<Integer> {
@CommandLine.Option(names = { "-d", "--data" }, paramLabel = "NUM", description = "Create this much data (MB) pr. row [default: ${DEFAULT-VALUE}]") @CommandLine.Option(names = { "-d", "--data" }, paramLabel = "NUM", description = "Create this much data (MB) pr. row [default: ${DEFAULT-VALUE}]")
int maxDataPerRow = 100; int maxDataPerRow = 100;
@CommandLine.Option(names = { "-i", "--iterations" }, paramLabel = "NUM", description = "Iterate test his many times [default: ${DEFAULT-VALUE}]")
int iterations = 3;
@Override @Override
public Integer call() throws Exception { public Integer call() throws Exception {
MyDatabase database = new MyDatabase(maxTables, maxRowsPerTable, maxDataPerRow); long writeTimeMillis = 0;
database.build("testDb"); long readTimeMillis = 0;
System.out.println("TODO: How to search / read from data stored in rows?");
Scanner scanner = new Scanner(System.in);
System.out.println("Press ENTER to stop");
String s= scanner.nextLine();
scanner.close();
for(int i = 1; i <= iterations; i++) {
log.info("Starting test {} of {}", i, iterations);
MemDatabase database = new MemDatabase(maxTables, maxRowsPerTable, maxDataPerRow);
writeTimeMillis += database.write("testDb");
readTimeMillis += database.read("testDb");
database.destroy("testDb"); database.destroy("testDb");
}
log.info("Average writing time: {}", Duration.ofMillis(writeTimeMillis / iterations));
log.info("Average reading time: {}", Duration.ofMillis(readTimeMillis / iterations));
return 0; return 0;
} }

View file

@ -12,17 +12,16 @@ import java.time.Instant;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.HashMap; import java.util.HashMap;
import java.util.Random; import java.util.concurrent.atomic.AtomicLong;
public class MyDatabase { public class MemDatabase {
final Logger log = LoggerFactory.getLogger(MyDatabase.class); final Logger log = LoggerFactory.getLogger(MemDatabase.class);
private final int BYTE_SIZE_1MB = 1_000_000; private final int BYTE_SIZE_1MB = 1_000_000;
private final int BYTE_SIZE_1GB = 1_000_000_000; private final int BYTE_SIZE_1GB = 1_000_000_000;
private final DatabaseManager databaseManager = new DatabaseManager(); private final DatabaseManager databaseManager = new DatabaseManager();
private final Random random = new Random();
// Use when searching or using later? // Use when searching or using later?
private final ArrayList<Table> tables = new ArrayList<Table>(); private final ArrayList<Table> tables = new ArrayList<Table>();
@ -36,7 +35,7 @@ public class MyDatabase {
private char[] baseCar; private char[] baseCar;
private byte[] byteBase; private byte[] byteBase;
public MyDatabase(int tables, int rows, int size) { public MemDatabase(int tables, int rows, int size) {
this.maxTables = tables; this.maxTables = tables;
this.maxRowsPerTable = rows; this.maxRowsPerTable = rows;
this.maxDataPerRow = size; this.maxDataPerRow = size;
@ -49,13 +48,15 @@ public class MyDatabase {
} }
public Database build(String dbName) { public long write(String dbName) {
Instant instant1 = Instant.now(); Instant instant1 = Instant.now();
Database database = databaseManager.createDatabase(dbName); Database database = databaseManager.createDatabase(dbName);
AtomicLong bytesWritten = new AtomicLong();
for (int t = 1; t <= maxTables; t++) { for (int t = 1; t <= maxTables; t++) {
String tableName = String.format("table_%d", t); String tableName = String.format("table_%d", t);
log.info("Creating table \"{}\"", tableName); log.debug("Creating table \"{}\"", tableName);
Table table = database.createTable(tableName); Table table = database.createTable(tableName);
@ -64,6 +65,7 @@ public class MyDatabase {
HashMap<String, ByteBuffer> map = new HashMap<String, ByteBuffer>(); HashMap<String, ByteBuffer> map = new HashMap<String, ByteBuffer>();
for (int m = 1; m <= maxDataPerRow; m++) { for (int m = 1; m <= maxDataPerRow; m++) {
map.put(randomString(), randomBytes()); map.put(randomString(), randomBytes());
bytesWritten.addAndGet(byteBase.length);
} }
table.insertEntry(rowIdx, map); table.insertEntry(rowIdx, map);
} }
@ -72,10 +74,40 @@ public class MyDatabase {
} }
Instant instant2 = Instant.now(); Instant instant2 = Instant.now();
log.info("Done building in-memory database \"{}\" in {}", dbName, Duration.between(instant1, instant2)); Duration duration = Duration.between(instant1, instant2);
return database; log.info("Done writing {} bytes -> \"{}\" in {}", bytesWritten, dbName, duration);
return duration.toMillis();
} }
public long read(String dbName) {
Instant instant1 = Instant.now();
Database database = databaseManager.getDatabase(dbName);
AtomicLong bytesRead = new AtomicLong();
for(Table table : tables) {
table.getRows().forEach((idx, row) -> {
HashMap<String, ByteBuffer> values = row.getColumnValuesMap();
values.forEach((str, byteBuffer) -> {
byteBuffer.rewind();
while (byteBuffer.hasRemaining()) {
byte[] tmp = new byte[BYTE_SIZE_1MB];
byteBuffer.get(tmp);
bytesRead.addAndGet(tmp.length);
}
});
});
}
Instant instant2 = Instant.now();
Duration duration = Duration.between(instant1, instant2);
log.info("Done reading {} bytes <- \"{}\" in {}", bytesRead.get(), dbName, duration);
return duration.toMillis();
}
String randomString() { String randomString() {
baseCar[(idx++) % 128]++; baseCar[(idx++) % 128]++;
return new String(baseCar); return new String(baseCar);

View file

@ -3,14 +3,19 @@
*/ */
package biz.nellemann.memstress.db; package biz.nellemann.memstress.db;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.HashMap; import java.util.HashMap;
public class DatabaseManager { public class DatabaseManager {
final Logger log = LoggerFactory.getLogger(DatabaseManager.class);
private HashMap<String, Database> databaseHashMap; private HashMap<String, Database> databaseHashMap;
public Database createDatabase(String databaseName) { public Database createDatabase(String databaseName) {
if (databaseHashMap.containsKey(databaseName)) { if (databaseHashMap.containsKey(databaseName)) {
System.out.println("A database already exists with this name"); log.warn("createDatabase() - A database already exists with this name: {}", databaseName);
} else { } else {
databaseHashMap.put(databaseName, new Database(databaseName)); databaseHashMap.put(databaseName, new Database(databaseName));
} }
@ -21,6 +26,17 @@ public class DatabaseManager {
databaseHashMap.remove(databaseName); databaseHashMap.remove(databaseName);
} }
public Database getDatabase(String databaseName) {
if (databaseHashMap.containsKey(databaseName)) {
return databaseHashMap.get(databaseName);
} else {
log.warn("getDatabase() - Database was not found: {}", databaseName);
}
return null;
}
public DatabaseManager() { public DatabaseManager() {
this.databaseHashMap = new HashMap<>(); this.databaseHashMap = new HashMap<>();
} }