Merged in gelf-chunked (pull request #9)

Gelf chunked

build.gradle

@@ -19,9 +19,9 @@ dependencies {
     implementation 'org.slf4j:slf4j-simple:2.0.5'
     implementation 'com.fasterxml.jackson.core:jackson-databind:2.14.1'
     implementation 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.14.1'
+    implementation 'org.apache.commons:commons-collections4:4.4'
 
     testImplementation 'org.spockframework:spock-core:2.3-groovy-3.0'
-    testImplementation 'org.slf4j:slf4j-api:2.0.5'
 }
 
 application {
@@ -60,7 +60,7 @@ jacocoTestCoverageVerification {
             }
             limit {
                 counter = 'BRANCH'
-                minimum = 0.2
+                minimum = 0.3
             }
             limit {
                 counter = 'CLASS'

src/main/java/biz/nellemann/syslogd/LogReceiveEvent.java

@@ -17,6 +17,7 @@ package biz.nellemann.syslogd;
 
 import java.net.DatagramPacket;
 import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
 import java.util.EventObject;
 
 public class LogReceiveEvent extends EventObject {
@@ -42,7 +43,7 @@ public class LogReceiveEvent extends EventObject {
     }
 
     public byte[] getBytes() {
-        return packet.getData();
+        return Arrays.copyOfRange(packet.getData(), packet.getOffset(), packet.getLength());
     }
 
 }

src/main/java/biz/nellemann/syslogd/parser/GelfParser.java

@@ -4,25 +4,81 @@ import biz.nellemann.syslogd.msg.SyslogMessage;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
+import org.apache.commons.collections4.map.PassiveExpiringMap;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.UnsupportedEncodingException;
 import java.time.Instant;
-import java.util.zip.DataFormatException;
+import java.util.Arrays;
+import java.util.TreeMap;
 
+/*
+    For more information about the GELF format, visit:  https://go2docs.graylog.org/5-0/getting_in_log_data/gelf.html
+ */
 public class GelfParser extends SyslogParser {
 
     private final static Logger log = LoggerFactory.getLogger(GelfParser.class);
 
     private final ObjectMapper objectMapper;
 
+    private final int expiryInMills = 10_000;
+    private final PassiveExpiringMap<Integer, TreeMap<Integer, byte[]>> expiringMap = new PassiveExpiringMap<>(expiryInMills);
+
+
     public GelfParser() {
         objectMapper = new ObjectMapper();
         objectMapper.registerModule(new JavaTimeModule());
     }
 
 
+    /*
+        Magic Bytes - 2 bytes: 0x1e 0x0f
+        Message ID - 8 bytes: Must be the same for every chunk of this message.
+            Identifies the whole message and is used to reassemble the chunks later.
+            Generate from millisecond timestamp + hostname, for example.
+        Sequence number - 1 byte: The sequence number of this chunk starts at 0 and is always less than the sequence count.
+        Sequence count - 1 byte: Total number of chunks this message has.
+
+        All chunks MUST arrive within 5 seconds or the server will discard all chunks that have arrived or are in the process of arriving.
+        A message MUST NOT consist of more than 128 chunks.
+    */
+    private SyslogMessage parseChunked(byte[] input) {
+
+        if(input.length < 12) return null;
+
+        byte[] messageId = { input[2], input[3], input[4], input[5], input[6], input[7], input[8], input[9] };
+        byte seqNumber = input[10];
+        byte seqTotal = input[11];
+        byte[] payload = Arrays.copyOfRange(input, 12, input.length);
+        log.debug("parseChunked() - msgId: {}, seqNo: {}, seqTot: {}, payload: {}", messageId, seqNumber, seqTotal, byteArrayToString(payload));
+
+        // messageId byte[] to int
+        int id = 0;
+        for (byte b : messageId) {
+            id = (id << 8) + (b & 0xFF);
+        }
+
+        TreeMap<Integer, byte[]> integerTreeMap;
+        if(expiringMap.containsKey(id)) {
+            integerTreeMap = expiringMap.get(id);
+        } else {
+            integerTreeMap = new TreeMap<>();
+        }
+        integerTreeMap.put((int)seqNumber, payload);
+        expiringMap.put(id, integerTreeMap);
+
+        if(integerTreeMap.size() >= seqTotal) {
+            StringBuilder sb = new StringBuilder();
+            integerTreeMap.forEach( (i, p) -> {
+                sb.append(byteArrayToString(p));
+            });
+            return parse(sb.toString());
+        }
+
+        return null;
+    }
+
+
     @Override
     public SyslogMessage parse(String input) {
         SyslogMessage message = null;
@@ -36,8 +92,6 @@ public class GelfParser extends SyslogParser {
 
 
     /*
-        https://go2docs.graylog.org/5-0/getting_in_log_data/gelf.html
-
         zlib signatures at offset 0
         78 01 : No Compression (no preset dictionary)
         78 5E : Best speed (no preset dictionary)
@@ -54,18 +108,20 @@ public class GelfParser extends SyslogParser {
 
     @Override
     public SyslogMessage parse(byte[] input) {
-        String text = null;
+
-        if(input[0] == (byte)0x78 && input[1] == (byte)0x9c) {  // Compressed data - TODO: better detection ?
+        if(input.length < 8) return null;   // TODO: Find proper minimum input length ?
-            try {
+
-                text = decompress(input);
+        // Compressed data: 0x78 0x9c
-            } catch (DataFormatException | UnsupportedEncodingException e) {
+        if(input[0] == (byte)0x78 && input[1] == (byte)0x9c) {
-                log.error("parse() - error: {}", e.getMessage());
+            input = decompress(input);
-            }
-        } else {
-            text = byteArrayToString(input);
         }
 
-        return parse(text);
+        // Magic Bytes: 0x1e 0x0f
+        if(input[0] == (byte)0x1e && input[1] == (byte)0x0f) {
+            return parseChunked(input);
+        }
+
+        return parse(byteArrayToString(input));
     }
 
 

src/main/java/biz/nellemann/syslogd/parser/SyslogParser.java

@@ -16,6 +16,8 @@
 package biz.nellemann.syslogd.parser;
 
 import biz.nellemann.syslogd.msg.SyslogMessage;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.io.UnsupportedEncodingException;
 import java.nio.charset.StandardCharsets;
@@ -25,6 +27,8 @@ import java.util.zip.Inflater;
 
 public abstract class SyslogParser {
 
+    private final static Logger log = LoggerFactory.getLogger(SyslogParser.class);
+
     public abstract SyslogMessage parse(final String input);
     public abstract SyslogMessage parse(final byte[] input);
 
@@ -62,17 +66,24 @@ public abstract class SyslogParser {
     }
 
 
-    protected String decompress(byte[] data) throws UnsupportedEncodingException, DataFormatException {
+    protected byte[] decompress(byte[] data) {
 
+        byte[] result = new byte[data.length * 2];
+        try {
             // Decompress the bytes
             Inflater decompressor = new Inflater();
             decompressor.setInput(data, 0, data.length);
-        byte[] result = new byte[data.length * 2];
+            //byte[] result = new byte[data.length * 2];
             int resultLength = decompressor.inflate(result);
             decompressor.end();
 
             // Decode the bytes into a String
-        return new String(result, 0, resultLength, StandardCharsets.UTF_8);
+            //uncompressed = new String(result, 0, resultLength, StandardCharsets.UTF_8);
-
+        } catch (DataFormatException e) {
+            log.error("decompress() - error: {}", e.getMessage());
         }
+
+        return result;
+    }
+
 }

src/test/groovy/biz/nellemann/syslogd/GelfParserTest.groovy

@@ -18,9 +18,6 @@ class GelfParserTest extends Specification {
 
     void "uncompressed GELF message"() {
 
-        /*
-        0x7b 0x220x760x650x720x730x690x6f0x6e0x220x3a0x220x310x2e0x310x220x2c0x220x680x6f0x730x740x220x3a0x220x700x6f0x700x2d0x6f0x730x2e0x6c0x6f0x630x610x6c0x640x6f0x6d0x610x690x6e0x220x2c0x220x730x680x6f0x720x740x5f0x6d0x650x730x730x610x670x650x220x3a0x220x6d0x610x690x6e0x280x290x200x2d0x200x530x740x610x720x740x690x6e0x670x200x560x540x440x2d0x430x610x6d0x650x720x610x220x2c0x220x660x750x6c0x6c0x5f0x6d0x650x730x730x610x670x650x220x3a0x220x6d0x610x690x6e0x280x290x200x2d0x200x530x740x610x720x740x690x6e0x670x200x560x540x440x2d0x430x610x6d0x650x720x610x5c0x5c0x6e0x220x2c0x220x740x690x6d0x650x730x740x610x6d0x700x220x3a0x310x360x370x300x330x350x370x370x380x330x2e0x360x390x340x2c0x220x6c0x650x760x650x6c0x220x3a0x340x2c0x220x5f0x740x680x720x650x610x640x5f0x6e0x610x6d0x650x220x3a0x220x6d0x610x690x6e0x220x2c0x220x5f0x6c0x6f0x670x670x650x720x5f0x6e0x610x6d0x650x220x3a0x220x760x740x640x2e0x630x610x6d0x650x720x610x2e0x410x700x700x6c0x690x630x610x740x690x6f0x6e0x220x7d0x0a0x560x1f0x210x470x710x2c0xeb0x090x740x8e0x060x6f0x7f0xe70x7a0x330x890xb80xe10x6e0xa40x090xe90x9f0x870x1c0x290x900xcd0x880x4b0xea0xce0x9b0xf30x660xbc0x880x7b0x6f0x410x610x8f0x4c0xf00x570xaa0x590x620x180x8b0x820x730x040x350x3e0x120x8b0x930x7b0x130x780x5b0xb50x150x870x4f0xd00x5c0xb60xec0xfb0x070x9a0x690xc70xe8
-         */
         setup:
         def input = '{"version":"1.1","host":"pop-os.localdomain","short_message":"main() - Starting VTD-Camera","full_message":"main() - Starting VTD-Camera\\n","timestamp":1670357783.694,"level":4,"_thread_name":"main","_logger_name":"vtd.camera.Application"}'
 
@@ -39,9 +36,6 @@ class GelfParserTest extends Specification {
 
     void "compressed GELF message"() {
 
-        // GELF Magic Bytes: 0x1e, 0x0f
-
-
         setup:
         byte[] input = [ 120, -100, -51, 81, 77, 107, -61, 48, 12, -3, 43, -63, -25, -38, -115, -77, -75, -51, 2, -127, -11, -80, -61, 96, -73, 94, 11, -63, 56, 90, -30, -59, 31, -63, -106, -53, -54, -40, 127, -97, 92, 86, -40, 79, -40, 69, -78, -97, -11, -12, -98, -28, 47, 118, -127, -104, 76, -16, -84, 99, 82, 72, -74, 97, 115, 72, 72, 23, -77, 114, 89, 115, 73, 65, -14, 102, 87, 11, -56, 92, -125, -57, -88, 44, -105, 66, 7, -73, 102, 4, 97, 60, 66, -12, -54, 18, 45, -51, 33, -30, -32, 32, 37, 53, 1, -15, -31, 66, -43, -3, 49, -29, 76, -39, 104, -123, -92, 113, -54, 90, 83, -63, 75, 121, -86, 114, 42, 84, 7, -67, 83, 113, 17, 30, -84, 5, -89, -68, 127, -98, -100, 50, -74, 40, 84, 8, 94, 81, -113, -90, -118, -32, 2, -62, 113, 28, 35, -79, 123, -39, -18, -124, 108, -91, 104, -102, 7, 33, -27, -95, 74, 4, 82, -13, -41, -79, -9, -39, 22, 43, -17, -108, -2, -127, -109, -77, 39, 47, 104, -56, 8, 42, -73, -78, 78, -18, 15, -11, -82, -106, -121, -89, 86, -44, -113, -5, 13, -77, 100, -52, -78, -114, 78, -125, 90, -41, 65, -121, 76, -21, -67, -110, -31, 113, 97, -65, 88, 113, 69, -128, -93, 61, -57, -126, -39, 48, 77, 16, -17, -80, -6, 8, 57, -118, 99, -119, 39, -48, 57, 26, -68, -2, -99, -21, -51, 36, -14, 13, 55, 34, 77, 72, -1, 60, -28, 72, -126, 108, 70, 92, 83, 119, -34, -98, -73, -29, 34, 110, -67, 5, -119, -35, 53, -63, 95, -88, 102, -115, 97, 44, 8, -50, 17, -44, 120, 87, 44, 76, -18, 77, -32, 109, -35, -42, 28, 62, 65, -13, -122, 125, -1, 0, -40, 60, -57, -72 ];
 
@@ -59,4 +53,36 @@ class GelfParserTest extends Specification {
         msg.timestamp.toString() == "2022-12-08T12:16:38.046Z"
     }
 
+
+    void "chunked GELF message"() {
+
+        setup:
+        byte[] chunk1 = [30, 15, -103, 51, -96, -10, -51, -31, 39, -41, 0, 2, 123, 34, 118, 101, 114, 115, 105, 111, 110, 34, 58, 34, 49, 46, 49, 34, 44, 34, 104, 111, 115, 116, 34, 58, 34, 105, 112, 45, 49, 48, 45, 49, 45, 49, 48, 49, 45, 49, 52, 46, 101, 117, 45, 99, 101, 110, 116, 114, 97, 108, 45, 49, 46, 99, 111, 109, 112, 117, 116, 101, 46, 105, 110, 116, 101, 114, 110, 97, 108, 34, 44, 34, 115, 104, 111, 114, 116, 95, 109, 101, 115, 115, 97, 103, 101, 34, 58, 34, 101, 118, 101, 110, 116, 61, 65, 117, 116, 104, 101, 110, 116, 105, 99, 97, 116, 105, 111, 110, 83, 117, 99, 99, 101, 115, 115, 69, 118, 101, 110, 116, 32, 117, 115, 101, 114, 110, 97, 109, 101, 61, 109, 97, 114, 107, 46, 110, 101, 108, 108, 101, 109, 97, 110, 110, 64, 103, 109, 97, 105, 108, 46, 99, 111, 109, 32, 116, 101, 110, 97, 110, 116, 61, 50, 32, 114, 101, 109, 111, 116, 101, 65, 100, 100, 114, 101, 115, 115, 61, 49, 56, 53, 46, 49, 56, 49, 46, 50, 50, 51, 46, 49, 49, 55, 32, 115, 101, 115, 115, 105, 111, 110, 73, 100, 61, 110, 117, 108, 108, 34, 44, 34, 102, 117, 108, 108, 95, 109, 101, 115, 115, 97, 103, 101, 34, 58, 34, 101, 118, 101, 110, 116, 61, 65, 117, 116, 104, 101, 110, 116, 105, 99, 97, 116, 105, 111, 110, 83, 117, 99, 99, 101, 115, 115, 69, 118, 101, 110, 116, 32, 117, 115, 101, 114, 110, 97, 109, 101, 61, 109, 97, 114, 107, 46, 110, 101, 108, 108, 101, 109, 97, 110, 110, 64, 103, 109, 97, 105, 108, 46, 99, 111, 109, 32, 116, 101, 110, 97, 110, 116, 61, 50, 32, 114, 101, 109, 111, 116, 101, 65, 100, 100, 114, 101, 115, 115, 61, 49, 56, 53, 46, 49, 56, 49, 46, 50, 50, 51, 46, 49, 49, 55, 32, 115, 101, 115, 115, 105, 111, 110, 73, 100, 61, 110, 117, 108, 108, 92, 110, 34, 44, 34, 116, 105, 109, 101, 115, 116, 97, 109, 112, 34, 58, 49, 54, 55, 49, 48, 48, 48, 53, 53, 54, 46, 56, 48, 50, 44, 34, 108, 101, 118, 101, 108, 34, 58, 54, 44, 34, 95, 97, 112, 112, 95, 99, 111, 117, 110, 116, 114, 121, 34, 58, 34, 100, 107, 34, 44, 34, 95, 97, 112, 112, 95, 110, 97, 109, 101, 34, 58, 34, 109, 105, 110, 116, 114, 34, 44, 34, 95, 108, 111, 103, 103, 101, 114, 95, 110, 97, 109, 101, 34, 58, 34, 97, 106, 111, 117, 114, 46, 65, 106, 111, 117, 114, 83, 101, 99, 117, 114, 105, 116, 121, 83, 117, 99, 99, 101, 115, 115, 69, 118, 101, 110, 116, 76, 105, 115, 116, 101, 110, 101, 114, 34, 44, 34, 95, 115, 101, 114, 118, 101, 114, 95, 117, 114, 108]
+        byte[] chunk2 = [30, 15, -103, 51, -96, -10, -51, -31, 39, -41, 1, 2, 34, 58, 34, 104, 116, 116, 112, 115, 58, 92, 47, 92, 47, 100, 107, 46, 109, 105, 110, 116, 114, 46, 97, 112, 112, 34, 44, 34, 95, 97, 112, 112, 95, 101, 110, 118, 34, 58, 34, 112, 114, 111, 100, 34, 44, 34, 95, 116, 104, 114, 101, 97, 100, 95, 110, 97, 109, 101, 34, 58, 34, 104, 116, 116, 112, 45, 110, 105, 111, 45, 56, 48, 56, 48, 45, 101, 120, 101, 99, 45, 52, 34, 125]
+
+        when:
+        syslogParser.parse(chunk1)
+        SyslogMessage msg = syslogParser.parse(chunk2)
+
+        then:
+        msg.message == "event=AuthenticationSuccessEvent username=mark.nellemann@gmail.com tenant=2 remoteAddress=185.181.223.117 sessionId=null"
+
+    }
+
+
+    void "chunked GELF unordered message"() {
+
+        setup:
+        byte[] chunk1 = [30, 15, -103, 51, -96, -10, -51, -31, 39, -41, 0, 2, 123, 34, 118, 101, 114, 115, 105, 111, 110, 34, 58, 34, 49, 46, 49, 34, 44, 34, 104, 111, 115, 116, 34, 58, 34, 105, 112, 45, 49, 48, 45, 49, 45, 49, 48, 49, 45, 49, 52, 46, 101, 117, 45, 99, 101, 110, 116, 114, 97, 108, 45, 49, 46, 99, 111, 109, 112, 117, 116, 101, 46, 105, 110, 116, 101, 114, 110, 97, 108, 34, 44, 34, 115, 104, 111, 114, 116, 95, 109, 101, 115, 115, 97, 103, 101, 34, 58, 34, 101, 118, 101, 110, 116, 61, 65, 117, 116, 104, 101, 110, 116, 105, 99, 97, 116, 105, 111, 110, 83, 117, 99, 99, 101, 115, 115, 69, 118, 101, 110, 116, 32, 117, 115, 101, 114, 110, 97, 109, 101, 61, 109, 97, 114, 107, 46, 110, 101, 108, 108, 101, 109, 97, 110, 110, 64, 103, 109, 97, 105, 108, 46, 99, 111, 109, 32, 116, 101, 110, 97, 110, 116, 61, 50, 32, 114, 101, 109, 111, 116, 101, 65, 100, 100, 114, 101, 115, 115, 61, 49, 56, 53, 46, 49, 56, 49, 46, 50, 50, 51, 46, 49, 49, 55, 32, 115, 101, 115, 115, 105, 111, 110, 73, 100, 61, 110, 117, 108, 108, 34, 44, 34, 102, 117, 108, 108, 95, 109, 101, 115, 115, 97, 103, 101, 34, 58, 34, 101, 118, 101, 110, 116, 61, 65, 117, 116, 104, 101, 110, 116, 105, 99, 97, 116, 105, 111, 110, 83, 117, 99, 99, 101, 115, 115, 69, 118, 101, 110, 116, 32, 117, 115, 101, 114, 110, 97, 109, 101, 61, 109, 97, 114, 107, 46, 110, 101, 108, 108, 101, 109, 97, 110, 110, 64, 103, 109, 97, 105, 108, 46, 99, 111, 109, 32, 116, 101, 110, 97, 110, 116, 61, 50, 32, 114, 101, 109, 111, 116, 101, 65, 100, 100, 114, 101, 115, 115, 61, 49, 56, 53, 46, 49, 56, 49, 46, 50, 50, 51, 46, 49, 49, 55, 32, 115, 101, 115, 115, 105, 111, 110, 73, 100, 61, 110, 117, 108, 108, 92, 110, 34, 44, 34, 116, 105, 109, 101, 115, 116, 97, 109, 112, 34, 58, 49, 54, 55, 49, 48, 48, 48, 53, 53, 54, 46, 56, 48, 50, 44, 34, 108, 101, 118, 101, 108, 34, 58, 54, 44, 34, 95, 97, 112, 112, 95, 99, 111, 117, 110, 116, 114, 121, 34, 58, 34, 100, 107, 34, 44, 34, 95, 97, 112, 112, 95, 110, 97, 109, 101, 34, 58, 34, 109, 105, 110, 116, 114, 34, 44, 34, 95, 108, 111, 103, 103, 101, 114, 95, 110, 97, 109, 101, 34, 58, 34, 97, 106, 111, 117, 114, 46, 65, 106, 111, 117, 114, 83, 101, 99, 117, 114, 105, 116, 121, 83, 117, 99, 99, 101, 115, 115, 69, 118, 101, 110, 116, 76, 105, 115, 116, 101, 110, 101, 114, 34, 44, 34, 95, 115, 101, 114, 118, 101, 114, 95, 117, 114, 108]
+        byte[] chunk2 = [30, 15, -103, 51, -96, -10, -51, -31, 39, -41, 1, 2, 34, 58, 34, 104, 116, 116, 112, 115, 58, 92, 47, 92, 47, 100, 107, 46, 109, 105, 110, 116, 114, 46, 97, 112, 112, 34, 44, 34, 95, 97, 112, 112, 95, 101, 110, 118, 34, 58, 34, 112, 114, 111, 100, 34, 44, 34, 95, 116, 104, 114, 101, 97, 100, 95, 110, 97, 109, 101, 34, 58, 34, 104, 116, 116, 112, 45, 110, 105, 111, 45, 56, 48, 56, 48, 45, 101, 120, 101, 99, 45, 52, 34, 125]
+
+        when:
+        syslogParser.parse(chunk2)
+        SyslogMessage msg = syslogParser.parse(chunk1)
+
+        then:
+        msg.message == "event=AuthenticationSuccessEvent username=mark.nellemann@gmail.com tenant=2 remoteAddress=185.181.223.117 sessionId=null"
+
+    }
+
 }

src/test/groovy/biz/nellemann/syslogd/JsonUtilTest.groovy

@@ -39,4 +39,14 @@ class JsonUtilTest extends Specification {
         result == 'here it comes " to wreck the day...'
     }
 
+    def "test newline decode"() {
+        setup:
+        def testQuote = 'here it comes \n to wreck the day...'
+
+        when:
+        def result = JsonUtil.decode(testQuote)
+
+        then:
+        result == 'here it comes \n to wreck the day...'
+    }
 }