From cd6e15584a43ab2ce2341d8fab085c652f81f709 Mon Sep 17 00:00:00 2001 From: Mark Nellemann Date: Wed, 14 Dec 2022 08:14:00 +0100 Subject: [PATCH] Correctly parse chuncked GELF messages. --- build.gradle | 2 +- .../nellemann/syslogd/parser/GelfParser.java | 22 +++++++---------- .../nellemann/syslogd/GelfParserTest.groovy | 24 +++++++++++++++---- .../biz/nellemann/syslogd/JsonUtilTest.groovy | 10 ++++++++ 4 files changed, 38 insertions(+), 20 deletions(-) diff --git a/build.gradle b/build.gradle index 3c50416..bfe2adf 100644 --- a/build.gradle +++ b/build.gradle @@ -60,7 +60,7 @@ jacocoTestCoverageVerification { } limit { counter = 'BRANCH' - minimum = 0.2 + minimum = 0.3 } limit { counter = 'CLASS' diff --git a/src/main/java/biz/nellemann/syslogd/parser/GelfParser.java b/src/main/java/biz/nellemann/syslogd/parser/GelfParser.java index 0861a2c..fa73d1f 100644 --- a/src/main/java/biz/nellemann/syslogd/parser/GelfParser.java +++ b/src/main/java/biz/nellemann/syslogd/parser/GelfParser.java @@ -9,8 +9,12 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.time.Instant; -import java.util.*; +import java.util.Arrays; +import java.util.TreeMap; +/* + For more information about the GELF format, visit: https://go2docs.graylog.org/5-0/getting_in_log_data/gelf.html + */ public class GelfParser extends SyslogParser { private final static Logger log = LoggerFactory.getLogger(GelfParser.class); @@ -35,7 +39,8 @@ public class GelfParser extends SyslogParser { Sequence number - 1 byte: The sequence number of this chunk starts at 0 and is always less than the sequence count. Sequence count - 1 byte: Total number of chunks this message has. - All chunks MUST arrive within 5 seconds or the server will discard all chunks that have arrived or are in the process of arriving. A message MUST NOT consist of more than 128 chunks. + All chunks MUST arrive within 5 seconds or the server will discard all chunks that have arrived or are in the process of arriving. + A message MUST NOT consist of more than 128 chunks. */ private SyslogMessage parseChunked(byte[] input) { @@ -62,7 +67,7 @@ public class GelfParser extends SyslogParser { integerTreeMap.put((int)seqNumber, payload); expiringMap.put(id, integerTreeMap); - if(seqNumber+1 >= seqTotal) { + if(integerTreeMap.size() >= seqTotal) { StringBuilder sb = new StringBuilder(); integerTreeMap.forEach( (i, p) -> { sb.append(byteArrayToString(p)); @@ -87,8 +92,6 @@ public class GelfParser extends SyslogParser { /* - https://go2docs.graylog.org/5-0/getting_in_log_data/gelf.html - zlib signatures at offset 0 78 01 : No Compression (no preset dictionary) 78 5E : Best speed (no preset dictionary) @@ -106,15 +109,6 @@ public class GelfParser extends SyslogParser { @Override public SyslogMessage parse(byte[] input) { - - for(byte b : input) { - if(b > 0x0) { - System.out.printf("%d, ", (b & 0xff)); - } - } - System.out.println(); - - if(input.length < 8) return null; // TODO: Find proper minimum input length ? // Compressed data: 0x78 0x9c diff --git a/src/test/groovy/biz/nellemann/syslogd/GelfParserTest.groovy b/src/test/groovy/biz/nellemann/syslogd/GelfParserTest.groovy index 0532c2e..4424c4a 100644 --- a/src/test/groovy/biz/nellemann/syslogd/GelfParserTest.groovy +++ b/src/test/groovy/biz/nellemann/syslogd/GelfParserTest.groovy @@ -5,7 +5,6 @@ import biz.nellemann.syslogd.msg.Severity import biz.nellemann.syslogd.msg.SyslogMessage import biz.nellemann.syslogd.parser.GelfParser import biz.nellemann.syslogd.parser.SyslogParser -import spock.lang.Ignore import spock.lang.Specification @@ -55,19 +54,34 @@ class GelfParserTest extends Specification { } - @Ignore void "chunked GELF message"() { setup: - byte[] chunk1 = [ 0x1E, 0x0F, 0x5A, 0x6D, 0x46, 0x28, 0x20, 0x02, 0x7B, 0x22, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6F, 0x6E, 0x22, 0x3A, 0x22, 0x31, 0x2E, 0x31, 0x22, 0x2C, 0x22, 0x68, 0x6F, 0x73, 0x74, 0x22, 0x3A, 0x22, 0x69, 0x70, 0x2D, 0x31, 0x30, 0x2D, 0x31, 0x2D, 0x31, 0x30, 0x32, 0x2D, 0x37, 0x35, 0x2E, 0x65, 0x75, 0x2D, 0x63, 0x65, 0x6E, 0x74, 0x72, 0x61, 0x6C, 0x2D, 0x31, 0x2E, 0x63, 0x6F, 0x6D, 0x70, 0x75, 0x74, 0x65, 0x2E, 0x69, 0x6E, 0x74, 0x65, 0x72, 0x6E, 0x61, 0x6C, 0x22, 0x2C, 0x22, 0x73, 0x68, 0x6F, 0x72, 0x74, 0x5F, 0x6D, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x3A, 0x22, 0x65, 0x76, 0x65, 0x6E, 0x74, 0x3D, 0x41, 0x75, 0x74, 0x68, 0x65, 0x6E, 0x74, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6F, 0x6E, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x45, 0x76, 0x65, 0x6E, 0x74, 0x20, 0x75, 0x73, 0x65, 0x72, 0x6E, 0x61, 0x6D, 0x65, 0x3D, 0x6D, 0x61, 0x72, 0x6B, 0x2E, 0x6E, 0x65, 0x6C, 0x6C, 0x65, 0x6D, 0x61, 0x6E, 0x6E, 0x40, 0x67, 0x6D, 0x61, 0x69, 0x6C, 0x2E, 0x63, 0x6F, 0x6D, 0x20, 0x74, 0x65, 0x6E, 0x61, 0x6E, 0x74, 0x3D, 0x33, 0x20, 0x72, 0x65, 0x6D, 0x6F, 0x74, 0x65, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x3D, 0x31, 0x32, 0x39, 0x2E, 0x34, 0x31, 0x2E, 0x34, 0x36, 0x2E, 0x37, 0x20, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6F, 0x6E, 0x49, 0x64, 0x3D, 0x32, 0x37, 0x38, 0x33, 0x35, 0x38, 0x43, 0x33, 0x33, 0x31, 0x45, 0x31, 0x41, 0x44, 0x30, 0x36, 0x36, 0x32, 0x42, 0x38, 0x37, 0x38, 0x43, 0x34, 0x37, 0x32, 0x46, 0x30, 0x32, 0x39, 0x30, 0x41, 0x22, 0x2C, 0x22, 0x66, 0x75, 0x6C, 0x6C, 0x5F, 0x6D, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x3A, 0x22, 0x65, 0x76, 0x65, 0x6E, 0x74, 0x3D, 0x41, 0x75, 0x74, 0x68, 0x65, 0x6E, 0x74, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6F, 0x6E, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x45, 0x76, 0x65, 0x6E, 0x74, 0x20, 0x75, 0x73, 0x65, 0x72, 0x6E, 0x61, 0x6D, 0x65, 0x3D, 0x6D, 0x61, 0x72, 0x6B, 0x2E, 0x6E, 0x65, 0x6C, 0x6C, 0x65, 0x6D, 0x61, 0x6E, 0x6E, 0x40, 0x67, 0x6D, 0x61, 0x69, 0x6C, 0x2E, 0x63, 0x6F, 0x6D, 0x20, 0x74, 0x65, 0x6E, 0x61, 0x6E, 0x74, 0x3D, 0x33, 0x20, 0x72, 0x65, 0x6D, 0x6F, 0x74, 0x65, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x3D, 0x31, 0x32, 0x39, 0x2E, 0x34, 0x31, 0x2E, 0x34, 0x36, 0x2E, 0x37, 0x20, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6F, 0x6E, 0x49, 0x64, 0x3D, 0x32, 0x37, 0x38, 0x33, 0x35, 0x38, 0x43, 0x33, 0x33, 0x31, 0x45, 0x31, 0x41, 0x44, 0x30, 0x36, 0x36, 0x32, 0x42, 0x38, 0x37, 0x38, 0x43, 0x34, 0x37, 0x32, 0x46, 0x30, 0x32, 0x39, 0x30, 0x41, 0x5C, 0x6E, 0x22, 0x2C, 0x22, 0x74, 0x69, 0x6D, 0x65, 0x73, 0x74, 0x61, 0x6D, 0x70, 0x22, 0x3A, 0x31, 0x36, 0x37, 0x30, 0x39, 0x34, 0x36, 0x32, 0x32, 0x33, 0x2E, 0x36, 0x36, 0x30, 0x2C, 0x22, 0x6C, 0x65, 0x76, 0x65, 0x6C, 0x22, 0x3A, 0x36, 0x2C, 0x22, 0x5F, 0x61, 0x70, 0x70, 0x5F, 0x63, 0x6F, 0x75, 0x6E, 0x74, 0x72, 0x79, 0x22, 0x3A, 0x22, 0x64, 0x6B, 0x22, 0x2C, 0x22, 0x5F, 0x61, 0x70, 0x70, 0x5F, 0x6E, 0x61, 0x6D, 0x65, 0x22, 0x3A, 0x22, 0x6D, 0x69, 0x6E, 0x74, 0x72, 0x22, 0x2C, 0x22, 0x5F, 0x6C, 0x6F, 0x67, 0x67, 0x65, 0x72, 0x5F, 0x6E, 0x61, 0x6D, 0x65, 0x22, 0x3A, 0x22, 0x61, 0x6A, 0x6F, 0x75, 0x72 ] - byte[] chunk2 = [ 0x1E, 0x0F, 0x5A, 0x6D, 0x46, 0x28, 0x20, 0x01, 0x02, 0x2E, 0x41, 0x6A, 0x6F, 0x75, 0x72, 0x53, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x45, 0x76, 0x65, 0x6E, 0x74, 0x4C, 0x69, 0x73, 0x74, 0x65, 0x6E, 0x65, 0x72, 0x22, 0x2C, 0x22, 0x5F, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5F, 0x75, 0x72, 0x6C, 0x22, 0x3A, 0x22, 0x68, 0x74, 0x74, 0x70, 0x73, 0x3A, 0x5C, 0x2F, 0x5C, 0x2F, 0x74, 0x65, 0x73, 0x74, 0x2E, 0x6D, 0x69, 0x6E, 0x74, 0x72, 0x2E, 0x61, 0x70, 0x70, 0x22, 0x2C, 0x22, 0x5F, 0x61, 0x70, 0x70, 0x5F, 0x65, 0x6E, 0x76, 0x22, 0x3A, 0x22, 0x74, 0x65, 0x73, 0x74, 0x22, 0x2C, 0x22, 0x5F, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x5F, 0x6E, 0x61, 0x6D, 0x65, 0x22, 0x3A, 0x22, 0x68, 0x74, 0x74, 0x70, 0x2D, 0x6E, 0x69, 0x6F, 0x2D, 0x38, 0x30, 0x38, 0x30, 0x2D, 0x65, 0x78, 0x65, 0x63, 0x2D, 0x38, 0x22, 0x7D ] + byte[] chunk1 = [30, 15, -103, 51, -96, -10, -51, -31, 39, -41, 0, 2, 123, 34, 118, 101, 114, 115, 105, 111, 110, 34, 58, 34, 49, 46, 49, 34, 44, 34, 104, 111, 115, 116, 34, 58, 34, 105, 112, 45, 49, 48, 45, 49, 45, 49, 48, 49, 45, 49, 52, 46, 101, 117, 45, 99, 101, 110, 116, 114, 97, 108, 45, 49, 46, 99, 111, 109, 112, 117, 116, 101, 46, 105, 110, 116, 101, 114, 110, 97, 108, 34, 44, 34, 115, 104, 111, 114, 116, 95, 109, 101, 115, 115, 97, 103, 101, 34, 58, 34, 101, 118, 101, 110, 116, 61, 65, 117, 116, 104, 101, 110, 116, 105, 99, 97, 116, 105, 111, 110, 83, 117, 99, 99, 101, 115, 115, 69, 118, 101, 110, 116, 32, 117, 115, 101, 114, 110, 97, 109, 101, 61, 109, 97, 114, 107, 46, 110, 101, 108, 108, 101, 109, 97, 110, 110, 64, 103, 109, 97, 105, 108, 46, 99, 111, 109, 32, 116, 101, 110, 97, 110, 116, 61, 50, 32, 114, 101, 109, 111, 116, 101, 65, 100, 100, 114, 101, 115, 115, 61, 49, 56, 53, 46, 49, 56, 49, 46, 50, 50, 51, 46, 49, 49, 55, 32, 115, 101, 115, 115, 105, 111, 110, 73, 100, 61, 110, 117, 108, 108, 34, 44, 34, 102, 117, 108, 108, 95, 109, 101, 115, 115, 97, 103, 101, 34, 58, 34, 101, 118, 101, 110, 116, 61, 65, 117, 116, 104, 101, 110, 116, 105, 99, 97, 116, 105, 111, 110, 83, 117, 99, 99, 101, 115, 115, 69, 118, 101, 110, 116, 32, 117, 115, 101, 114, 110, 97, 109, 101, 61, 109, 97, 114, 107, 46, 110, 101, 108, 108, 101, 109, 97, 110, 110, 64, 103, 109, 97, 105, 108, 46, 99, 111, 109, 32, 116, 101, 110, 97, 110, 116, 61, 50, 32, 114, 101, 109, 111, 116, 101, 65, 100, 100, 114, 101, 115, 115, 61, 49, 56, 53, 46, 49, 56, 49, 46, 50, 50, 51, 46, 49, 49, 55, 32, 115, 101, 115, 115, 105, 111, 110, 73, 100, 61, 110, 117, 108, 108, 92, 110, 34, 44, 34, 116, 105, 109, 101, 115, 116, 97, 109, 112, 34, 58, 49, 54, 55, 49, 48, 48, 48, 53, 53, 54, 46, 56, 48, 50, 44, 34, 108, 101, 118, 101, 108, 34, 58, 54, 44, 34, 95, 97, 112, 112, 95, 99, 111, 117, 110, 116, 114, 121, 34, 58, 34, 100, 107, 34, 44, 34, 95, 97, 112, 112, 95, 110, 97, 109, 101, 34, 58, 34, 109, 105, 110, 116, 114, 34, 44, 34, 95, 108, 111, 103, 103, 101, 114, 95, 110, 97, 109, 101, 34, 58, 34, 97, 106, 111, 117, 114, 46, 65, 106, 111, 117, 114, 83, 101, 99, 117, 114, 105, 116, 121, 83, 117, 99, 99, 101, 115, 115, 69, 118, 101, 110, 116, 76, 105, 115, 116, 101, 110, 101, 114, 34, 44, 34, 95, 115, 101, 114, 118, 101, 114, 95, 117, 114, 108] + byte[] chunk2 = [30, 15, -103, 51, -96, -10, -51, -31, 39, -41, 1, 2, 34, 58, 34, 104, 116, 116, 112, 115, 58, 92, 47, 92, 47, 100, 107, 46, 109, 105, 110, 116, 114, 46, 97, 112, 112, 34, 44, 34, 95, 97, 112, 112, 95, 101, 110, 118, 34, 58, 34, 112, 114, 111, 100, 34, 44, 34, 95, 116, 104, 114, 101, 97, 100, 95, 110, 97, 109, 101, 34, 58, 34, 104, 116, 116, 112, 45, 110, 105, 111, 45, 56, 48, 56, 48, 45, 101, 120, 101, 99, 45, 52, 34, 125] when: syslogParser.parse(chunk1) SyslogMessage msg = syslogParser.parse(chunk2) then: - msg.message == "fobar" + msg.message == "event=AuthenticationSuccessEvent username=mark.nellemann@gmail.com tenant=2 remoteAddress=185.181.223.117 sessionId=null" + + } + + + void "chunked GELF unordered message"() { + + setup: + byte[] chunk1 = [30, 15, -103, 51, -96, -10, -51, -31, 39, -41, 0, 2, 123, 34, 118, 101, 114, 115, 105, 111, 110, 34, 58, 34, 49, 46, 49, 34, 44, 34, 104, 111, 115, 116, 34, 58, 34, 105, 112, 45, 49, 48, 45, 49, 45, 49, 48, 49, 45, 49, 52, 46, 101, 117, 45, 99, 101, 110, 116, 114, 97, 108, 45, 49, 46, 99, 111, 109, 112, 117, 116, 101, 46, 105, 110, 116, 101, 114, 110, 97, 108, 34, 44, 34, 115, 104, 111, 114, 116, 95, 109, 101, 115, 115, 97, 103, 101, 34, 58, 34, 101, 118, 101, 110, 116, 61, 65, 117, 116, 104, 101, 110, 116, 105, 99, 97, 116, 105, 111, 110, 83, 117, 99, 99, 101, 115, 115, 69, 118, 101, 110, 116, 32, 117, 115, 101, 114, 110, 97, 109, 101, 61, 109, 97, 114, 107, 46, 110, 101, 108, 108, 101, 109, 97, 110, 110, 64, 103, 109, 97, 105, 108, 46, 99, 111, 109, 32, 116, 101, 110, 97, 110, 116, 61, 50, 32, 114, 101, 109, 111, 116, 101, 65, 100, 100, 114, 101, 115, 115, 61, 49, 56, 53, 46, 49, 56, 49, 46, 50, 50, 51, 46, 49, 49, 55, 32, 115, 101, 115, 115, 105, 111, 110, 73, 100, 61, 110, 117, 108, 108, 34, 44, 34, 102, 117, 108, 108, 95, 109, 101, 115, 115, 97, 103, 101, 34, 58, 34, 101, 118, 101, 110, 116, 61, 65, 117, 116, 104, 101, 110, 116, 105, 99, 97, 116, 105, 111, 110, 83, 117, 99, 99, 101, 115, 115, 69, 118, 101, 110, 116, 32, 117, 115, 101, 114, 110, 97, 109, 101, 61, 109, 97, 114, 107, 46, 110, 101, 108, 108, 101, 109, 97, 110, 110, 64, 103, 109, 97, 105, 108, 46, 99, 111, 109, 32, 116, 101, 110, 97, 110, 116, 61, 50, 32, 114, 101, 109, 111, 116, 101, 65, 100, 100, 114, 101, 115, 115, 61, 49, 56, 53, 46, 49, 56, 49, 46, 50, 50, 51, 46, 49, 49, 55, 32, 115, 101, 115, 115, 105, 111, 110, 73, 100, 61, 110, 117, 108, 108, 92, 110, 34, 44, 34, 116, 105, 109, 101, 115, 116, 97, 109, 112, 34, 58, 49, 54, 55, 49, 48, 48, 48, 53, 53, 54, 46, 56, 48, 50, 44, 34, 108, 101, 118, 101, 108, 34, 58, 54, 44, 34, 95, 97, 112, 112, 95, 99, 111, 117, 110, 116, 114, 121, 34, 58, 34, 100, 107, 34, 44, 34, 95, 97, 112, 112, 95, 110, 97, 109, 101, 34, 58, 34, 109, 105, 110, 116, 114, 34, 44, 34, 95, 108, 111, 103, 103, 101, 114, 95, 110, 97, 109, 101, 34, 58, 34, 97, 106, 111, 117, 114, 46, 65, 106, 111, 117, 114, 83, 101, 99, 117, 114, 105, 116, 121, 83, 117, 99, 99, 101, 115, 115, 69, 118, 101, 110, 116, 76, 105, 115, 116, 101, 110, 101, 114, 34, 44, 34, 95, 115, 101, 114, 118, 101, 114, 95, 117, 114, 108] + byte[] chunk2 = [30, 15, -103, 51, -96, -10, -51, -31, 39, -41, 1, 2, 34, 58, 34, 104, 116, 116, 112, 115, 58, 92, 47, 92, 47, 100, 107, 46, 109, 105, 110, 116, 114, 46, 97, 112, 112, 34, 44, 34, 95, 97, 112, 112, 95, 101, 110, 118, 34, 58, 34, 112, 114, 111, 100, 34, 44, 34, 95, 116, 104, 114, 101, 97, 100, 95, 110, 97, 109, 101, 34, 58, 34, 104, 116, 116, 112, 45, 110, 105, 111, 45, 56, 48, 56, 48, 45, 101, 120, 101, 99, 45, 52, 34, 125] + + when: + syslogParser.parse(chunk2) + SyslogMessage msg = syslogParser.parse(chunk1) + + then: + msg.message == "event=AuthenticationSuccessEvent username=mark.nellemann@gmail.com tenant=2 remoteAddress=185.181.223.117 sessionId=null" } diff --git a/src/test/groovy/biz/nellemann/syslogd/JsonUtilTest.groovy b/src/test/groovy/biz/nellemann/syslogd/JsonUtilTest.groovy index 7a05dd8..5bab382 100644 --- a/src/test/groovy/biz/nellemann/syslogd/JsonUtilTest.groovy +++ b/src/test/groovy/biz/nellemann/syslogd/JsonUtilTest.groovy @@ -39,4 +39,14 @@ class JsonUtilTest extends Specification { result == 'here it comes " to wreck the day...' } + def "test newline decode"() { + setup: + def testQuote = 'here it comes \n to wreck the day...' + + when: + def result = JsonUtil.decode(testQuote) + + then: + result == 'here it comes \n to wreck the day...' + } }