From 4cc11b05874c5a7134e0e598f8fb86ff7990f270 Mon Sep 17 00:00:00 2001 From: Mark Nellemann Date: Tue, 13 Dec 2022 09:14:21 +0100 Subject: [PATCH 1/3] Work on detecting chunked messages. --- .../nellemann/syslogd/parser/GelfParser.java | 41 ++++++++++++++----- 1 file changed, 31 insertions(+), 10 deletions(-) diff --git a/src/main/java/biz/nellemann/syslogd/parser/GelfParser.java b/src/main/java/biz/nellemann/syslogd/parser/GelfParser.java index abc5984..e0c6c3b 100644 --- a/src/main/java/biz/nellemann/syslogd/parser/GelfParser.java +++ b/src/main/java/biz/nellemann/syslogd/parser/GelfParser.java @@ -9,6 +9,7 @@ import org.slf4j.LoggerFactory; import java.io.UnsupportedEncodingException; import java.time.Instant; +import java.util.Arrays; import java.util.zip.DataFormatException; public class GelfParser extends SyslogParser { @@ -54,18 +55,38 @@ public class GelfParser extends SyslogParser { @Override public SyslogMessage parse(byte[] input) { - String text = null; - if(input[0] == (byte)0x78 && input[1] == (byte)0x9c) { // Compressed data - TODO: better detection ? - try { - text = decompress(input); - } catch (DataFormatException | UnsupportedEncodingException e) { - log.error("parse() - error: {}", e.getMessage()); - } - } else { - text = byteArrayToString(input); + + if(input.length < 8) return null; // TODO: Find proper minimum input length ? + String text; + + // GELF Magic Bytes: 0x1e 0x0f + if(input[0] == (byte)0x1e && input[1] == (byte)0x0f) { + /* + Message ID - 8 bytes: Must be the same for every chunk of this message. + Identifies the whole message and is used to reassemble the chunks later. + Generate from millisecond timestamp + hostname, for example. + Sequence number - 1 byte: The sequence number of this chunk starts at 0 and is always less than the sequence count. + Sequence count - 1 byte: Total number of chunks this message has. + + All chunks MUST arrive within 5 seconds or the server will discard all chunks that have arrived or are in the process of arriving. A message MUST NOT consist of more than 128 chunks. + */ + log.warn("parse() - Found Magic Bytes, can't parse yet."); + byte[] newInput = Arrays.copyOfRange(input, 12, input.length); + //return parse(byteArrayToString(newInput)); + return null; } - return parse(text); + // Compressed data: 0x78 0x9c + if(input[0] == (byte)0x78 && input[1] == (byte)0x9c) { // TODO: better detection ? + try { + return parse(decompress(input)); + } catch (DataFormatException | UnsupportedEncodingException e) { + log.error("parse() - error: {}", e.getMessage()); + return null; + } + } + + return parse(byteArrayToString(input)); } From 5104bd07506d08baf62d6ea48d5580fc9963499f Mon Sep 17 00:00:00 2001 From: Mark Nellemann Date: Tue, 13 Dec 2022 16:59:28 +0100 Subject: [PATCH 2/3] Work on chunked GELF messages. --- build.gradle | 2 +- .../nellemann/syslogd/LogReceiveEvent.java | 3 +- .../nellemann/syslogd/parser/GelfParser.java | 95 +++++++++++++------ .../syslogd/parser/SyslogParser.java | 27 ++++-- .../nellemann/syslogd/GelfParserTest.groovy | 24 +++-- 5 files changed, 108 insertions(+), 43 deletions(-) diff --git a/build.gradle b/build.gradle index 023f6ca..3c50416 100644 --- a/build.gradle +++ b/build.gradle @@ -19,9 +19,9 @@ dependencies { implementation 'org.slf4j:slf4j-simple:2.0.5' implementation 'com.fasterxml.jackson.core:jackson-databind:2.14.1' implementation 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.14.1' + implementation 'org.apache.commons:commons-collections4:4.4' testImplementation 'org.spockframework:spock-core:2.3-groovy-3.0' - testImplementation 'org.slf4j:slf4j-api:2.0.5' } application { diff --git a/src/main/java/biz/nellemann/syslogd/LogReceiveEvent.java b/src/main/java/biz/nellemann/syslogd/LogReceiveEvent.java index 203d65e..432245b 100644 --- a/src/main/java/biz/nellemann/syslogd/LogReceiveEvent.java +++ b/src/main/java/biz/nellemann/syslogd/LogReceiveEvent.java @@ -17,6 +17,7 @@ package biz.nellemann.syslogd; import java.net.DatagramPacket; import java.nio.charset.StandardCharsets; +import java.util.Arrays; import java.util.EventObject; public class LogReceiveEvent extends EventObject { @@ -42,7 +43,7 @@ public class LogReceiveEvent extends EventObject { } public byte[] getBytes() { - return packet.getData(); + return Arrays.copyOfRange(packet.getData(), packet.getOffset(), packet.getLength()); } } diff --git a/src/main/java/biz/nellemann/syslogd/parser/GelfParser.java b/src/main/java/biz/nellemann/syslogd/parser/GelfParser.java index e0c6c3b..0861a2c 100644 --- a/src/main/java/biz/nellemann/syslogd/parser/GelfParser.java +++ b/src/main/java/biz/nellemann/syslogd/parser/GelfParser.java @@ -4,13 +4,12 @@ import biz.nellemann.syslogd.msg.SyslogMessage; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import org.apache.commons.collections4.map.PassiveExpiringMap; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.UnsupportedEncodingException; import java.time.Instant; -import java.util.Arrays; -import java.util.zip.DataFormatException; +import java.util.*; public class GelfParser extends SyslogParser { @@ -18,12 +17,63 @@ public class GelfParser extends SyslogParser { private final ObjectMapper objectMapper; + private final int expiryInMills = 10_000; + private final PassiveExpiringMap> expiringMap = new PassiveExpiringMap<>(expiryInMills); + + public GelfParser() { objectMapper = new ObjectMapper(); objectMapper.registerModule(new JavaTimeModule()); } + /* + Magic Bytes - 2 bytes: 0x1e 0x0f + Message ID - 8 bytes: Must be the same for every chunk of this message. + Identifies the whole message and is used to reassemble the chunks later. + Generate from millisecond timestamp + hostname, for example. + Sequence number - 1 byte: The sequence number of this chunk starts at 0 and is always less than the sequence count. + Sequence count - 1 byte: Total number of chunks this message has. + + All chunks MUST arrive within 5 seconds or the server will discard all chunks that have arrived or are in the process of arriving. A message MUST NOT consist of more than 128 chunks. + */ + private SyslogMessage parseChunked(byte[] input) { + + if(input.length < 12) return null; + + byte[] messageId = { input[2], input[3], input[4], input[5], input[6], input[7], input[8], input[9] }; + byte seqNumber = input[10]; + byte seqTotal = input[11]; + byte[] payload = Arrays.copyOfRange(input, 12, input.length); + log.debug("parseChunked() - msgId: {}, seqNo: {}, seqTot: {}, payload: {}", messageId, seqNumber, seqTotal, byteArrayToString(payload)); + + // messageId byte[] to int + int id = 0; + for (byte b : messageId) { + id = (id << 8) + (b & 0xFF); + } + + TreeMap integerTreeMap; + if(expiringMap.containsKey(id)) { + integerTreeMap = expiringMap.get(id); + } else { + integerTreeMap = new TreeMap<>(); + } + integerTreeMap.put((int)seqNumber, payload); + expiringMap.put(id, integerTreeMap); + + if(seqNumber+1 >= seqTotal) { + StringBuilder sb = new StringBuilder(); + integerTreeMap.forEach( (i, p) -> { + sb.append(byteArrayToString(p)); + }); + return parse(sb.toString()); + } + + return null; + } + + @Override public SyslogMessage parse(String input) { SyslogMessage message = null; @@ -56,34 +106,25 @@ public class GelfParser extends SyslogParser { @Override public SyslogMessage parse(byte[] input) { - if(input.length < 8) return null; // TODO: Find proper minimum input length ? - String text; - // GELF Magic Bytes: 0x1e 0x0f - if(input[0] == (byte)0x1e && input[1] == (byte)0x0f) { - /* - Message ID - 8 bytes: Must be the same for every chunk of this message. - Identifies the whole message and is used to reassemble the chunks later. - Generate from millisecond timestamp + hostname, for example. - Sequence number - 1 byte: The sequence number of this chunk starts at 0 and is always less than the sequence count. - Sequence count - 1 byte: Total number of chunks this message has. - - All chunks MUST arrive within 5 seconds or the server will discard all chunks that have arrived or are in the process of arriving. A message MUST NOT consist of more than 128 chunks. - */ - log.warn("parse() - Found Magic Bytes, can't parse yet."); - byte[] newInput = Arrays.copyOfRange(input, 12, input.length); - //return parse(byteArrayToString(newInput)); - return null; + for(byte b : input) { + if(b > 0x0) { + System.out.printf("%d, ", (b & 0xff)); + } } + System.out.println(); + + + if(input.length < 8) return null; // TODO: Find proper minimum input length ? // Compressed data: 0x78 0x9c - if(input[0] == (byte)0x78 && input[1] == (byte)0x9c) { // TODO: better detection ? - try { - return parse(decompress(input)); - } catch (DataFormatException | UnsupportedEncodingException e) { - log.error("parse() - error: {}", e.getMessage()); - return null; - } + if(input[0] == (byte)0x78 && input[1] == (byte)0x9c) { + input = decompress(input); + } + + // Magic Bytes: 0x1e 0x0f + if(input[0] == (byte)0x1e && input[1] == (byte)0x0f) { + return parseChunked(input); } return parse(byteArrayToString(input)); diff --git a/src/main/java/biz/nellemann/syslogd/parser/SyslogParser.java b/src/main/java/biz/nellemann/syslogd/parser/SyslogParser.java index d2cc9fa..0c029b8 100644 --- a/src/main/java/biz/nellemann/syslogd/parser/SyslogParser.java +++ b/src/main/java/biz/nellemann/syslogd/parser/SyslogParser.java @@ -16,6 +16,8 @@ package biz.nellemann.syslogd.parser; import biz.nellemann.syslogd.msg.SyslogMessage; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.UnsupportedEncodingException; import java.nio.charset.StandardCharsets; @@ -25,6 +27,8 @@ import java.util.zip.Inflater; public abstract class SyslogParser { + private final static Logger log = LoggerFactory.getLogger(SyslogParser.class); + public abstract SyslogMessage parse(final String input); public abstract SyslogMessage parse(final byte[] input); @@ -62,17 +66,24 @@ public abstract class SyslogParser { } - protected String decompress(byte[] data) throws UnsupportedEncodingException, DataFormatException { + protected byte[] decompress(byte[] data) { - // Decompress the bytes - Inflater decompressor = new Inflater(); - decompressor.setInput(data, 0, data.length); byte[] result = new byte[data.length * 2]; - int resultLength = decompressor.inflate(result); - decompressor.end(); + try { + // Decompress the bytes + Inflater decompressor = new Inflater(); + decompressor.setInput(data, 0, data.length); + //byte[] result = new byte[data.length * 2]; + int resultLength = decompressor.inflate(result); + decompressor.end(); - // Decode the bytes into a String - return new String(result, 0, resultLength, StandardCharsets.UTF_8); + // Decode the bytes into a String + //uncompressed = new String(result, 0, resultLength, StandardCharsets.UTF_8); + } catch (DataFormatException e) { + log.error("decompress() - error: {}", e.getMessage()); + } + return result; } + } diff --git a/src/test/groovy/biz/nellemann/syslogd/GelfParserTest.groovy b/src/test/groovy/biz/nellemann/syslogd/GelfParserTest.groovy index d1fec72..0532c2e 100644 --- a/src/test/groovy/biz/nellemann/syslogd/GelfParserTest.groovy +++ b/src/test/groovy/biz/nellemann/syslogd/GelfParserTest.groovy @@ -5,6 +5,7 @@ import biz.nellemann.syslogd.msg.Severity import biz.nellemann.syslogd.msg.SyslogMessage import biz.nellemann.syslogd.parser.GelfParser import biz.nellemann.syslogd.parser.SyslogParser +import spock.lang.Ignore import spock.lang.Specification @@ -18,9 +19,6 @@ class GelfParserTest extends Specification { void "uncompressed GELF message"() { - /* - 0x7b 0x220x760x650x720x730x690x6f0x6e0x220x3a0x220x310x2e0x310x220x2c0x220x680x6f0x730x740x220x3a0x220x700x6f0x700x2d0x6f0x730x2e0x6c0x6f0x630x610x6c0x640x6f0x6d0x610x690x6e0x220x2c0x220x730x680x6f0x720x740x5f0x6d0x650x730x730x610x670x650x220x3a0x220x6d0x610x690x6e0x280x290x200x2d0x200x530x740x610x720x740x690x6e0x670x200x560x540x440x2d0x430x610x6d0x650x720x610x220x2c0x220x660x750x6c0x6c0x5f0x6d0x650x730x730x610x670x650x220x3a0x220x6d0x610x690x6e0x280x290x200x2d0x200x530x740x610x720x740x690x6e0x670x200x560x540x440x2d0x430x610x6d0x650x720x610x5c0x5c0x6e0x220x2c0x220x740x690x6d0x650x730x740x610x6d0x700x220x3a0x310x360x370x300x330x350x370x370x380x330x2e0x360x390x340x2c0x220x6c0x650x760x650x6c0x220x3a0x340x2c0x220x5f0x740x680x720x650x610x640x5f0x6e0x610x6d0x650x220x3a0x220x6d0x610x690x6e0x220x2c0x220x5f0x6c0x6f0x670x670x650x720x5f0x6e0x610x6d0x650x220x3a0x220x760x740x640x2e0x630x610x6d0x650x720x610x2e0x410x700x700x6c0x690x630x610x740x690x6f0x6e0x220x7d0x0a0x560x1f0x210x470x710x2c0xeb0x090x740x8e0x060x6f0x7f0xe70x7a0x330x890xb80xe10x6e0xa40x090xe90x9f0x870x1c0x290x900xcd0x880x4b0xea0xce0x9b0xf30x660xbc0x880x7b0x6f0x410x610x8f0x4c0xf00x570xaa0x590x620x180x8b0x820x730x040x350x3e0x120x8b0x930x7b0x130x780x5b0xb50x150x870x4f0xd00x5c0xb60xec0xfb0x070x9a0x690xc70xe8 - */ setup: def input = '{"version":"1.1","host":"pop-os.localdomain","short_message":"main() - Starting VTD-Camera","full_message":"main() - Starting VTD-Camera\\n","timestamp":1670357783.694,"level":4,"_thread_name":"main","_logger_name":"vtd.camera.Application"}' @@ -39,9 +37,6 @@ class GelfParserTest extends Specification { void "compressed GELF message"() { - // GELF Magic Bytes: 0x1e, 0x0f - - setup: byte[] input = [ 120, -100, -51, 81, 77, 107, -61, 48, 12, -3, 43, -63, -25, -38, -115, -77, -75, -51, 2, -127, -11, -80, -61, 96, -73, 94, 11, -63, 56, 90, -30, -59, 31, -63, -106, -53, -54, -40, 127, -97, 92, 86, -40, 79, -40, 69, -78, -97, -11, -12, -98, -28, 47, 118, -127, -104, 76, -16, -84, 99, 82, 72, -74, 97, 115, 72, 72, 23, -77, 114, 89, 115, 73, 65, -14, 102, 87, 11, -56, 92, -125, -57, -88, 44, -105, 66, 7, -73, 102, 4, 97, 60, 66, -12, -54, 18, 45, -51, 33, -30, -32, 32, 37, 53, 1, -15, -31, 66, -43, -3, 49, -29, 76, -39, 104, -123, -92, 113, -54, 90, 83, -63, 75, 121, -86, 114, 42, 84, 7, -67, 83, 113, 17, 30, -84, 5, -89, -68, 127, -98, -100, 50, -74, 40, 84, 8, 94, 81, -113, -90, -118, -32, 2, -62, 113, 28, 35, -79, 123, -39, -18, -124, 108, -91, 104, -102, 7, 33, -27, -95, 74, 4, 82, -13, -41, -79, -9, -39, 22, 43, -17, -108, -2, -127, -109, -77, 39, 47, 104, -56, 8, 42, -73, -78, 78, -18, 15, -11, -82, -106, -121, -89, 86, -44, -113, -5, 13, -77, 100, -52, -78, -114, 78, -125, 90, -41, 65, -121, 76, -21, -67, -110, -31, 113, 97, -65, 88, 113, 69, -128, -93, 61, -57, -126, -39, 48, 77, 16, -17, -80, -6, 8, 57, -118, 99, -119, 39, -48, 57, 26, -68, -2, -99, -21, -51, 36, -14, 13, 55, 34, 77, 72, -1, 60, -28, 72, -126, 108, 70, 92, 83, 119, -34, -98, -73, -29, 34, 110, -67, 5, -119, -35, 53, -63, 95, -88, 102, -115, 97, 44, 8, -50, 17, -44, 120, 87, 44, 76, -18, 77, -32, 109, -35, -42, 28, 62, 65, -13, -122, 125, -1, 0, -40, 60, -57, -72 ]; @@ -59,4 +54,21 @@ class GelfParserTest extends Specification { msg.timestamp.toString() == "2022-12-08T12:16:38.046Z" } + + @Ignore + void "chunked GELF message"() { + + setup: + byte[] chunk1 = [ 0x1E, 0x0F, 0x5A, 0x6D, 0x46, 0x28, 0x20, 0x02, 0x7B, 0x22, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6F, 0x6E, 0x22, 0x3A, 0x22, 0x31, 0x2E, 0x31, 0x22, 0x2C, 0x22, 0x68, 0x6F, 0x73, 0x74, 0x22, 0x3A, 0x22, 0x69, 0x70, 0x2D, 0x31, 0x30, 0x2D, 0x31, 0x2D, 0x31, 0x30, 0x32, 0x2D, 0x37, 0x35, 0x2E, 0x65, 0x75, 0x2D, 0x63, 0x65, 0x6E, 0x74, 0x72, 0x61, 0x6C, 0x2D, 0x31, 0x2E, 0x63, 0x6F, 0x6D, 0x70, 0x75, 0x74, 0x65, 0x2E, 0x69, 0x6E, 0x74, 0x65, 0x72, 0x6E, 0x61, 0x6C, 0x22, 0x2C, 0x22, 0x73, 0x68, 0x6F, 0x72, 0x74, 0x5F, 0x6D, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x3A, 0x22, 0x65, 0x76, 0x65, 0x6E, 0x74, 0x3D, 0x41, 0x75, 0x74, 0x68, 0x65, 0x6E, 0x74, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6F, 0x6E, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x45, 0x76, 0x65, 0x6E, 0x74, 0x20, 0x75, 0x73, 0x65, 0x72, 0x6E, 0x61, 0x6D, 0x65, 0x3D, 0x6D, 0x61, 0x72, 0x6B, 0x2E, 0x6E, 0x65, 0x6C, 0x6C, 0x65, 0x6D, 0x61, 0x6E, 0x6E, 0x40, 0x67, 0x6D, 0x61, 0x69, 0x6C, 0x2E, 0x63, 0x6F, 0x6D, 0x20, 0x74, 0x65, 0x6E, 0x61, 0x6E, 0x74, 0x3D, 0x33, 0x20, 0x72, 0x65, 0x6D, 0x6F, 0x74, 0x65, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x3D, 0x31, 0x32, 0x39, 0x2E, 0x34, 0x31, 0x2E, 0x34, 0x36, 0x2E, 0x37, 0x20, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6F, 0x6E, 0x49, 0x64, 0x3D, 0x32, 0x37, 0x38, 0x33, 0x35, 0x38, 0x43, 0x33, 0x33, 0x31, 0x45, 0x31, 0x41, 0x44, 0x30, 0x36, 0x36, 0x32, 0x42, 0x38, 0x37, 0x38, 0x43, 0x34, 0x37, 0x32, 0x46, 0x30, 0x32, 0x39, 0x30, 0x41, 0x22, 0x2C, 0x22, 0x66, 0x75, 0x6C, 0x6C, 0x5F, 0x6D, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x3A, 0x22, 0x65, 0x76, 0x65, 0x6E, 0x74, 0x3D, 0x41, 0x75, 0x74, 0x68, 0x65, 0x6E, 0x74, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6F, 0x6E, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x45, 0x76, 0x65, 0x6E, 0x74, 0x20, 0x75, 0x73, 0x65, 0x72, 0x6E, 0x61, 0x6D, 0x65, 0x3D, 0x6D, 0x61, 0x72, 0x6B, 0x2E, 0x6E, 0x65, 0x6C, 0x6C, 0x65, 0x6D, 0x61, 0x6E, 0x6E, 0x40, 0x67, 0x6D, 0x61, 0x69, 0x6C, 0x2E, 0x63, 0x6F, 0x6D, 0x20, 0x74, 0x65, 0x6E, 0x61, 0x6E, 0x74, 0x3D, 0x33, 0x20, 0x72, 0x65, 0x6D, 0x6F, 0x74, 0x65, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x3D, 0x31, 0x32, 0x39, 0x2E, 0x34, 0x31, 0x2E, 0x34, 0x36, 0x2E, 0x37, 0x20, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6F, 0x6E, 0x49, 0x64, 0x3D, 0x32, 0x37, 0x38, 0x33, 0x35, 0x38, 0x43, 0x33, 0x33, 0x31, 0x45, 0x31, 0x41, 0x44, 0x30, 0x36, 0x36, 0x32, 0x42, 0x38, 0x37, 0x38, 0x43, 0x34, 0x37, 0x32, 0x46, 0x30, 0x32, 0x39, 0x30, 0x41, 0x5C, 0x6E, 0x22, 0x2C, 0x22, 0x74, 0x69, 0x6D, 0x65, 0x73, 0x74, 0x61, 0x6D, 0x70, 0x22, 0x3A, 0x31, 0x36, 0x37, 0x30, 0x39, 0x34, 0x36, 0x32, 0x32, 0x33, 0x2E, 0x36, 0x36, 0x30, 0x2C, 0x22, 0x6C, 0x65, 0x76, 0x65, 0x6C, 0x22, 0x3A, 0x36, 0x2C, 0x22, 0x5F, 0x61, 0x70, 0x70, 0x5F, 0x63, 0x6F, 0x75, 0x6E, 0x74, 0x72, 0x79, 0x22, 0x3A, 0x22, 0x64, 0x6B, 0x22, 0x2C, 0x22, 0x5F, 0x61, 0x70, 0x70, 0x5F, 0x6E, 0x61, 0x6D, 0x65, 0x22, 0x3A, 0x22, 0x6D, 0x69, 0x6E, 0x74, 0x72, 0x22, 0x2C, 0x22, 0x5F, 0x6C, 0x6F, 0x67, 0x67, 0x65, 0x72, 0x5F, 0x6E, 0x61, 0x6D, 0x65, 0x22, 0x3A, 0x22, 0x61, 0x6A, 0x6F, 0x75, 0x72 ] + byte[] chunk2 = [ 0x1E, 0x0F, 0x5A, 0x6D, 0x46, 0x28, 0x20, 0x01, 0x02, 0x2E, 0x41, 0x6A, 0x6F, 0x75, 0x72, 0x53, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x45, 0x76, 0x65, 0x6E, 0x74, 0x4C, 0x69, 0x73, 0x74, 0x65, 0x6E, 0x65, 0x72, 0x22, 0x2C, 0x22, 0x5F, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5F, 0x75, 0x72, 0x6C, 0x22, 0x3A, 0x22, 0x68, 0x74, 0x74, 0x70, 0x73, 0x3A, 0x5C, 0x2F, 0x5C, 0x2F, 0x74, 0x65, 0x73, 0x74, 0x2E, 0x6D, 0x69, 0x6E, 0x74, 0x72, 0x2E, 0x61, 0x70, 0x70, 0x22, 0x2C, 0x22, 0x5F, 0x61, 0x70, 0x70, 0x5F, 0x65, 0x6E, 0x76, 0x22, 0x3A, 0x22, 0x74, 0x65, 0x73, 0x74, 0x22, 0x2C, 0x22, 0x5F, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x5F, 0x6E, 0x61, 0x6D, 0x65, 0x22, 0x3A, 0x22, 0x68, 0x74, 0x74, 0x70, 0x2D, 0x6E, 0x69, 0x6F, 0x2D, 0x38, 0x30, 0x38, 0x30, 0x2D, 0x65, 0x78, 0x65, 0x63, 0x2D, 0x38, 0x22, 0x7D ] + + when: + syslogParser.parse(chunk1) + SyslogMessage msg = syslogParser.parse(chunk2) + + then: + msg.message == "fobar" + + } + } From cd6e15584a43ab2ce2341d8fab085c652f81f709 Mon Sep 17 00:00:00 2001 From: Mark Nellemann Date: Wed, 14 Dec 2022 08:14:00 +0100 Subject: [PATCH 3/3] Correctly parse chuncked GELF messages. --- build.gradle | 2 +- .../nellemann/syslogd/parser/GelfParser.java | 22 +++++++---------- .../nellemann/syslogd/GelfParserTest.groovy | 24 +++++++++++++++---- .../biz/nellemann/syslogd/JsonUtilTest.groovy | 10 ++++++++ 4 files changed, 38 insertions(+), 20 deletions(-) diff --git a/build.gradle b/build.gradle index 3c50416..bfe2adf 100644 --- a/build.gradle +++ b/build.gradle @@ -60,7 +60,7 @@ jacocoTestCoverageVerification { } limit { counter = 'BRANCH' - minimum = 0.2 + minimum = 0.3 } limit { counter = 'CLASS' diff --git a/src/main/java/biz/nellemann/syslogd/parser/GelfParser.java b/src/main/java/biz/nellemann/syslogd/parser/GelfParser.java index 0861a2c..fa73d1f 100644 --- a/src/main/java/biz/nellemann/syslogd/parser/GelfParser.java +++ b/src/main/java/biz/nellemann/syslogd/parser/GelfParser.java @@ -9,8 +9,12 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.time.Instant; -import java.util.*; +import java.util.Arrays; +import java.util.TreeMap; +/* + For more information about the GELF format, visit: https://go2docs.graylog.org/5-0/getting_in_log_data/gelf.html + */ public class GelfParser extends SyslogParser { private final static Logger log = LoggerFactory.getLogger(GelfParser.class); @@ -35,7 +39,8 @@ public class GelfParser extends SyslogParser { Sequence number - 1 byte: The sequence number of this chunk starts at 0 and is always less than the sequence count. Sequence count - 1 byte: Total number of chunks this message has. - All chunks MUST arrive within 5 seconds or the server will discard all chunks that have arrived or are in the process of arriving. A message MUST NOT consist of more than 128 chunks. + All chunks MUST arrive within 5 seconds or the server will discard all chunks that have arrived or are in the process of arriving. + A message MUST NOT consist of more than 128 chunks. */ private SyslogMessage parseChunked(byte[] input) { @@ -62,7 +67,7 @@ public class GelfParser extends SyslogParser { integerTreeMap.put((int)seqNumber, payload); expiringMap.put(id, integerTreeMap); - if(seqNumber+1 >= seqTotal) { + if(integerTreeMap.size() >= seqTotal) { StringBuilder sb = new StringBuilder(); integerTreeMap.forEach( (i, p) -> { sb.append(byteArrayToString(p)); @@ -87,8 +92,6 @@ public class GelfParser extends SyslogParser { /* - https://go2docs.graylog.org/5-0/getting_in_log_data/gelf.html - zlib signatures at offset 0 78 01 : No Compression (no preset dictionary) 78 5E : Best speed (no preset dictionary) @@ -106,15 +109,6 @@ public class GelfParser extends SyslogParser { @Override public SyslogMessage parse(byte[] input) { - - for(byte b : input) { - if(b > 0x0) { - System.out.printf("%d, ", (b & 0xff)); - } - } - System.out.println(); - - if(input.length < 8) return null; // TODO: Find proper minimum input length ? // Compressed data: 0x78 0x9c diff --git a/src/test/groovy/biz/nellemann/syslogd/GelfParserTest.groovy b/src/test/groovy/biz/nellemann/syslogd/GelfParserTest.groovy index 0532c2e..4424c4a 100644 --- a/src/test/groovy/biz/nellemann/syslogd/GelfParserTest.groovy +++ b/src/test/groovy/biz/nellemann/syslogd/GelfParserTest.groovy @@ -5,7 +5,6 @@ import biz.nellemann.syslogd.msg.Severity import biz.nellemann.syslogd.msg.SyslogMessage import biz.nellemann.syslogd.parser.GelfParser import biz.nellemann.syslogd.parser.SyslogParser -import spock.lang.Ignore import spock.lang.Specification @@ -55,19 +54,34 @@ class GelfParserTest extends Specification { } - @Ignore void "chunked GELF message"() { setup: - byte[] chunk1 = [ 0x1E, 0x0F, 0x5A, 0x6D, 0x46, 0x28, 0x20, 0x02, 0x7B, 0x22, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6F, 0x6E, 0x22, 0x3A, 0x22, 0x31, 0x2E, 0x31, 0x22, 0x2C, 0x22, 0x68, 0x6F, 0x73, 0x74, 0x22, 0x3A, 0x22, 0x69, 0x70, 0x2D, 0x31, 0x30, 0x2D, 0x31, 0x2D, 0x31, 0x30, 0x32, 0x2D, 0x37, 0x35, 0x2E, 0x65, 0x75, 0x2D, 0x63, 0x65, 0x6E, 0x74, 0x72, 0x61, 0x6C, 0x2D, 0x31, 0x2E, 0x63, 0x6F, 0x6D, 0x70, 0x75, 0x74, 0x65, 0x2E, 0x69, 0x6E, 0x74, 0x65, 0x72, 0x6E, 0x61, 0x6C, 0x22, 0x2C, 0x22, 0x73, 0x68, 0x6F, 0x72, 0x74, 0x5F, 0x6D, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x3A, 0x22, 0x65, 0x76, 0x65, 0x6E, 0x74, 0x3D, 0x41, 0x75, 0x74, 0x68, 0x65, 0x6E, 0x74, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6F, 0x6E, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x45, 0x76, 0x65, 0x6E, 0x74, 0x20, 0x75, 0x73, 0x65, 0x72, 0x6E, 0x61, 0x6D, 0x65, 0x3D, 0x6D, 0x61, 0x72, 0x6B, 0x2E, 0x6E, 0x65, 0x6C, 0x6C, 0x65, 0x6D, 0x61, 0x6E, 0x6E, 0x40, 0x67, 0x6D, 0x61, 0x69, 0x6C, 0x2E, 0x63, 0x6F, 0x6D, 0x20, 0x74, 0x65, 0x6E, 0x61, 0x6E, 0x74, 0x3D, 0x33, 0x20, 0x72, 0x65, 0x6D, 0x6F, 0x74, 0x65, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x3D, 0x31, 0x32, 0x39, 0x2E, 0x34, 0x31, 0x2E, 0x34, 0x36, 0x2E, 0x37, 0x20, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6F, 0x6E, 0x49, 0x64, 0x3D, 0x32, 0x37, 0x38, 0x33, 0x35, 0x38, 0x43, 0x33, 0x33, 0x31, 0x45, 0x31, 0x41, 0x44, 0x30, 0x36, 0x36, 0x32, 0x42, 0x38, 0x37, 0x38, 0x43, 0x34, 0x37, 0x32, 0x46, 0x30, 0x32, 0x39, 0x30, 0x41, 0x22, 0x2C, 0x22, 0x66, 0x75, 0x6C, 0x6C, 0x5F, 0x6D, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x3A, 0x22, 0x65, 0x76, 0x65, 0x6E, 0x74, 0x3D, 0x41, 0x75, 0x74, 0x68, 0x65, 0x6E, 0x74, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6F, 0x6E, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x45, 0x76, 0x65, 0x6E, 0x74, 0x20, 0x75, 0x73, 0x65, 0x72, 0x6E, 0x61, 0x6D, 0x65, 0x3D, 0x6D, 0x61, 0x72, 0x6B, 0x2E, 0x6E, 0x65, 0x6C, 0x6C, 0x65, 0x6D, 0x61, 0x6E, 0x6E, 0x40, 0x67, 0x6D, 0x61, 0x69, 0x6C, 0x2E, 0x63, 0x6F, 0x6D, 0x20, 0x74, 0x65, 0x6E, 0x61, 0x6E, 0x74, 0x3D, 0x33, 0x20, 0x72, 0x65, 0x6D, 0x6F, 0x74, 0x65, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x3D, 0x31, 0x32, 0x39, 0x2E, 0x34, 0x31, 0x2E, 0x34, 0x36, 0x2E, 0x37, 0x20, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6F, 0x6E, 0x49, 0x64, 0x3D, 0x32, 0x37, 0x38, 0x33, 0x35, 0x38, 0x43, 0x33, 0x33, 0x31, 0x45, 0x31, 0x41, 0x44, 0x30, 0x36, 0x36, 0x32, 0x42, 0x38, 0x37, 0x38, 0x43, 0x34, 0x37, 0x32, 0x46, 0x30, 0x32, 0x39, 0x30, 0x41, 0x5C, 0x6E, 0x22, 0x2C, 0x22, 0x74, 0x69, 0x6D, 0x65, 0x73, 0x74, 0x61, 0x6D, 0x70, 0x22, 0x3A, 0x31, 0x36, 0x37, 0x30, 0x39, 0x34, 0x36, 0x32, 0x32, 0x33, 0x2E, 0x36, 0x36, 0x30, 0x2C, 0x22, 0x6C, 0x65, 0x76, 0x65, 0x6C, 0x22, 0x3A, 0x36, 0x2C, 0x22, 0x5F, 0x61, 0x70, 0x70, 0x5F, 0x63, 0x6F, 0x75, 0x6E, 0x74, 0x72, 0x79, 0x22, 0x3A, 0x22, 0x64, 0x6B, 0x22, 0x2C, 0x22, 0x5F, 0x61, 0x70, 0x70, 0x5F, 0x6E, 0x61, 0x6D, 0x65, 0x22, 0x3A, 0x22, 0x6D, 0x69, 0x6E, 0x74, 0x72, 0x22, 0x2C, 0x22, 0x5F, 0x6C, 0x6F, 0x67, 0x67, 0x65, 0x72, 0x5F, 0x6E, 0x61, 0x6D, 0x65, 0x22, 0x3A, 0x22, 0x61, 0x6A, 0x6F, 0x75, 0x72 ] - byte[] chunk2 = [ 0x1E, 0x0F, 0x5A, 0x6D, 0x46, 0x28, 0x20, 0x01, 0x02, 0x2E, 0x41, 0x6A, 0x6F, 0x75, 0x72, 0x53, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x45, 0x76, 0x65, 0x6E, 0x74, 0x4C, 0x69, 0x73, 0x74, 0x65, 0x6E, 0x65, 0x72, 0x22, 0x2C, 0x22, 0x5F, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5F, 0x75, 0x72, 0x6C, 0x22, 0x3A, 0x22, 0x68, 0x74, 0x74, 0x70, 0x73, 0x3A, 0x5C, 0x2F, 0x5C, 0x2F, 0x74, 0x65, 0x73, 0x74, 0x2E, 0x6D, 0x69, 0x6E, 0x74, 0x72, 0x2E, 0x61, 0x70, 0x70, 0x22, 0x2C, 0x22, 0x5F, 0x61, 0x70, 0x70, 0x5F, 0x65, 0x6E, 0x76, 0x22, 0x3A, 0x22, 0x74, 0x65, 0x73, 0x74, 0x22, 0x2C, 0x22, 0x5F, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x5F, 0x6E, 0x61, 0x6D, 0x65, 0x22, 0x3A, 0x22, 0x68, 0x74, 0x74, 0x70, 0x2D, 0x6E, 0x69, 0x6F, 0x2D, 0x38, 0x30, 0x38, 0x30, 0x2D, 0x65, 0x78, 0x65, 0x63, 0x2D, 0x38, 0x22, 0x7D ] + byte[] chunk1 = [30, 15, -103, 51, -96, -10, -51, -31, 39, -41, 0, 2, 123, 34, 118, 101, 114, 115, 105, 111, 110, 34, 58, 34, 49, 46, 49, 34, 44, 34, 104, 111, 115, 116, 34, 58, 34, 105, 112, 45, 49, 48, 45, 49, 45, 49, 48, 49, 45, 49, 52, 46, 101, 117, 45, 99, 101, 110, 116, 114, 97, 108, 45, 49, 46, 99, 111, 109, 112, 117, 116, 101, 46, 105, 110, 116, 101, 114, 110, 97, 108, 34, 44, 34, 115, 104, 111, 114, 116, 95, 109, 101, 115, 115, 97, 103, 101, 34, 58, 34, 101, 118, 101, 110, 116, 61, 65, 117, 116, 104, 101, 110, 116, 105, 99, 97, 116, 105, 111, 110, 83, 117, 99, 99, 101, 115, 115, 69, 118, 101, 110, 116, 32, 117, 115, 101, 114, 110, 97, 109, 101, 61, 109, 97, 114, 107, 46, 110, 101, 108, 108, 101, 109, 97, 110, 110, 64, 103, 109, 97, 105, 108, 46, 99, 111, 109, 32, 116, 101, 110, 97, 110, 116, 61, 50, 32, 114, 101, 109, 111, 116, 101, 65, 100, 100, 114, 101, 115, 115, 61, 49, 56, 53, 46, 49, 56, 49, 46, 50, 50, 51, 46, 49, 49, 55, 32, 115, 101, 115, 115, 105, 111, 110, 73, 100, 61, 110, 117, 108, 108, 34, 44, 34, 102, 117, 108, 108, 95, 109, 101, 115, 115, 97, 103, 101, 34, 58, 34, 101, 118, 101, 110, 116, 61, 65, 117, 116, 104, 101, 110, 116, 105, 99, 97, 116, 105, 111, 110, 83, 117, 99, 99, 101, 115, 115, 69, 118, 101, 110, 116, 32, 117, 115, 101, 114, 110, 97, 109, 101, 61, 109, 97, 114, 107, 46, 110, 101, 108, 108, 101, 109, 97, 110, 110, 64, 103, 109, 97, 105, 108, 46, 99, 111, 109, 32, 116, 101, 110, 97, 110, 116, 61, 50, 32, 114, 101, 109, 111, 116, 101, 65, 100, 100, 114, 101, 115, 115, 61, 49, 56, 53, 46, 49, 56, 49, 46, 50, 50, 51, 46, 49, 49, 55, 32, 115, 101, 115, 115, 105, 111, 110, 73, 100, 61, 110, 117, 108, 108, 92, 110, 34, 44, 34, 116, 105, 109, 101, 115, 116, 97, 109, 112, 34, 58, 49, 54, 55, 49, 48, 48, 48, 53, 53, 54, 46, 56, 48, 50, 44, 34, 108, 101, 118, 101, 108, 34, 58, 54, 44, 34, 95, 97, 112, 112, 95, 99, 111, 117, 110, 116, 114, 121, 34, 58, 34, 100, 107, 34, 44, 34, 95, 97, 112, 112, 95, 110, 97, 109, 101, 34, 58, 34, 109, 105, 110, 116, 114, 34, 44, 34, 95, 108, 111, 103, 103, 101, 114, 95, 110, 97, 109, 101, 34, 58, 34, 97, 106, 111, 117, 114, 46, 65, 106, 111, 117, 114, 83, 101, 99, 117, 114, 105, 116, 121, 83, 117, 99, 99, 101, 115, 115, 69, 118, 101, 110, 116, 76, 105, 115, 116, 101, 110, 101, 114, 34, 44, 34, 95, 115, 101, 114, 118, 101, 114, 95, 117, 114, 108] + byte[] chunk2 = [30, 15, -103, 51, -96, -10, -51, -31, 39, -41, 1, 2, 34, 58, 34, 104, 116, 116, 112, 115, 58, 92, 47, 92, 47, 100, 107, 46, 109, 105, 110, 116, 114, 46, 97, 112, 112, 34, 44, 34, 95, 97, 112, 112, 95, 101, 110, 118, 34, 58, 34, 112, 114, 111, 100, 34, 44, 34, 95, 116, 104, 114, 101, 97, 100, 95, 110, 97, 109, 101, 34, 58, 34, 104, 116, 116, 112, 45, 110, 105, 111, 45, 56, 48, 56, 48, 45, 101, 120, 101, 99, 45, 52, 34, 125] when: syslogParser.parse(chunk1) SyslogMessage msg = syslogParser.parse(chunk2) then: - msg.message == "fobar" + msg.message == "event=AuthenticationSuccessEvent username=mark.nellemann@gmail.com tenant=2 remoteAddress=185.181.223.117 sessionId=null" + + } + + + void "chunked GELF unordered message"() { + + setup: + byte[] chunk1 = [30, 15, -103, 51, -96, -10, -51, -31, 39, -41, 0, 2, 123, 34, 118, 101, 114, 115, 105, 111, 110, 34, 58, 34, 49, 46, 49, 34, 44, 34, 104, 111, 115, 116, 34, 58, 34, 105, 112, 45, 49, 48, 45, 49, 45, 49, 48, 49, 45, 49, 52, 46, 101, 117, 45, 99, 101, 110, 116, 114, 97, 108, 45, 49, 46, 99, 111, 109, 112, 117, 116, 101, 46, 105, 110, 116, 101, 114, 110, 97, 108, 34, 44, 34, 115, 104, 111, 114, 116, 95, 109, 101, 115, 115, 97, 103, 101, 34, 58, 34, 101, 118, 101, 110, 116, 61, 65, 117, 116, 104, 101, 110, 116, 105, 99, 97, 116, 105, 111, 110, 83, 117, 99, 99, 101, 115, 115, 69, 118, 101, 110, 116, 32, 117, 115, 101, 114, 110, 97, 109, 101, 61, 109, 97, 114, 107, 46, 110, 101, 108, 108, 101, 109, 97, 110, 110, 64, 103, 109, 97, 105, 108, 46, 99, 111, 109, 32, 116, 101, 110, 97, 110, 116, 61, 50, 32, 114, 101, 109, 111, 116, 101, 65, 100, 100, 114, 101, 115, 115, 61, 49, 56, 53, 46, 49, 56, 49, 46, 50, 50, 51, 46, 49, 49, 55, 32, 115, 101, 115, 115, 105, 111, 110, 73, 100, 61, 110, 117, 108, 108, 34, 44, 34, 102, 117, 108, 108, 95, 109, 101, 115, 115, 97, 103, 101, 34, 58, 34, 101, 118, 101, 110, 116, 61, 65, 117, 116, 104, 101, 110, 116, 105, 99, 97, 116, 105, 111, 110, 83, 117, 99, 99, 101, 115, 115, 69, 118, 101, 110, 116, 32, 117, 115, 101, 114, 110, 97, 109, 101, 61, 109, 97, 114, 107, 46, 110, 101, 108, 108, 101, 109, 97, 110, 110, 64, 103, 109, 97, 105, 108, 46, 99, 111, 109, 32, 116, 101, 110, 97, 110, 116, 61, 50, 32, 114, 101, 109, 111, 116, 101, 65, 100, 100, 114, 101, 115, 115, 61, 49, 56, 53, 46, 49, 56, 49, 46, 50, 50, 51, 46, 49, 49, 55, 32, 115, 101, 115, 115, 105, 111, 110, 73, 100, 61, 110, 117, 108, 108, 92, 110, 34, 44, 34, 116, 105, 109, 101, 115, 116, 97, 109, 112, 34, 58, 49, 54, 55, 49, 48, 48, 48, 53, 53, 54, 46, 56, 48, 50, 44, 34, 108, 101, 118, 101, 108, 34, 58, 54, 44, 34, 95, 97, 112, 112, 95, 99, 111, 117, 110, 116, 114, 121, 34, 58, 34, 100, 107, 34, 44, 34, 95, 97, 112, 112, 95, 110, 97, 109, 101, 34, 58, 34, 109, 105, 110, 116, 114, 34, 44, 34, 95, 108, 111, 103, 103, 101, 114, 95, 110, 97, 109, 101, 34, 58, 34, 97, 106, 111, 117, 114, 46, 65, 106, 111, 117, 114, 83, 101, 99, 117, 114, 105, 116, 121, 83, 117, 99, 99, 101, 115, 115, 69, 118, 101, 110, 116, 76, 105, 115, 116, 101, 110, 101, 114, 34, 44, 34, 95, 115, 101, 114, 118, 101, 114, 95, 117, 114, 108] + byte[] chunk2 = [30, 15, -103, 51, -96, -10, -51, -31, 39, -41, 1, 2, 34, 58, 34, 104, 116, 116, 112, 115, 58, 92, 47, 92, 47, 100, 107, 46, 109, 105, 110, 116, 114, 46, 97, 112, 112, 34, 44, 34, 95, 97, 112, 112, 95, 101, 110, 118, 34, 58, 34, 112, 114, 111, 100, 34, 44, 34, 95, 116, 104, 114, 101, 97, 100, 95, 110, 97, 109, 101, 34, 58, 34, 104, 116, 116, 112, 45, 110, 105, 111, 45, 56, 48, 56, 48, 45, 101, 120, 101, 99, 45, 52, 34, 125] + + when: + syslogParser.parse(chunk2) + SyslogMessage msg = syslogParser.parse(chunk1) + + then: + msg.message == "event=AuthenticationSuccessEvent username=mark.nellemann@gmail.com tenant=2 remoteAddress=185.181.223.117 sessionId=null" } diff --git a/src/test/groovy/biz/nellemann/syslogd/JsonUtilTest.groovy b/src/test/groovy/biz/nellemann/syslogd/JsonUtilTest.groovy index 7a05dd8..5bab382 100644 --- a/src/test/groovy/biz/nellemann/syslogd/JsonUtilTest.groovy +++ b/src/test/groovy/biz/nellemann/syslogd/JsonUtilTest.groovy @@ -39,4 +39,14 @@ class JsonUtilTest extends Specification { result == 'here it comes " to wreck the day...' } + def "test newline decode"() { + setup: + def testQuote = 'here it comes \n to wreck the day...' + + when: + def result = JsonUtil.decode(testQuote) + + then: + result == 'here it comes \n to wreck the day...' + } }