nellemann/syslogd
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases 3 Wiki Activity

Correctly parse chuncked GELF messages.

Browse source
This commit is contained in:
Mark Nellemann 2022-12-14 08:14:00 +01:00
parent 5104bd0750
commit cd6e15584a
4 changed files with 38 additions and 20 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
build.gradle
View file

@ -60,7 +60,7 @@ jacocoTestCoverageVerification {
} }
limit { limit {
counter = 'BRANCH' counter = 'BRANCH'
minimum = 0.2 minimum = 0.3
} }
limit { limit {
counter = 'CLASS' counter = 'CLASS'

22
src/main/java/biz/nellemann/syslogd/parser/GelfParser.java
View file

@ -9,8 +9,12 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import java.time.Instant; import java.time.Instant;
import java.util.*; import java.util.Arrays;
import java.util.TreeMap;
/*
For more information about the GELF format, visit: https://go2docs.graylog.org/5-0/getting_in_log_data/gelf.html
*/
public class GelfParser extends SyslogParser { public class GelfParser extends SyslogParser {
private final static Logger log = LoggerFactory.getLogger(GelfParser.class); private final static Logger log = LoggerFactory.getLogger(GelfParser.class);
@ -35,7 +39,8 @@ public class GelfParser extends SyslogParser {
Sequence number - 1 byte: The sequence number of this chunk starts at 0 and is always less than the sequence count. Sequence number - 1 byte: The sequence number of this chunk starts at 0 and is always less than the sequence count.
Sequence count - 1 byte: Total number of chunks this message has. Sequence count - 1 byte: Total number of chunks this message has.
All chunks MUST arrive within 5 seconds or the server will discard all chunks that have arrived or are in the process of arriving. A message MUST NOT consist of more than 128 chunks. All chunks MUST arrive within 5 seconds or the server will discard all chunks that have arrived or are in the process of arriving.
A message MUST NOT consist of more than 128 chunks.
*/ */
private SyslogMessage parseChunked(byte[] input) { private SyslogMessage parseChunked(byte[] input) {
@ -62,7 +67,7 @@ public class GelfParser extends SyslogParser {
integerTreeMap.put((int)seqNumber, payload); integerTreeMap.put((int)seqNumber, payload);
expiringMap.put(id, integerTreeMap); expiringMap.put(id, integerTreeMap);
if(seqNumber+1 >= seqTotal) { if(integerTreeMap.size() >= seqTotal) {
StringBuilder sb = new StringBuilder(); StringBuilder sb = new StringBuilder();
integerTreeMap.forEach( (i, p) -> { integerTreeMap.forEach( (i, p) -> {
sb.append(byteArrayToString(p)); sb.append(byteArrayToString(p));
@ -87,8 +92,6 @@ public class GelfParser extends SyslogParser {
/* /*
https://go2docs.graylog.org/5-0/getting_in_log_data/gelf.html
zlib signatures at offset 0 zlib signatures at offset 0
78 01 : No Compression (no preset dictionary) 78 01 : No Compression (no preset dictionary)
78 5E : Best speed (no preset dictionary) 78 5E : Best speed (no preset dictionary)
@ -106,15 +109,6 @@ public class GelfParser extends SyslogParser {
@Override @Override
public SyslogMessage parse(byte[] input) { public SyslogMessage parse(byte[] input) {
for(byte b : input) {
if(b > 0x0) {
System.out.printf("%d, ", (b & 0xff));
}
}
System.out.println();
if(input.length < 8) return null; // TODO: Find proper minimum input length ? if(input.length < 8) return null; // TODO: Find proper minimum input length ?
// Compressed data: 0x78 0x9c // Compressed data: 0x78 0x9c

24
src/test/groovy/biz/nellemann/syslogd/GelfParserTest.groovy
View file

@ -5,7 +5,6 @@ import biz.nellemann.syslogd.msg.Severity
import biz.nellemann.syslogd.msg.SyslogMessage import biz.nellemann.syslogd.msg.SyslogMessage
import biz.nellemann.syslogd.parser.GelfParser import biz.nellemann.syslogd.parser.GelfParser
import biz.nellemann.syslogd.parser.SyslogParser import biz.nellemann.syslogd.parser.SyslogParser
import spock.lang.Ignore
import spock.lang.Specification import spock.lang.Specification
@ -55,19 +54,34 @@ class GelfParserTest extends Specification {
} }
@Ignore
void "chunked GELF message"() { void "chunked GELF message"() {
setup: setup:
byte[] chunk1 = [ 0x1E, 0x0F, 0x5A, 0x6D, 0x46, 0x28, 0x20, 0x02, 0x7B, 0x22, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6F, 0x6E, 0x22, 0x3A, 0x22, 0x31, 0x2E, 0x31, 0x22, 0x2C, 0x22, 0x68, 0x6F, 0x73, 0x74, 0x22, 0x3A, 0x22, 0x69, 0x70, 0x2D, 0x31, 0x30, 0x2D, 0x31, 0x2D, 0x31, 0x30, 0x32, 0x2D, 0x37, 0x35, 0x2E, 0x65, 0x75, 0x2D, 0x63, 0x65, 0x6E, 0x74, 0x72, 0x61, 0x6C, 0x2D, 0x31, 0x2E, 0x63, 0x6F, 0x6D, 0x70, 0x75, 0x74, 0x65, 0x2E, 0x69, 0x6E, 0x74, 0x65, 0x72, 0x6E, 0x61, 0x6C, 0x22, 0x2C, 0x22, 0x73, 0x68, 0x6F, 0x72, 0x74, 0x5F, 0x6D, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x3A, 0x22, 0x65, 0x76, 0x65, 0x6E, 0x74, 0x3D, 0x41, 0x75, 0x74, 0x68, 0x65, 0x6E, 0x74, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6F, 0x6E, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x45, 0x76, 0x65, 0x6E, 0x74, 0x20, 0x75, 0x73, 0x65, 0x72, 0x6E, 0x61, 0x6D, 0x65, 0x3D, 0x6D, 0x61, 0x72, 0x6B, 0x2E, 0x6E, 0x65, 0x6C, 0x6C, 0x65, 0x6D, 0x61, 0x6E, 0x6E, 0x40, 0x67, 0x6D, 0x61, 0x69, 0x6C, 0x2E, 0x63, 0x6F, 0x6D, 0x20, 0x74, 0x65, 0x6E, 0x61, 0x6E, 0x74, 0x3D, 0x33, 0x20, 0x72, 0x65, 0x6D, 0x6F, 0x74, 0x65, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x3D, 0x31, 0x32, 0x39, 0x2E, 0x34, 0x31, 0x2E, 0x34, 0x36, 0x2E, 0x37, 0x20, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6F, 0x6E, 0x49, 0x64, 0x3D, 0x32, 0x37, 0x38, 0x33, 0x35, 0x38, 0x43, 0x33, 0x33, 0x31, 0x45, 0x31, 0x41, 0x44, 0x30, 0x36, 0x36, 0x32, 0x42, 0x38, 0x37, 0x38, 0x43, 0x34, 0x37, 0x32, 0x46, 0x30, 0x32, 0x39, 0x30, 0x41, 0x22, 0x2C, 0x22, 0x66, 0x75, 0x6C, 0x6C, 0x5F, 0x6D, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x3A, 0x22, 0x65, 0x76, 0x65, 0x6E, 0x74, 0x3D, 0x41, 0x75, 0x74, 0x68, 0x65, 0x6E, 0x74, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6F, 0x6E, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x45, 0x76, 0x65, 0x6E, 0x74, 0x20, 0x75, 0x73, 0x65, 0x72, 0x6E, 0x61, 0x6D, 0x65, 0x3D, 0x6D, 0x61, 0x72, 0x6B, 0x2E, 0x6E, 0x65, 0x6C, 0x6C, 0x65, 0x6D, 0x61, 0x6E, 0x6E, 0x40, 0x67, 0x6D, 0x61, 0x69, 0x6C, 0x2E, 0x63, 0x6F, 0x6D, 0x20, 0x74, 0x65, 0x6E, 0x61, 0x6E, 0x74, 0x3D, 0x33, 0x20, 0x72, 0x65, 0x6D, 0x6F, 0x74, 0x65, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x3D, 0x31, 0x32, 0x39, 0x2E, 0x34, 0x31, 0x2E, 0x34, 0x36, 0x2E, 0x37, 0x20, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6F, 0x6E, 0x49, 0x64, 0x3D, 0x32, 0x37, 0x38, 0x33, 0x35, 0x38, 0x43, 0x33, 0x33, 0x31, 0x45, 0x31, 0x41, 0x44, 0x30, 0x36, 0x36, 0x32, 0x42, 0x38, 0x37, 0x38, 0x43, 0x34, 0x37, 0x32, 0x46, 0x30, 0x32, 0x39, 0x30, 0x41, 0x5C, 0x6E, 0x22, 0x2C, 0x22, 0x74, 0x69, 0x6D, 0x65, 0x73, 0x74, 0x61, 0x6D, 0x70, 0x22, 0x3A, 0x31, 0x36, 0x37, 0x30, 0x39, 0x34, 0x36, 0x32, 0x32, 0x33, 0x2E, 0x36, 0x36, 0x30, 0x2C, 0x22, 0x6C, 0x65, 0x76, 0x65, 0x6C, 0x22, 0x3A, 0x36, 0x2C, 0x22, 0x5F, 0x61, 0x70, 0x70, 0x5F, 0x63, 0x6F, 0x75, 0x6E, 0x74, 0x72, 0x79, 0x22, 0x3A, 0x22, 0x64, 0x6B, 0x22, 0x2C, 0x22, 0x5F, 0x61, 0x70, 0x70, 0x5F, 0x6E, 0x61, 0x6D, 0x65, 0x22, 0x3A, 0x22, 0x6D, 0x69, 0x6E, 0x74, 0x72, 0x22, 0x2C, 0x22, 0x5F, 0x6C, 0x6F, 0x67, 0x67, 0x65, 0x72, 0x5F, 0x6E, 0x61, 0x6D, 0x65, 0x22, 0x3A, 0x22, 0x61, 0x6A, 0x6F, 0x75, 0x72 ] byte[] chunk1 = [30, 15, -103, 51, -96, -10, -51, -31, 39, -41, 0, 2, 123, 34, 118, 101, 114, 115, 105, 111, 110, 34, 58, 34, 49, 46, 49, 34, 44, 34, 104, 111, 115, 116, 34, 58, 34, 105, 112, 45, 49, 48, 45, 49, 45, 49, 48, 49, 45, 49, 52, 46, 101, 117, 45, 99, 101, 110, 116, 114, 97, 108, 45, 49, 46, 99, 111, 109, 112, 117, 116, 101, 46, 105, 110, 116, 101, 114, 110, 97, 108, 34, 44, 34, 115, 104, 111, 114, 116, 95, 109, 101, 115, 115, 97, 103, 101, 34, 58, 34, 101, 118, 101, 110, 116, 61, 65, 117, 116, 104, 101, 110, 116, 105, 99, 97, 116, 105, 111, 110, 83, 117, 99, 99, 101, 115, 115, 69, 118, 101, 110, 116, 32, 117, 115, 101, 114, 110, 97, 109, 101, 61, 109, 97, 114, 107, 46, 110, 101, 108, 108, 101, 109, 97, 110, 110, 64, 103, 109, 97, 105, 108, 46, 99, 111, 109, 32, 116, 101, 110, 97, 110, 116, 61, 50, 32, 114, 101, 109, 111, 116, 101, 65, 100, 100, 114, 101, 115, 115, 61, 49, 56, 53, 46, 49, 56, 49, 46, 50, 50, 51, 46, 49, 49, 55, 32, 115, 101, 115, 115, 105, 111, 110, 73, 100, 61, 110, 117, 108, 108, 34, 44, 34, 102, 117, 108, 108, 95, 109, 101, 115, 115, 97, 103, 101, 34, 58, 34, 101, 118, 101, 110, 116, 61, 65, 117, 116, 104, 101, 110, 116, 105, 99, 97, 116, 105, 111, 110, 83, 117, 99, 99, 101, 115, 115, 69, 118, 101, 110, 116, 32, 117, 115, 101, 114, 110, 97, 109, 101, 61, 109, 97, 114, 107, 46, 110, 101, 108, 108, 101, 109, 97, 110, 110, 64, 103, 109, 97, 105, 108, 46, 99, 111, 109, 32, 116, 101, 110, 97, 110, 116, 61, 50, 32, 114, 101, 109, 111, 116, 101, 65, 100, 100, 114, 101, 115, 115, 61, 49, 56, 53, 46, 49, 56, 49, 46, 50, 50, 51, 46, 49, 49, 55, 32, 115, 101, 115, 115, 105, 111, 110, 73, 100, 61, 110, 117, 108, 108, 92, 110, 34, 44, 34, 116, 105, 109, 101, 115, 116, 97, 109, 112, 34, 58, 49, 54, 55, 49, 48, 48, 48, 53, 53, 54, 46, 56, 48, 50, 44, 34, 108, 101, 118, 101, 108, 34, 58, 54, 44, 34, 95, 97, 112, 112, 95, 99, 111, 117, 110, 116, 114, 121, 34, 58, 34, 100, 107, 34, 44, 34, 95, 97, 112, 112, 95, 110, 97, 109, 101, 34, 58, 34, 109, 105, 110, 116, 114, 34, 44, 34, 95, 108, 111, 103, 103, 101, 114, 95, 110, 97, 109, 101, 34, 58, 34, 97, 106, 111, 117, 114, 46, 65, 106, 111, 117, 114, 83, 101, 99, 117, 114, 105, 116, 121, 83, 117, 99, 99, 101, 115, 115, 69, 118, 101, 110, 116, 76, 105, 115, 116, 101, 110, 101, 114, 34, 44, 34, 95, 115, 101, 114, 118, 101, 114, 95, 117, 114, 108]
byte[] chunk2 = [ 0x1E, 0x0F, 0x5A, 0x6D, 0x46, 0x28, 0x20, 0x01, 0x02, 0x2E, 0x41, 0x6A, 0x6F, 0x75, 0x72, 0x53, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x45, 0x76, 0x65, 0x6E, 0x74, 0x4C, 0x69, 0x73, 0x74, 0x65, 0x6E, 0x65, 0x72, 0x22, 0x2C, 0x22, 0x5F, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5F, 0x75, 0x72, 0x6C, 0x22, 0x3A, 0x22, 0x68, 0x74, 0x74, 0x70, 0x73, 0x3A, 0x5C, 0x2F, 0x5C, 0x2F, 0x74, 0x65, 0x73, 0x74, 0x2E, 0x6D, 0x69, 0x6E, 0x74, 0x72, 0x2E, 0x61, 0x70, 0x70, 0x22, 0x2C, 0x22, 0x5F, 0x61, 0x70, 0x70, 0x5F, 0x65, 0x6E, 0x76, 0x22, 0x3A, 0x22, 0x74, 0x65, 0x73, 0x74, 0x22, 0x2C, 0x22, 0x5F, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x5F, 0x6E, 0x61, 0x6D, 0x65, 0x22, 0x3A, 0x22, 0x68, 0x74, 0x74, 0x70, 0x2D, 0x6E, 0x69, 0x6F, 0x2D, 0x38, 0x30, 0x38, 0x30, 0x2D, 0x65, 0x78, 0x65, 0x63, 0x2D, 0x38, 0x22, 0x7D ] byte[] chunk2 = [30, 15, -103, 51, -96, -10, -51, -31, 39, -41, 1, 2, 34, 58, 34, 104, 116, 116, 112, 115, 58, 92, 47, 92, 47, 100, 107, 46, 109, 105, 110, 116, 114, 46, 97, 112, 112, 34, 44, 34, 95, 97, 112, 112, 95, 101, 110, 118, 34, 58, 34, 112, 114, 111, 100, 34, 44, 34, 95, 116, 104, 114, 101, 97, 100, 95, 110, 97, 109, 101, 34, 58, 34, 104, 116, 116, 112, 45, 110, 105, 111, 45, 56, 48, 56, 48, 45, 101, 120, 101, 99, 45, 52, 34, 125]
when: when:
syslogParser.parse(chunk1) syslogParser.parse(chunk1)
SyslogMessage msg = syslogParser.parse(chunk2) SyslogMessage msg = syslogParser.parse(chunk2)
then: then:
msg.message == "fobar" msg.message == "event=AuthenticationSuccessEvent username=mark.nellemann@gmail.com tenant=2 remoteAddress=185.181.223.117 sessionId=null"
}
void "chunked GELF unordered message"() {
setup:
byte[] chunk1 = [30, 15, -103, 51, -96, -10, -51, -31, 39, -41, 0, 2, 123, 34, 118, 101, 114, 115, 105, 111, 110, 34, 58, 34, 49, 46, 49, 34, 44, 34, 104, 111, 115, 116, 34, 58, 34, 105, 112, 45, 49, 48, 45, 49, 45, 49, 48, 49, 45, 49, 52, 46, 101, 117, 45, 99, 101, 110, 116, 114, 97, 108, 45, 49, 46, 99, 111, 109, 112, 117, 116, 101, 46, 105, 110, 116, 101, 114, 110, 97, 108, 34, 44, 34, 115, 104, 111, 114, 116, 95, 109, 101, 115, 115, 97, 103, 101, 34, 58, 34, 101, 118, 101, 110, 116, 61, 65, 117, 116, 104, 101, 110, 116, 105, 99, 97, 116, 105, 111, 110, 83, 117, 99, 99, 101, 115, 115, 69, 118, 101, 110, 116, 32, 117, 115, 101, 114, 110, 97, 109, 101, 61, 109, 97, 114, 107, 46, 110, 101, 108, 108, 101, 109, 97, 110, 110, 64, 103, 109, 97, 105, 108, 46, 99, 111, 109, 32, 116, 101, 110, 97, 110, 116, 61, 50, 32, 114, 101, 109, 111, 116, 101, 65, 100, 100, 114, 101, 115, 115, 61, 49, 56, 53, 46, 49, 56, 49, 46, 50, 50, 51, 46, 49, 49, 55, 32, 115, 101, 115, 115, 105, 111, 110, 73, 100, 61, 110, 117, 108, 108, 34, 44, 34, 102, 117, 108, 108, 95, 109, 101, 115, 115, 97, 103, 101, 34, 58, 34, 101, 118, 101, 110, 116, 61, 65, 117, 116, 104, 101, 110, 116, 105, 99, 97, 116, 105, 111, 110, 83, 117, 99, 99, 101, 115, 115, 69, 118, 101, 110, 116, 32, 117, 115, 101, 114, 110, 97, 109, 101, 61, 109, 97, 114, 107, 46, 110, 101, 108, 108, 101, 109, 97, 110, 110, 64, 103, 109, 97, 105, 108, 46, 99, 111, 109, 32, 116, 101, 110, 97, 110, 116, 61, 50, 32, 114, 101, 109, 111, 116, 101, 65, 100, 100, 114, 101, 115, 115, 61, 49, 56, 53, 46, 49, 56, 49, 46, 50, 50, 51, 46, 49, 49, 55, 32, 115, 101, 115, 115, 105, 111, 110, 73, 100, 61, 110, 117, 108, 108, 92, 110, 34, 44, 34, 116, 105, 109, 101, 115, 116, 97, 109, 112, 34, 58, 49, 54, 55, 49, 48, 48, 48, 53, 53, 54, 46, 56, 48, 50, 44, 34, 108, 101, 118, 101, 108, 34, 58, 54, 44, 34, 95, 97, 112, 112, 95, 99, 111, 117, 110, 116, 114, 121, 34, 58, 34, 100, 107, 34, 44, 34, 95, 97, 112, 112, 95, 110, 97, 109, 101, 34, 58, 34, 109, 105, 110, 116, 114, 34, 44, 34, 95, 108, 111, 103, 103, 101, 114, 95, 110, 97, 109, 101, 34, 58, 34, 97, 106, 111, 117, 114, 46, 65, 106, 111, 117, 114, 83, 101, 99, 117, 114, 105, 116, 121, 83, 117, 99, 99, 101, 115, 115, 69, 118, 101, 110, 116, 76, 105, 115, 116, 101, 110, 101, 114, 34, 44, 34, 95, 115, 101, 114, 118, 101, 114, 95, 117, 114, 108]
byte[] chunk2 = [30, 15, -103, 51, -96, -10, -51, -31, 39, -41, 1, 2, 34, 58, 34, 104, 116, 116, 112, 115, 58, 92, 47, 92, 47, 100, 107, 46, 109, 105, 110, 116, 114, 46, 97, 112, 112, 34, 44, 34, 95, 97, 112, 112, 95, 101, 110, 118, 34, 58, 34, 112, 114, 111, 100, 34, 44, 34, 95, 116, 104, 114, 101, 97, 100, 95, 110, 97, 109, 101, 34, 58, 34, 104, 116, 116, 112, 45, 110, 105, 111, 45, 56, 48, 56, 48, 45, 101, 120, 101, 99, 45, 52, 34, 125]
when:
syslogParser.parse(chunk2)
SyslogMessage msg = syslogParser.parse(chunk1)
then:
msg.message == "event=AuthenticationSuccessEvent username=mark.nellemann@gmail.com tenant=2 remoteAddress=185.181.223.117 sessionId=null"
} }

10
src/test/groovy/biz/nellemann/syslogd/JsonUtilTest.groovy
View file

@ -39,4 +39,14 @@ class JsonUtilTest extends Specification {
result == 'here it comes " to wreck the day...' result == 'here it comes " to wreck the day...'
} }
def "test newline decode"() {
setup:
def testQuote = 'here it comes \n to wreck the day...'
when:
def result = JsonUtil.decode(testQuote)
then:
result == 'here it comes \n to wreck the day...'
}
} }