Correctly parse chuncked GELF messages.
This commit is contained in:
parent
5104bd0750
commit
cd6e15584a
|
|
@ -60,7 +60,7 @@ jacocoTestCoverageVerification {
|
|||
}
|
||||
limit {
|
||||
counter = 'BRANCH'
|
||||
minimum = 0.2
|
||||
minimum = 0.3
|
||||
}
|
||||
limit {
|
||||
counter = 'CLASS'
|
||||
|
|
|
|||
|
|
@ -9,8 +9,12 @@ import org.slf4j.Logger;
|
|||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.time.Instant;
|
||||
import java.util.*;
|
||||
import java.util.Arrays;
|
||||
import java.util.TreeMap;
|
||||
|
||||
/*
|
||||
For more information about the GELF format, visit: https://go2docs.graylog.org/5-0/getting_in_log_data/gelf.html
|
||||
*/
|
||||
public class GelfParser extends SyslogParser {
|
||||
|
||||
private final static Logger log = LoggerFactory.getLogger(GelfParser.class);
|
||||
|
|
@ -35,7 +39,8 @@ public class GelfParser extends SyslogParser {
|
|||
Sequence number - 1 byte: The sequence number of this chunk starts at 0 and is always less than the sequence count.
|
||||
Sequence count - 1 byte: Total number of chunks this message has.
|
||||
|
||||
All chunks MUST arrive within 5 seconds or the server will discard all chunks that have arrived or are in the process of arriving. A message MUST NOT consist of more than 128 chunks.
|
||||
All chunks MUST arrive within 5 seconds or the server will discard all chunks that have arrived or are in the process of arriving.
|
||||
A message MUST NOT consist of more than 128 chunks.
|
||||
*/
|
||||
private SyslogMessage parseChunked(byte[] input) {
|
||||
|
||||
|
|
@ -62,7 +67,7 @@ public class GelfParser extends SyslogParser {
|
|||
integerTreeMap.put((int)seqNumber, payload);
|
||||
expiringMap.put(id, integerTreeMap);
|
||||
|
||||
if(seqNumber+1 >= seqTotal) {
|
||||
if(integerTreeMap.size() >= seqTotal) {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
integerTreeMap.forEach( (i, p) -> {
|
||||
sb.append(byteArrayToString(p));
|
||||
|
|
@ -87,8 +92,6 @@ public class GelfParser extends SyslogParser {
|
|||
|
||||
|
||||
/*
|
||||
https://go2docs.graylog.org/5-0/getting_in_log_data/gelf.html
|
||||
|
||||
zlib signatures at offset 0
|
||||
78 01 : No Compression (no preset dictionary)
|
||||
78 5E : Best speed (no preset dictionary)
|
||||
|
|
@ -106,15 +109,6 @@ public class GelfParser extends SyslogParser {
|
|||
@Override
|
||||
public SyslogMessage parse(byte[] input) {
|
||||
|
||||
|
||||
for(byte b : input) {
|
||||
if(b > 0x0) {
|
||||
System.out.printf("%d, ", (b & 0xff));
|
||||
}
|
||||
}
|
||||
System.out.println();
|
||||
|
||||
|
||||
if(input.length < 8) return null; // TODO: Find proper minimum input length ?
|
||||
|
||||
// Compressed data: 0x78 0x9c
|
||||
|
|
|
|||
|
|
@ -5,7 +5,6 @@ import biz.nellemann.syslogd.msg.Severity
|
|||
import biz.nellemann.syslogd.msg.SyslogMessage
|
||||
import biz.nellemann.syslogd.parser.GelfParser
|
||||
import biz.nellemann.syslogd.parser.SyslogParser
|
||||
import spock.lang.Ignore
|
||||
import spock.lang.Specification
|
||||
|
||||
|
||||
|
|
@ -55,19 +54,34 @@ class GelfParserTest extends Specification {
|
|||
}
|
||||
|
||||
|
||||
@Ignore
|
||||
void "chunked GELF message"() {
|
||||
|
||||
setup:
|
||||
byte[] chunk1 = [ 0x1E, 0x0F, 0x5A, 0x6D, 0x46, 0x28, 0x20, 0x02, 0x7B, 0x22, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6F, 0x6E, 0x22, 0x3A, 0x22, 0x31, 0x2E, 0x31, 0x22, 0x2C, 0x22, 0x68, 0x6F, 0x73, 0x74, 0x22, 0x3A, 0x22, 0x69, 0x70, 0x2D, 0x31, 0x30, 0x2D, 0x31, 0x2D, 0x31, 0x30, 0x32, 0x2D, 0x37, 0x35, 0x2E, 0x65, 0x75, 0x2D, 0x63, 0x65, 0x6E, 0x74, 0x72, 0x61, 0x6C, 0x2D, 0x31, 0x2E, 0x63, 0x6F, 0x6D, 0x70, 0x75, 0x74, 0x65, 0x2E, 0x69, 0x6E, 0x74, 0x65, 0x72, 0x6E, 0x61, 0x6C, 0x22, 0x2C, 0x22, 0x73, 0x68, 0x6F, 0x72, 0x74, 0x5F, 0x6D, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x3A, 0x22, 0x65, 0x76, 0x65, 0x6E, 0x74, 0x3D, 0x41, 0x75, 0x74, 0x68, 0x65, 0x6E, 0x74, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6F, 0x6E, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x45, 0x76, 0x65, 0x6E, 0x74, 0x20, 0x75, 0x73, 0x65, 0x72, 0x6E, 0x61, 0x6D, 0x65, 0x3D, 0x6D, 0x61, 0x72, 0x6B, 0x2E, 0x6E, 0x65, 0x6C, 0x6C, 0x65, 0x6D, 0x61, 0x6E, 0x6E, 0x40, 0x67, 0x6D, 0x61, 0x69, 0x6C, 0x2E, 0x63, 0x6F, 0x6D, 0x20, 0x74, 0x65, 0x6E, 0x61, 0x6E, 0x74, 0x3D, 0x33, 0x20, 0x72, 0x65, 0x6D, 0x6F, 0x74, 0x65, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x3D, 0x31, 0x32, 0x39, 0x2E, 0x34, 0x31, 0x2E, 0x34, 0x36, 0x2E, 0x37, 0x20, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6F, 0x6E, 0x49, 0x64, 0x3D, 0x32, 0x37, 0x38, 0x33, 0x35, 0x38, 0x43, 0x33, 0x33, 0x31, 0x45, 0x31, 0x41, 0x44, 0x30, 0x36, 0x36, 0x32, 0x42, 0x38, 0x37, 0x38, 0x43, 0x34, 0x37, 0x32, 0x46, 0x30, 0x32, 0x39, 0x30, 0x41, 0x22, 0x2C, 0x22, 0x66, 0x75, 0x6C, 0x6C, 0x5F, 0x6D, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x3A, 0x22, 0x65, 0x76, 0x65, 0x6E, 0x74, 0x3D, 0x41, 0x75, 0x74, 0x68, 0x65, 0x6E, 0x74, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6F, 0x6E, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x45, 0x76, 0x65, 0x6E, 0x74, 0x20, 0x75, 0x73, 0x65, 0x72, 0x6E, 0x61, 0x6D, 0x65, 0x3D, 0x6D, 0x61, 0x72, 0x6B, 0x2E, 0x6E, 0x65, 0x6C, 0x6C, 0x65, 0x6D, 0x61, 0x6E, 0x6E, 0x40, 0x67, 0x6D, 0x61, 0x69, 0x6C, 0x2E, 0x63, 0x6F, 0x6D, 0x20, 0x74, 0x65, 0x6E, 0x61, 0x6E, 0x74, 0x3D, 0x33, 0x20, 0x72, 0x65, 0x6D, 0x6F, 0x74, 0x65, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x3D, 0x31, 0x32, 0x39, 0x2E, 0x34, 0x31, 0x2E, 0x34, 0x36, 0x2E, 0x37, 0x20, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6F, 0x6E, 0x49, 0x64, 0x3D, 0x32, 0x37, 0x38, 0x33, 0x35, 0x38, 0x43, 0x33, 0x33, 0x31, 0x45, 0x31, 0x41, 0x44, 0x30, 0x36, 0x36, 0x32, 0x42, 0x38, 0x37, 0x38, 0x43, 0x34, 0x37, 0x32, 0x46, 0x30, 0x32, 0x39, 0x30, 0x41, 0x5C, 0x6E, 0x22, 0x2C, 0x22, 0x74, 0x69, 0x6D, 0x65, 0x73, 0x74, 0x61, 0x6D, 0x70, 0x22, 0x3A, 0x31, 0x36, 0x37, 0x30, 0x39, 0x34, 0x36, 0x32, 0x32, 0x33, 0x2E, 0x36, 0x36, 0x30, 0x2C, 0x22, 0x6C, 0x65, 0x76, 0x65, 0x6C, 0x22, 0x3A, 0x36, 0x2C, 0x22, 0x5F, 0x61, 0x70, 0x70, 0x5F, 0x63, 0x6F, 0x75, 0x6E, 0x74, 0x72, 0x79, 0x22, 0x3A, 0x22, 0x64, 0x6B, 0x22, 0x2C, 0x22, 0x5F, 0x61, 0x70, 0x70, 0x5F, 0x6E, 0x61, 0x6D, 0x65, 0x22, 0x3A, 0x22, 0x6D, 0x69, 0x6E, 0x74, 0x72, 0x22, 0x2C, 0x22, 0x5F, 0x6C, 0x6F, 0x67, 0x67, 0x65, 0x72, 0x5F, 0x6E, 0x61, 0x6D, 0x65, 0x22, 0x3A, 0x22, 0x61, 0x6A, 0x6F, 0x75, 0x72 ]
|
||||
byte[] chunk2 = [ 0x1E, 0x0F, 0x5A, 0x6D, 0x46, 0x28, 0x20, 0x01, 0x02, 0x2E, 0x41, 0x6A, 0x6F, 0x75, 0x72, 0x53, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x45, 0x76, 0x65, 0x6E, 0x74, 0x4C, 0x69, 0x73, 0x74, 0x65, 0x6E, 0x65, 0x72, 0x22, 0x2C, 0x22, 0x5F, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5F, 0x75, 0x72, 0x6C, 0x22, 0x3A, 0x22, 0x68, 0x74, 0x74, 0x70, 0x73, 0x3A, 0x5C, 0x2F, 0x5C, 0x2F, 0x74, 0x65, 0x73, 0x74, 0x2E, 0x6D, 0x69, 0x6E, 0x74, 0x72, 0x2E, 0x61, 0x70, 0x70, 0x22, 0x2C, 0x22, 0x5F, 0x61, 0x70, 0x70, 0x5F, 0x65, 0x6E, 0x76, 0x22, 0x3A, 0x22, 0x74, 0x65, 0x73, 0x74, 0x22, 0x2C, 0x22, 0x5F, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x5F, 0x6E, 0x61, 0x6D, 0x65, 0x22, 0x3A, 0x22, 0x68, 0x74, 0x74, 0x70, 0x2D, 0x6E, 0x69, 0x6F, 0x2D, 0x38, 0x30, 0x38, 0x30, 0x2D, 0x65, 0x78, 0x65, 0x63, 0x2D, 0x38, 0x22, 0x7D ]
|
||||
byte[] chunk1 = [30, 15, -103, 51, -96, -10, -51, -31, 39, -41, 0, 2, 123, 34, 118, 101, 114, 115, 105, 111, 110, 34, 58, 34, 49, 46, 49, 34, 44, 34, 104, 111, 115, 116, 34, 58, 34, 105, 112, 45, 49, 48, 45, 49, 45, 49, 48, 49, 45, 49, 52, 46, 101, 117, 45, 99, 101, 110, 116, 114, 97, 108, 45, 49, 46, 99, 111, 109, 112, 117, 116, 101, 46, 105, 110, 116, 101, 114, 110, 97, 108, 34, 44, 34, 115, 104, 111, 114, 116, 95, 109, 101, 115, 115, 97, 103, 101, 34, 58, 34, 101, 118, 101, 110, 116, 61, 65, 117, 116, 104, 101, 110, 116, 105, 99, 97, 116, 105, 111, 110, 83, 117, 99, 99, 101, 115, 115, 69, 118, 101, 110, 116, 32, 117, 115, 101, 114, 110, 97, 109, 101, 61, 109, 97, 114, 107, 46, 110, 101, 108, 108, 101, 109, 97, 110, 110, 64, 103, 109, 97, 105, 108, 46, 99, 111, 109, 32, 116, 101, 110, 97, 110, 116, 61, 50, 32, 114, 101, 109, 111, 116, 101, 65, 100, 100, 114, 101, 115, 115, 61, 49, 56, 53, 46, 49, 56, 49, 46, 50, 50, 51, 46, 49, 49, 55, 32, 115, 101, 115, 115, 105, 111, 110, 73, 100, 61, 110, 117, 108, 108, 34, 44, 34, 102, 117, 108, 108, 95, 109, 101, 115, 115, 97, 103, 101, 34, 58, 34, 101, 118, 101, 110, 116, 61, 65, 117, 116, 104, 101, 110, 116, 105, 99, 97, 116, 105, 111, 110, 83, 117, 99, 99, 101, 115, 115, 69, 118, 101, 110, 116, 32, 117, 115, 101, 114, 110, 97, 109, 101, 61, 109, 97, 114, 107, 46, 110, 101, 108, 108, 101, 109, 97, 110, 110, 64, 103, 109, 97, 105, 108, 46, 99, 111, 109, 32, 116, 101, 110, 97, 110, 116, 61, 50, 32, 114, 101, 109, 111, 116, 101, 65, 100, 100, 114, 101, 115, 115, 61, 49, 56, 53, 46, 49, 56, 49, 46, 50, 50, 51, 46, 49, 49, 55, 32, 115, 101, 115, 115, 105, 111, 110, 73, 100, 61, 110, 117, 108, 108, 92, 110, 34, 44, 34, 116, 105, 109, 101, 115, 116, 97, 109, 112, 34, 58, 49, 54, 55, 49, 48, 48, 48, 53, 53, 54, 46, 56, 48, 50, 44, 34, 108, 101, 118, 101, 108, 34, 58, 54, 44, 34, 95, 97, 112, 112, 95, 99, 111, 117, 110, 116, 114, 121, 34, 58, 34, 100, 107, 34, 44, 34, 95, 97, 112, 112, 95, 110, 97, 109, 101, 34, 58, 34, 109, 105, 110, 116, 114, 34, 44, 34, 95, 108, 111, 103, 103, 101, 114, 95, 110, 97, 109, 101, 34, 58, 34, 97, 106, 111, 117, 114, 46, 65, 106, 111, 117, 114, 83, 101, 99, 117, 114, 105, 116, 121, 83, 117, 99, 99, 101, 115, 115, 69, 118, 101, 110, 116, 76, 105, 115, 116, 101, 110, 101, 114, 34, 44, 34, 95, 115, 101, 114, 118, 101, 114, 95, 117, 114, 108]
|
||||
byte[] chunk2 = [30, 15, -103, 51, -96, -10, -51, -31, 39, -41, 1, 2, 34, 58, 34, 104, 116, 116, 112, 115, 58, 92, 47, 92, 47, 100, 107, 46, 109, 105, 110, 116, 114, 46, 97, 112, 112, 34, 44, 34, 95, 97, 112, 112, 95, 101, 110, 118, 34, 58, 34, 112, 114, 111, 100, 34, 44, 34, 95, 116, 104, 114, 101, 97, 100, 95, 110, 97, 109, 101, 34, 58, 34, 104, 116, 116, 112, 45, 110, 105, 111, 45, 56, 48, 56, 48, 45, 101, 120, 101, 99, 45, 52, 34, 125]
|
||||
|
||||
when:
|
||||
syslogParser.parse(chunk1)
|
||||
SyslogMessage msg = syslogParser.parse(chunk2)
|
||||
|
||||
then:
|
||||
msg.message == "fobar"
|
||||
msg.message == "event=AuthenticationSuccessEvent username=mark.nellemann@gmail.com tenant=2 remoteAddress=185.181.223.117 sessionId=null"
|
||||
|
||||
}
|
||||
|
||||
|
||||
void "chunked GELF unordered message"() {
|
||||
|
||||
setup:
|
||||
byte[] chunk1 = [30, 15, -103, 51, -96, -10, -51, -31, 39, -41, 0, 2, 123, 34, 118, 101, 114, 115, 105, 111, 110, 34, 58, 34, 49, 46, 49, 34, 44, 34, 104, 111, 115, 116, 34, 58, 34, 105, 112, 45, 49, 48, 45, 49, 45, 49, 48, 49, 45, 49, 52, 46, 101, 117, 45, 99, 101, 110, 116, 114, 97, 108, 45, 49, 46, 99, 111, 109, 112, 117, 116, 101, 46, 105, 110, 116, 101, 114, 110, 97, 108, 34, 44, 34, 115, 104, 111, 114, 116, 95, 109, 101, 115, 115, 97, 103, 101, 34, 58, 34, 101, 118, 101, 110, 116, 61, 65, 117, 116, 104, 101, 110, 116, 105, 99, 97, 116, 105, 111, 110, 83, 117, 99, 99, 101, 115, 115, 69, 118, 101, 110, 116, 32, 117, 115, 101, 114, 110, 97, 109, 101, 61, 109, 97, 114, 107, 46, 110, 101, 108, 108, 101, 109, 97, 110, 110, 64, 103, 109, 97, 105, 108, 46, 99, 111, 109, 32, 116, 101, 110, 97, 110, 116, 61, 50, 32, 114, 101, 109, 111, 116, 101, 65, 100, 100, 114, 101, 115, 115, 61, 49, 56, 53, 46, 49, 56, 49, 46, 50, 50, 51, 46, 49, 49, 55, 32, 115, 101, 115, 115, 105, 111, 110, 73, 100, 61, 110, 117, 108, 108, 34, 44, 34, 102, 117, 108, 108, 95, 109, 101, 115, 115, 97, 103, 101, 34, 58, 34, 101, 118, 101, 110, 116, 61, 65, 117, 116, 104, 101, 110, 116, 105, 99, 97, 116, 105, 111, 110, 83, 117, 99, 99, 101, 115, 115, 69, 118, 101, 110, 116, 32, 117, 115, 101, 114, 110, 97, 109, 101, 61, 109, 97, 114, 107, 46, 110, 101, 108, 108, 101, 109, 97, 110, 110, 64, 103, 109, 97, 105, 108, 46, 99, 111, 109, 32, 116, 101, 110, 97, 110, 116, 61, 50, 32, 114, 101, 109, 111, 116, 101, 65, 100, 100, 114, 101, 115, 115, 61, 49, 56, 53, 46, 49, 56, 49, 46, 50, 50, 51, 46, 49, 49, 55, 32, 115, 101, 115, 115, 105, 111, 110, 73, 100, 61, 110, 117, 108, 108, 92, 110, 34, 44, 34, 116, 105, 109, 101, 115, 116, 97, 109, 112, 34, 58, 49, 54, 55, 49, 48, 48, 48, 53, 53, 54, 46, 56, 48, 50, 44, 34, 108, 101, 118, 101, 108, 34, 58, 54, 44, 34, 95, 97, 112, 112, 95, 99, 111, 117, 110, 116, 114, 121, 34, 58, 34, 100, 107, 34, 44, 34, 95, 97, 112, 112, 95, 110, 97, 109, 101, 34, 58, 34, 109, 105, 110, 116, 114, 34, 44, 34, 95, 108, 111, 103, 103, 101, 114, 95, 110, 97, 109, 101, 34, 58, 34, 97, 106, 111, 117, 114, 46, 65, 106, 111, 117, 114, 83, 101, 99, 117, 114, 105, 116, 121, 83, 117, 99, 99, 101, 115, 115, 69, 118, 101, 110, 116, 76, 105, 115, 116, 101, 110, 101, 114, 34, 44, 34, 95, 115, 101, 114, 118, 101, 114, 95, 117, 114, 108]
|
||||
byte[] chunk2 = [30, 15, -103, 51, -96, -10, -51, -31, 39, -41, 1, 2, 34, 58, 34, 104, 116, 116, 112, 115, 58, 92, 47, 92, 47, 100, 107, 46, 109, 105, 110, 116, 114, 46, 97, 112, 112, 34, 44, 34, 95, 97, 112, 112, 95, 101, 110, 118, 34, 58, 34, 112, 114, 111, 100, 34, 44, 34, 95, 116, 104, 114, 101, 97, 100, 95, 110, 97, 109, 101, 34, 58, 34, 104, 116, 116, 112, 45, 110, 105, 111, 45, 56, 48, 56, 48, 45, 101, 120, 101, 99, 45, 52, 34, 125]
|
||||
|
||||
when:
|
||||
syslogParser.parse(chunk2)
|
||||
SyslogMessage msg = syslogParser.parse(chunk1)
|
||||
|
||||
then:
|
||||
msg.message == "event=AuthenticationSuccessEvent username=mark.nellemann@gmail.com tenant=2 remoteAddress=185.181.223.117 sessionId=null"
|
||||
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -39,4 +39,14 @@ class JsonUtilTest extends Specification {
|
|||
result == 'here it comes " to wreck the day...'
|
||||
}
|
||||
|
||||
def "test newline decode"() {
|
||||
setup:
|
||||
def testQuote = 'here it comes \n to wreck the day...'
|
||||
|
||||
when:
|
||||
def result = JsonUtil.decode(testQuote)
|
||||
|
||||
then:
|
||||
result == 'here it comes \n to wreck the day...'
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue