From 0752c0b6a64a9a272dd26a1b37bef57dbaad4102 Mon Sep 17 00:00:00 2001 From: Mark Nellemann Date: Wed, 27 Jan 2021 01:29:38 +0100 Subject: [PATCH] Improve rfc5424 parsing. --- build.gradle | 3 +- doc/syslogd.service | 2 +- .../biz/nellemann/syslogd/Application.java | 4 +- .../biz/nellemann/syslogd/SyslogMessage.java | 8 +- .../biz/nellemann/syslogd/SyslogParser.java | 46 ++++++-- .../biz/nellemann/syslogd/SyslogPrinter.java | 20 +++- .../nellemann/syslogd/SyslogParserTest.groovy | 100 +++++++++++++++++- src/test/resources/simplelogger.properties | 6 ++ 8 files changed, 169 insertions(+), 20 deletions(-) create mode 100644 src/test/resources/simplelogger.properties diff --git a/build.gradle b/build.gradle index f68bf6c..3e86756 100644 --- a/build.gradle +++ b/build.gradle @@ -18,7 +18,8 @@ dependencies { implementation 'org.slf4j:slf4j-simple:1.7.30' testImplementation('org.spockframework:spock-core:2.0-M4-groovy-3.0') - testImplementation("org.slf4j:slf4j-simple:1.7.30") + testImplementation 'org.slf4j:slf4j-api:1.7.30' + testRuntime("org.slf4j:slf4j-simple:1.7.30") } application { diff --git a/doc/syslogd.service b/doc/syslogd.service index 2520742..6b4598d 100644 --- a/doc/syslogd.service +++ b/doc/syslogd.service @@ -4,7 +4,7 @@ Description=Simple Syslog Service [Service] TimeoutStartSec=0 Restart=always -ExecStart=/opt/syslogd/bin/syslogd --no-stdout --forward localhost:1514 +ExecStart=/opt/syslogd/bin/syslogd --no-stdout --forward=localhost:1514 [Install] WantedBy=default.target diff --git a/src/main/java/biz/nellemann/syslogd/Application.java b/src/main/java/biz/nellemann/syslogd/Application.java index 1eaf712..d5f38fd 100644 --- a/src/main/java/biz/nellemann/syslogd/Application.java +++ b/src/main/java/biz/nellemann/syslogd/Application.java @@ -53,7 +53,7 @@ public class Application implements Callable, LogListener { @CommandLine.Option(names = "--rfc5424", description = "Parse RFC-5424 messages [default: RFC-3164].", defaultValue = "false") private boolean rfc5424; - @CommandLine.Option(names = { "-f", "--forward"}, description = "Forward to UDP host[:port] (RFC-3164).", paramLabel = "") + @CommandLine.Option(names = { "-f", "--forward"}, description = "Forward to UDP host[:port] (RFC-5424).", paramLabel = "") private String forward; @CommandLine.Option(names = { "-d", "--debug" }, description = "Enable debugging [default: 'false'].") @@ -131,7 +131,7 @@ public class Application implements Callable, LogListener { if(doForward) { try { - udpClient.send(SyslogPrinter.toRfc3164(msg)); + udpClient.send(SyslogPrinter.toRfc5424(msg)); } catch (Exception e) { e.printStackTrace(); } diff --git a/src/main/java/biz/nellemann/syslogd/SyslogMessage.java b/src/main/java/biz/nellemann/syslogd/SyslogMessage.java index 28c13f9..5dfa80a 100644 --- a/src/main/java/biz/nellemann/syslogd/SyslogMessage.java +++ b/src/main/java/biz/nellemann/syslogd/SyslogMessage.java @@ -32,16 +32,16 @@ public class SyslogMessage { protected String hostname; // The APP-NAME field SHOULD identify the device or application that originated the message. - protected String application; + protected String application = "-"; // The PROCID field is often used to provide the process name or process ID associated with a syslog system. - protected String processId; + protected String processId = "-"; // The MSGID SHOULD identify the type of message. - protected String messageId; + protected String messageId = "-"; // STRUCTURED-DATA provides a mechanism to express information in a well defined, easily parseable and interpretable data format. - protected String structuredData; + protected String structuredData = "-"; // The MSG part contains a free-form message that provides information about the event. protected final String message; diff --git a/src/main/java/biz/nellemann/syslogd/SyslogParser.java b/src/main/java/biz/nellemann/syslogd/SyslogParser.java index 93b1cc3..52271a8 100644 --- a/src/main/java/biz/nellemann/syslogd/SyslogParser.java +++ b/src/main/java/biz/nellemann/syslogd/SyslogParser.java @@ -18,9 +18,13 @@ package biz.nellemann.syslogd; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.text.ParseException; +import java.text.SimpleDateFormat; import java.time.*; import java.time.format.DateTimeFormatter; import java.time.format.DateTimeParseException; +import java.util.Arrays; +import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -82,11 +86,15 @@ public class SyslogParser { */ public static SyslogMessage parseRfc5424(final String input) throws NumberFormatException { - Pattern pattern = Pattern.compile("^<(\\d{1,3})>(\\d+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\[.*\\])\\s+(\\S+)", Pattern.CASE_INSENSITIVE); + log.warn("parseRfc5424() " + input); + + // "<165>1 2003-08-24T05:14:15.000003-07:00 192.0.2.1 myproc 8710 - - %% It's time to make the do-nuts." + // '<13>1 2020-09-23T08:57:30.950699+02:00 xps13 mark - - [exampleSDID@32473 iut="3" eventSource="Application" eventID="1011"] adfdfdf3432434565656' + Pattern pattern = Pattern.compile("^<(\\d{1,3})>(\\d+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\[.*\\]|-)\\s+(\\S+)", Pattern.CASE_INSENSITIVE); Matcher matcher = pattern.matcher(input); boolean matchFound = matcher.find(); if(!matchFound) { - //log.warn("parseRfc5424() - Match not found in: " + input); + log.debug("parseRfc5424() - Match not found in: " + input); System.err.println("!" + input); return null; } @@ -116,7 +124,8 @@ public class SyslogParser { syslogMessage.processId = procId; if(msgId != null && !msgId.equals("-")) syslogMessage.messageId = msgId; - syslogMessage.structuredData = data; + if(data != null && !data.equals("-")) + syslogMessage.structuredData = data; return syslogMessage; } @@ -154,15 +163,40 @@ public class SyslogParser { */ static protected Instant parseRfc5424Timestamp(String dateString) { - Instant instant = null; + /* + https://docs.oracle.com/javase/8/docs/api/java/time/format/DateTimeFormatter.html + + ex1: 1985-04-12T23:20:50.52Z + ex2: 1985-04-12T19:20:50.52-04:00 + ex3: 2003-10-11T22:14:15.003Z + ex4: 2003-08-24T05:14:15.000003-07:00 + ex5: 2003-08-24T05:14:15.000000003-07:00 + */ + + List formatStrings = Arrays.asList( + "yyyy-MM-dd'T'HH:mm:ss.SS'Z'", + "yyyy-MM-dd'T'HH:mm:ss.SSXXX", + "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", + "yyyy-MM-dd'T'HH:mm:ss.SSSSSSXXX", + "yyyy-MM-dd'T'HH:mm:ss.SSSSSSSSSXXX" + ); + + for(String formatString : formatStrings) + { + try { + return new SimpleDateFormat(formatString).parse(dateString).toInstant(); + } + catch (ParseException e) {} + } + /* try { DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ISO_OFFSET_DATE_TIME; instant = Instant.from(dateTimeFormatter.parse(dateString)); } catch(DateTimeParseException e) { log.error("parseTimestamp()", e); } - - return instant; + return instant;*/ + return null; } diff --git a/src/main/java/biz/nellemann/syslogd/SyslogPrinter.java b/src/main/java/biz/nellemann/syslogd/SyslogPrinter.java index 6004afc..da6be63 100644 --- a/src/main/java/biz/nellemann/syslogd/SyslogPrinter.java +++ b/src/main/java/biz/nellemann/syslogd/SyslogPrinter.java @@ -1,7 +1,12 @@ package biz.nellemann.syslogd; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + public class SyslogPrinter { + private final static Logger log = LoggerFactory.getLogger(SyslogPrinter.class); + public static String toString(SyslogMessage msg) { StringBuilder sb = new StringBuilder(); sb.append(msg.timestamp.toString()); @@ -44,23 +49,28 @@ public class SyslogPrinter { sb.append(" " + msg.application); sb.append(": " + msg.message); + log.debug(sb.toString()); return sb.toString(); } // <13>1 2020-09-23T08:57:30.950699+02:00 xps13 mark - - [timeQuality tzKnown="1" isSynced="1" syncAccuracy="125500"] adfdfdf3432434565656 + // <34>1 2003-10-11T22:14:15.003Z mymachine.example.com su - ID47 - BOM'su root' failed for lonvick on /dev/pts/8 public static String toRfc5424(SyslogMessage msg) { StringBuilder sb = new StringBuilder(); - sb.append(getPri(msg.facility, msg.severity)); - sb.append("1"); // Version - sb.append(" " + new java.text.SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'").format(new java.util.Date(msg.timestamp.toEpochMilli()))); + sb.append(getPri(msg.facility, msg.severity)).append("1"); + sb.append(" " + new java.text.SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'").format(new java.util.Date(msg.timestamp.toEpochMilli()))); sb.append(" " + msg.hostname); sb.append(" " + msg.application); - sb.append(": " + msg.message); - + sb.append(" " + msg.processId); + sb.append(" " + msg.messageId); + sb.append(" " + msg.structuredData); + sb.append(" " + msg.message); + log.debug(sb.toString()); return sb.toString(); } + static private String getPri(Facility facility, Severity severity) { int prival = (facility.toNumber() * 8) + severity.toNumber(); return String.format("<%d>", prival); diff --git a/src/test/groovy/biz/nellemann/syslogd/SyslogParserTest.groovy b/src/test/groovy/biz/nellemann/syslogd/SyslogParserTest.groovy index d1d2a0c..ff35567 100644 --- a/src/test/groovy/biz/nellemann/syslogd/SyslogParserTest.groovy +++ b/src/test/groovy/biz/nellemann/syslogd/SyslogParserTest.groovy @@ -1,5 +1,6 @@ package biz.nellemann.syslogd +import spock.lang.Ignore import spock.lang.Specification import java.time.Instant import java.time.OffsetDateTime; @@ -9,15 +10,49 @@ class SyslogParserTest extends Specification { void "test rfc5424 message"() { setup: - def input = "<13>1 2020-09-23T08:57:30.950699+02:00 xps13 mark - - [timeQuality tzKnown=\"1\" isSynced=\"1\" syncAccuracy=\"125500\"] adfdfdf3432434565656" + def input = '<13>1 2020-09-23T08:57:30.950699+02:00 xps13 mark - - [exampleSDID@32473 iut="3" eventSource="Application" eventID="1011"] adfdfdf3432434565656' when: SyslogMessage msg = SyslogParser.parseRfc5424(input) then: msg.message == "adfdfdf3432434565656" + msg.processId == "-" } + void "test rfc5424 example message"() { + + setup: + def input = "<34>1 2003-10-11T22:14:15.003Z mymachine.example.com su - ID47 - BOM'su root' failed for lonvick on /dev/pts/8" + + when: + SyslogMessage msg = SyslogParser.parseRfc5424(input) + + then: + msg.hostname == "mymachine.example.com" + msg.application == "su" + msg.processId == "-" + msg.messageId == "ID47" + msg.structuredData == "-" + } + + void "test rfc5424 example2 message"() { + + setup: + def input = "<165>1 2003-08-24T05:14:15.000003-07:00 192.0.2.1 myproc 8710 - - %% It's time to make the do-nuts." + + when: + SyslogMessage msg = SyslogParser.parseRfc5424(input) + + then: + msg.hostname == "192.0.2.1" + msg.application == "myproc" + msg.processId == "8710" + msg.messageId == "-" + msg.structuredData == "-" + } + + void "test rfc3164 aix/vios message"() { setup: @@ -109,6 +144,69 @@ class SyslogParserTest extends Specification { inst.toString() == "${odt.getYear()}-09-12T20:50:13Z" } + void "test parseRfc5424Timestamp ex1"() { + setup: + String dateString = "1985-04-12T23:20:50.52Z" + + when: + Instant inst = SyslogParser.parseRfc5424Timestamp(dateString) + + then: + inst.toString() == "1985-04-12T21:20:50.052Z" + inst.toEpochMilli() == 482188850052 + } + + void "test parseRfc5424Timestamp ex2"() { + setup: + String dateString = "1985-04-12T19:20:50.52-04:00" + + when: + Instant inst = SyslogParser.parseRfc5424Timestamp(dateString) + + then: + inst.toString() == "1985-04-12T23:20:50.052Z" + inst.toEpochMilli() == 482196050052 + } + + void "test parseRfc5424Timestamp ex3"() { + setup: + String dateString = "2003-10-11T22:14:15.003Z" + + when: + Instant inst = SyslogParser.parseRfc5424Timestamp(dateString) + + then: + inst.toString() == "2003-10-11T20:14:15.003Z" + inst.toEpochMilli() == 1065903255003 + } + + void "test parseRfc5424Timestamp ex4"() { + setup: + String dateString = "2003-08-24T05:14:15.000003-07:00" + + when: + Instant inst = SyslogParser.parseRfc5424Timestamp(dateString) + + then: + inst.toString() == "2003-08-24T12:14:15.003Z" + inst.toEpochMilli() == 1061727255003 + } + + void "test parseRfc5424Timestamp ex5"() { + setup: + String dateString = "2003-08-24T05:14:15.000000003-07:00" + + when: + Instant inst = SyslogParser.parseRfc5424Timestamp(dateString) + + then: + inst.toString() == "2003-08-24T12:14:15.003Z" + inst.toEpochMilli() == 1061727255003 + } + + + + @Ignore void "test parseRfc5424Timestamp"() { setup: diff --git a/src/test/resources/simplelogger.properties b/src/test/resources/simplelogger.properties new file mode 100644 index 0000000..876e5b5 --- /dev/null +++ b/src/test/resources/simplelogger.properties @@ -0,0 +1,6 @@ +org.slf4j.simpleLogger.logFile=System.err +org.slf4j.simpleLogger.showDateTime=false +org.slf4j.simpleLogger.showShortLogName=true +org.slf4j.simpleLogger.dateTimeFormat=yyyy-MM-dd HH:mm:ss.SSS +org.slf4j.simpleLogger.levelInBrackets=true +org.slf4j.simpleLogger.defaultLogLevel=debug