Merge pull request #17 from Graylog2/additional-syslog-events

Support FortiGate and Cisco syslog events
This commit is contained in:
Dennis Oelkers 2017-03-30 11:49:06 +02:00 committed by GitHub
commit 3b24718b1f
6 changed files with 503 additions and 7 deletions

View File

@ -2,9 +2,6 @@ sudo: required
dist: trusty
language: java
jdk:
# - openjdk6
# - openjdk7
- oraclejdk7
- oraclejdk8
addons:
# Fix OpenJDK builds

12
pom.xml
View File

@ -49,6 +49,8 @@
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<maven.compiler.source>1.8</maven.compiler.source>
<maven.compiler.target>1.8</maven.compiler.target>
</properties>
<distributionManagement>
@ -105,6 +107,12 @@
<version>2.4.0</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.assertj</groupId>
<artifactId>assertj-core</artifactId>
<version>3.6.2</version>
<scope>test</scope>
</dependency>
</dependencies>
<build>
@ -120,10 +128,6 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.5.1</version>
<configuration>
<source>1.6</source>
<target>1.6</target>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>

View File

@ -0,0 +1,157 @@
package org.graylog2.syslog4j.server.impl.event;
import java.net.InetAddress;
import java.time.ZoneOffset;
import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeParseException;
import java.util.Calendar;
import java.util.Date;
import java.util.Locale;
/**
* CiscoSyslogServerEvent provides an implementation of the
* SyslogServerEventIF interface that supports receiving of
* Cisco syslog messages.
*
* @see <a href="http://www.ciscopress.com/articles/article.asp?p=426638">An Overview of the syslog Protocol</a>
*/
public class CiscoSyslogServerEvent extends SyslogServerEvent {
private static final DateTimeFormatter DEFAULT_FORMATTER =
DateTimeFormatter
.ofPattern("yyyy MMM ppd HH:mm:ss[.SSS][ zzz]", Locale.ROOT)
.withZone(ZoneOffset.UTC);
private int sequenceNumber = 0;
public CiscoSyslogServerEvent(final byte[] message, int length, InetAddress inetAddress) {
super();
initialize(message, length, inetAddress);
parse();
}
public CiscoSyslogServerEvent(final String message, InetAddress inetAddress) {
super();
initialize(message, inetAddress);
parse();
}
@Override
protected void parsePriority() {
if (this.message.charAt(0) == '<') {
int i = this.message.indexOf(">");
if (i <= 4 && i > -1) {
String priorityStr = this.message.substring(1, i);
int priority = 0;
try {
priority = Integer.parseInt(priorityStr);
this.facility = priority >> 3;
this.level = priority - (this.facility << 3);
this.message = this.message.substring(i + 1);
parseSequenceNumber();
parseDate();
} catch (NumberFormatException nfe) {
//
}
parseHost();
}
}
}
private void parseSequenceNumber() {
int i = this.message.indexOf(':');
if (i > -1) {
String sequenceNumberStr = this.message.substring(0, i);
if (sequenceNumberStr.isEmpty()) {
this.message = this.message.substring(i + 1);
} else {
try {
this.sequenceNumber = Integer.parseInt(sequenceNumberStr);
this.message = this.message.substring(i + 1);
} catch (NumberFormatException nfe) {
//
}
}
}
}
@Override
protected void parseHost() {
if (message.indexOf('%') < 1) {
this.host = "";
} else {
int i = this.message.indexOf(' ');
if (i > -1) {
this.host = this.message.substring(0, i).trim();
// Skip ' ' and ':' characters
char c = message.charAt(i);
while (c == ' ' || c == ':') {
c = message.charAt(++i);
}
this.message = this.message.substring(i);
}
}
}
@Override
protected void parseDate() {
// Skip leading spaces
while (message.charAt(0) == ' ') {
message = message.substring(1);
}
// Skip leading asterisk
if (message.charAt(0) == '*') {
message = message.substring(1);
}
int dateLength = message.indexOf(": ");
if (this.message.length() > dateLength) {
boolean isYearMissing = Character.isLetter(message.charAt(0));
String originalDate = this.message.substring(0, dateLength);
DateTimeFormatter formatter = DEFAULT_FORMATTER;
// Hacky override for: "Mar 06 2016 12:53:10 DEVICENAME :"
if (Character.isDigit(message.charAt(7))
&& Character.isDigit(message.charAt(8))
&& Character.isDigit(message.charAt(9))
&& Character.isDigit(message.charAt(10))) {
dateLength = 20;
isYearMissing = false;
originalDate = this.message.substring(0, dateLength);
formatter = DateTimeFormatter
.ofPattern("MMM ppd yyyy HH:mm:ss", Locale.ROOT)
.withZone(ZoneOffset.UTC);
}
try {
if (isYearMissing) {
String year = Integer.toString(Calendar.getInstance().get(Calendar.YEAR));
String modifiedDate = year + " " + originalDate;
final ZonedDateTime dateTime = ZonedDateTime.parse(modifiedDate, formatter);
this.date = Date.from(dateTime.toInstant());
} else {
final ZonedDateTime dateTime = ZonedDateTime.parse(originalDate, formatter);
this.date = Date.from(dateTime.toInstant());
}
this.message = this.message.substring(dateLength + 1);
} catch (DateTimeParseException pe) {
this.date = new Date();
}
}
}
public int getSequenceNumber() {
return sequenceNumber;
}
}

View File

@ -0,0 +1,176 @@
package org.graylog2.syslog4j.server.impl.event;
import org.graylog2.syslog4j.server.SyslogServerEventIF;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.time.LocalDate;
import java.time.LocalTime;
import java.time.ZoneOffset;
import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import static java.util.Objects.requireNonNull;
/**
* FortiGateSyslogEvent provides an implementation of the
* SyslogServerEventIF interface that supports receiving of
* FortiGate/FortiOS syslog messages.
*
* @see <a href="http://help.fortinet.com/fos50hlp/54/Content/FortiOS/fortigate-logging-reporting-54/logs.htm#Log_messages">FortiGate logging and reporting overview</a>
*/
public class FortiGateSyslogEvent implements SyslogServerEventIF {
private static final Pattern PRI_PATTERN = Pattern.compile("^<(\\d{1,3})>(.*)$");
private static final Pattern KV_PATTERN = Pattern.compile("(\\w+)=([^\\s\"]*)");
private static final Pattern QUOTED_KV_PATTERN = Pattern.compile("(\\w+)=\"([^\"]*)\"");
private final String rawEvent;
private Date date;
private int facility;
private int level;
private String host;
private String message;
private Charset charSet = StandardCharsets.UTF_8;
private Map<String, String> fields = Collections.emptyMap();
public FortiGateSyslogEvent(final String rawEvent) {
this.rawEvent = requireNonNull(rawEvent, "rawEvent");
parse(rawEvent);
}
private void parse(String event) {
final Matcher matcher = PRI_PATTERN.matcher(event);
if (!matcher.find()) {
throw new IllegalArgumentException("Invalid Fortigate syslog message");
} else {
final String priority = matcher.group(1);
final String message = matcher.group(2);
parsePriority(priority);
setMessage(message);
parseFields(message);
parseDate(fields.get("date"), fields.get("time"));
setHost(fields.get("devname"));
}
}
private void parsePriority(String priorityString) {
try {
final int priority = Integer.parseInt(priorityString);
setFacility(priority / 8);
setLevel(priority % 8);
} catch (NumberFormatException e) {
throw new IllegalArgumentException("Couldn't parse message priority", e);
}
}
private void parseFields(String event) {
final Map<String, String> fields = new HashMap<>();
final Matcher matcher = KV_PATTERN.matcher(event);
while (matcher.find()) {
fields.put(matcher.group(1), matcher.group(2));
}
final Matcher quotedMatcher = QUOTED_KV_PATTERN.matcher(event);
while (quotedMatcher.find()) {
fields.put(quotedMatcher.group(1), quotedMatcher.group(2));
}
setFields(fields);
}
private void parseDate(String date, String time) {
if (date != null && time != null) {
final ZonedDateTime dateTime = ZonedDateTime.of(
LocalDate.parse(date, DateTimeFormatter.ISO_LOCAL_DATE.withZone(ZoneOffset.UTC)),
LocalTime.parse(time, DateTimeFormatter.ISO_LOCAL_TIME.withZone(ZoneOffset.UTC)),
ZoneOffset.UTC);
setDate(Date.from(dateTime.toInstant()));
} else {
setDate(new Date());
}
}
@Override
public byte[] getRaw() {
return rawEvent.getBytes(charSet);
}
@Override
public int getFacility() {
return facility;
}
@Override
public void setFacility(int facility) {
this.facility = requireNonNull(facility, "facility");
}
@Override
public Date getDate() {
return date;
}
@Override
public void setDate(Date date) {
this.date = requireNonNull(date, "date");
}
@Override
public int getLevel() {
return level;
}
@Override
public void setLevel(int level) {
this.level = requireNonNull(level, "level");
}
@Override
public String getHost() {
return host;
}
@Override
public void setHost(String host) {
this.host = host;
}
@Override
public boolean isHostStrippedFromMessage() {
return false;
}
@Override
public String getMessage() {
return message;
}
@Override
public void setMessage(String message) {
this.message = requireNonNull(message, "message");
}
@Override
public String getCharSet() {
return charSet.name();
}
@Override
public void setCharSet(String charSet) {
this.charSet = Charset.forName(charSet);
}
public Map<String, String> getFields() {
return Collections.unmodifiableMap(fields);
}
public void setFields(Map<String, String> fields) {
this.fields = requireNonNull(fields, "fields");
}
}

View File

@ -0,0 +1,128 @@
package org.graylog2.syslog4j.server.impl.event;
import org.junit.Test;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.time.ZoneId;
import java.time.ZonedDateTime;
import java.util.Date;
import static java.time.ZoneOffset.UTC;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.Assert.assertEquals;
public class CiscoSyslogServerEventTest {
private static final InetAddress INET_ADDR = new InetSocketAddress(514).getAddress();
private static final ZoneId CET = ZoneId.of("CET");
private static final int YEAR = ZonedDateTime.now().getYear();
private CiscoSyslogServerEvent buildEvent(String message) {
return new CiscoSyslogServerEvent(message, INET_ADDR);
}
@Test
public void testCisco1() throws Exception {
final String message = "<166>Mar 06 2016 12:53:10 DEVICENAME : %ASA-6-302013: Built inbound TCP connection 723494125 for FRONTEND:IP/11288 (IP/11288) to BACKEND:IP/27180 (IP/27180)";
final CiscoSyslogServerEvent event = buildEvent(message);
assertThat(toZonedDateTime(event.getDate(), UTC)).isEqualTo(ZonedDateTime.of(2016, 3, 6, 12, 53, 10, 0, UTC));
assertThat(event.getFacility()).isEqualTo(20);
assertThat(event.getLevel()).isEqualTo(6);
assertThat(event.getSequenceNumber()).isEqualTo(0);
assertThat(event.getHost()).isEqualTo("DEVICENAME");
assertThat(event.getMessage()).isEqualTo("%ASA-6-302013: Built inbound TCP connection 723494125 for FRONTEND:IP/11288 (IP/11288) to BACKEND:IP/27180 (IP/27180)");
}
@Test
public void testCisco2() throws Exception {
final String message = "<186>1541800: Feb 27 06:08:59.485: %HARDWARE-2-FAN_ERROR: Fan Failure";
final CiscoSyslogServerEvent event = buildEvent(message);
assertThat(toZonedDateTime(event.getDate(), UTC)).isEqualTo(ZonedDateTime.of(YEAR, 2, 27, 6, 8, 59, 485_000_000, UTC));
assertThat(event.getFacility()).isEqualTo(23);
assertThat(event.getLevel()).isEqualTo(2);
assertThat(event.getSequenceNumber()).isEqualTo(1541800);
assertThat(event.getHost()).isEmpty();
assertThat(event.getMessage()).isEqualTo("%HARDWARE-2-FAN_ERROR: Fan Failure");
}
@Test
public void testCisco3() throws Exception {
final String message = "<187>148094: Feb 27 06:07:29.716: %LINK-3-UPDOWN: Interface GigabitEthernet1/0/15, changed state to down";
final CiscoSyslogServerEvent event = buildEvent(message);
assertThat(toZonedDateTime(event.getDate(), UTC)).isEqualTo(ZonedDateTime.of(YEAR, 2, 27, 6, 7, 29, 716_000_000, UTC));
assertThat(event.getFacility()).isEqualTo(23);
assertThat(event.getLevel()).isEqualTo(3);
assertThat(event.getSequenceNumber()).isEqualTo(148094);
assertThat(event.getHost()).isEmpty();
assertEquals("%LINK-3-UPDOWN: Interface GigabitEthernet1/0/15, changed state to down", event.getMessage());
}
@Test
public void testCisco4() throws Exception {
final String message = "<190>530470: *Sep 28 17:13:35.098: %SEC-6-IPACCESSLOGP: list MGMT_IN denied udp IP(49964) -> IP(161), 11 packets";
final CiscoSyslogServerEvent event = buildEvent(message);
assertThat(toZonedDateTime(event.getDate(), UTC)).isEqualTo(ZonedDateTime.of(YEAR, 9, 28, 17, 13, 35, 98_000_000, UTC));
assertThat(event.getFacility()).isEqualTo(23);
assertThat(event.getLevel()).isEqualTo(6);
assertThat(event.getSequenceNumber()).isEqualTo(530470);
assertThat(event.getHost()).isEmpty();
assertEquals("%SEC-6-IPACCESSLOGP: list MGMT_IN denied udp IP(49964) -> IP(161), 11 packets", event.getMessage());
}
@Test
public void testCisco5() throws Exception {
final String message = "<190>: 2016 Mar 06 09:22:34 CET: %AUTHPRIV-6-SYSTEM_MSG: START: rsync pid=4311 from=::ffff:IP - xinetd[6219]";
final CiscoSyslogServerEvent event = buildEvent(message);
assertThat(toZonedDateTime(event.getDate(), CET)).isEqualTo(ZonedDateTime.of(2016, 3, 6, 9, 22, 34, 0, CET));
assertThat(event.getFacility()).isEqualTo(23);
assertThat(event.getLevel()).isEqualTo(6);
assertThat(event.getSequenceNumber()).isEqualTo(0);
assertThat(event.getHost()).isEmpty();
assertThat(event.getMessage()).isEqualTo("%AUTHPRIV-6-SYSTEM_MSG: START: rsync pid=4311 from=::ffff:IP - xinetd[6219]");
}
@Test
public void testCisco6() throws Exception {
final String message = "<134>: 2016 Mar 6 12:53:10 UTC: %POLICY_ENGINE-6-POLICY_LOOKUP_EVENT: policy=POLICYNAME rule=RULENAME action=Permit direction=egress src.net.ip-address=IP src.net.port=38321 dst.net.ip-address=IP dst.net.port=5666 net.protocol=6 net.ethertype=800 net.service=\"protocol 6 port 5666\"";
final CiscoSyslogServerEvent event = buildEvent(message);
assertThat(toZonedDateTime(event.getDate(), UTC)).isEqualTo(ZonedDateTime.of(2016, 3, 6, 12, 53, 10, 0, UTC));
assertThat(event.getFacility()).isEqualTo(16);
assertThat(event.getLevel()).isEqualTo(6);
assertThat(event.getSequenceNumber()).isEqualTo(0);
assertThat(event.getHost()).isEmpty();
assertThat(event.getMessage()).isEqualTo("%POLICY_ENGINE-6-POLICY_LOOKUP_EVENT: policy=POLICYNAME rule=RULENAME action=Permit direction=egress src.net.ip-address=IP src.net.port=38321 dst.net.ip-address=IP dst.net.port=5666 net.protocol=6 net.ethertype=800 net.service=\"protocol 6 port 5666\"");
}
@Test
public void testCisco7() throws Exception {
final String message = "<166>%ASA-6-302015: Built inbound UDP connection 23631055 for inside:192.168.19.91/44764 (192.168.19.91/44764) to identity:192.168.249.33/161 (192.168.249.33/161)";
final CiscoSyslogServerEvent event = buildEvent(message);
assertThat(event.getDate())
.isInThePast()
.isInSameMinuteWindowAs(new Date());
assertThat(event.getFacility()).isEqualTo(20);
assertThat(event.getLevel()).isEqualTo(6);
assertThat(event.getSequenceNumber()).isEqualTo(0);
assertThat(event.getHost()).isEmpty();
assertThat(event.getMessage()).isEqualTo("%ASA-6-302015: Built inbound UDP connection 23631055 for inside:192.168.19.91/44764 (192.168.19.91/44764) to identity:192.168.249.33/161 (192.168.249.33/161)");
}
private ZonedDateTime toZonedDateTime(Date date, ZoneId zoneId) {
return ZonedDateTime.ofInstant(date.toInstant(), zoneId);
}
}

View File

@ -0,0 +1,34 @@
package org.graylog2.syslog4j.server.impl.event;
import org.junit.Test;
import java.nio.charset.StandardCharsets;
import java.time.ZoneOffset;
import java.time.ZonedDateTime;
import static org.assertj.core.api.Assertions.assertThat;
public class FortiGateSyslogEventTest {
@Test
public void testFortiGateMessage() {
final String rawMessage = "<45>date=2017-03-06 time=12:53:10 devname=DEVICENAME devid=DEVICEID logid=0000000013 type=traffic subtype=forward level=notice vd=ALIAS srcip=IP srcport=45748 srcintf=\"IF\" dstip=IP dstport=443 dstintf=\"IF\" sessionid=1122686199 status=close policyid=77 dstcountry=\"COUNTRY\" srccountry=\"COUNTRY\" trandisp=dnat tranip=IP tranport=443 service=HTTPS proto=6 appid=41540 app=\"SSL_TLSv1.2\" appcat=\"Network.Service\" applist=\"ACLNAME\" appact=detected duration=1 sentbyte=2313 rcvdbyte=14883 sentpkt=19 rcvdpkt=19 utmaction=passthrough utmevent=app-ctrl attack=\"SSL\" hostname=\"HOSTNAME\" custom=\"white space\"";
final FortiGateSyslogEvent event = new FortiGateSyslogEvent(rawMessage);
assertThat(event).isNotNull();
assertThat(event.getFacility()).isEqualTo(5);
assertThat(event.getLevel()).isEqualTo(5);
assertThat(event.getHost()).isEqualTo("DEVICENAME");
assertThat(ZonedDateTime.ofInstant(event.getDate().toInstant(), ZoneOffset.UTC))
.isEqualTo(ZonedDateTime.of(2017, 3, 6, 12, 53, 10, 0, ZoneOffset.UTC));
assertThat(event.getRaw()).isEqualTo(rawMessage.getBytes(StandardCharsets.UTF_8));
assertThat(event.getMessage()).isEqualTo("date=2017-03-06 time=12:53:10 devname=DEVICENAME devid=DEVICEID logid=0000000013 type=traffic subtype=forward level=notice vd=ALIAS srcip=IP srcport=45748 srcintf=\"IF\" dstip=IP dstport=443 dstintf=\"IF\" sessionid=1122686199 status=close policyid=77 dstcountry=\"COUNTRY\" srccountry=\"COUNTRY\" trandisp=dnat tranip=IP tranport=443 service=HTTPS proto=6 appid=41540 app=\"SSL_TLSv1.2\" appcat=\"Network.Service\" applist=\"ACLNAME\" appact=detected duration=1 sentbyte=2313 rcvdbyte=14883 sentpkt=19 rcvdpkt=19 utmaction=passthrough utmevent=app-ctrl attack=\"SSL\" hostname=\"HOSTNAME\" custom=\"white space\"");
assertThat(event.getFields())
.containsEntry("date", "2017-03-06")
.containsEntry("time", "12:53:10")
.containsEntry("devname", "DEVICENAME")
.containsEntry("devid", "DEVICEID")
.containsEntry("hostname", "HOSTNAME")
.containsEntry("custom", "white space");
}
}