# # Kiss Péter - 2015.03.26 # kiss.peter@linuxadm.hu # input { udp { port => 10000 type => syslog } } filter { grok { type => "syslog" pattern => [ "<%{POSINT:syslog_pri}>%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" ] add_field => [ "received_at", "%{@timestamp}" ] add_field => [ "received_from", "%{@source_host}" ] add_tag => ["Syslog"] } syslog_pri { } date { type => "syslog" match => [ "syslog_timestamp", "MMM d HH:mm:ss.SSS","MMM d HH:mm:ss", "MMM dd HH:mm:ss.SSS","MMM dd HH:mm:ss","dd/MMM/yyyy:HH:mm:ss +0100","yyyy/MM/dd HH:mm:ss" ] } mutate { type => "syslog" exclude_tags => "_grokparsefailure" replace => [ "@source_host", "%{syslog_hostname}" ] replace => [ "@message", "%{syslog_message}" ] } mutate { type => "syslog" remove => [ "syslog_hostname", "syslog_message", "syslog_timestamp" , "received_from"] } if [syslog_facility] == "local1" or [syslog_facility] == "user-level" { grok{ type => "syslog" # Nginx log: pattern => ["%{SYSLOGTIMESTAMP:syslog_timestamp} %{IPORHOST:http_host} %{IPORHOST:clientip} (%{NUMBER:bytes}|-) (%{NUMBER:user-identifier}|-) \[%{HTTPDATE:timestamp}\] \"(?:%{WORD:verb} %{NOTSPACE:request} (?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})\" %{NUMBER:response} (?:%{NUMBER:bytes}|\"-\") (%{QS:referrer}||\"-\")"] } mutate { remove_tag => ["_grokparsefailure"] remove_tag => ["Syslog"] add_tag => ["Nginx"] } } } output { elasticsearch { host => "localhost" protocol => "http" } if [type] == "syslog" and "_grokparsefailure" in [tags] { file { path => "/var/log/logstash/failed_syslog_events-%{+YYYY-MM-dd}.log" } } }