############################################################################# # Copyright (c) 2015 Balabit # # This program is free software; you can redistribute it and/or modify it # under the terms of the GNU General Public License version 2 as published # by the Free Software Foundation, or (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA # # As an additional exemption you are allowed to compile & link against the # OpenSSL libraries as published by the OpenSSL project. See the file # COPYING for details. # ############################################################################# # Parse apache access.log # # Formats recognized: # # LogFormat "%v:%p %h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" vhost_combined # virtualhost:443 127.0.0.1 - frank [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326 "http://www.example.com/start.html" "Mozilla/4.08 [en] (Win98; I ;Nav)" # # LogFormat "%v:%p %h %l %u %t \"%r\" %>s %b" vhost_common # virtualhost:443 127.0.0.1 - frank [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326 # # LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\"" combined # 127.0.0.1 - frank [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326 "http://www.example.com/start.html" "Mozilla/4.08 [en] (Win98; I ;Nav)" # # LogFormat "%h %l %u %t \"%r\" %>s %b" common # 127.0.0.1 - frank [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326 block parser apache-accesslog-parser-vhost(prefix() template()) { channel { rewrite { set("`template`" value("1")); }; filter { match("^[A-Za-z0-9\-\._]+:[0-9]+ " value("1")); }; parser { csv-parser( dialect(escape-double-char) flags(strip-whitespace) delimiters(" ") template($1) quote-pairs('""[]') columns("2", "`prefix`clientip", "`prefix`ident", "`prefix`auth", "`prefix`timestamp", "`prefix`rawrequest", "`prefix`response", "`prefix`bytes", "`prefix`referrer", "`prefix`agent")); csv-parser( prefix(`prefix`) template("$2") delimiters(":") dialect(escape-none) columns("vhost", "port")); }; }; }; # combined & common format without vhost # LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined # LogFormat "%h %l %u %t \"%r\" %>s %b" common block parser apache-accesslog-parser-combined(prefix() template()) { channel { parser { csv-parser( prefix(`prefix`) dialect(escape-double-char) flags(strip-whitespace) delimiters(" ") template(`template`) quote-pairs('""[]') columns("clientip", "ident", "auth", "timestamp", "rawrequest", "response", "bytes", "referrer", "agent")); }; }; }; block parser apache-accesslog-parser(prefix(".apache.") template("${MESSAGE}")) { # parse into a logstash-like schema # https://github.com/elastic/logstash/blob/v1.4.2/patterns/grok-patterns#L90 channel { # parser for formats including vhost:port if { parser { apache-accesslog-parser-vhost(prefix(`prefix`) template(`template`)); }; # parser for standard formats } else { parser { apache-accesslog-parser-combined(prefix(`prefix`) template(`template`)); }; }; # mungle values to match Kibana/elastic schema and common to all # supported formats. parser { csv-parser( prefix(`prefix`) template("${`prefix`rawrequest}") delimiters(" ") dialect(escape-none) flags(strip-whitespace) columns("verb", "request", "httpversion")); date-parser(format("%d/%b/%Y:%H:%M:%S %z") template("${`prefix`timestamp}")); }; rewrite { subst("^HTTP/(.*)$", "$1", value("`prefix`httpversion")); }; }; };