Parsing apache logs using PostgreSQL - apache

Question

This O'Reilly article gives an example of a PostgreSQL statement that parses an Apache log line:
INSERT INTO http_log(log_date,ip_addr,record)
SELECT CAST(substr(record,strpos(record,'[')+1,20) AS date),
CAST(substr(record,0,strpos(record,' ')) AS cidr),
record
FROM tmp_apache;
Obviously this only extracts the IP and timestamp fields. Is there a canonical statement for extracting all fields from a typical combined log format record? If there isn't, I will write one and I promise to post the result here!

Answer 1

OK, here is my solution:
insert into accesslog
select m[1], m[2], m[3],
(to_char(to_timestamp(m[4], 'DD/Mon/YYYY:HH24:MI:SS'), 'YYYY-MM-DD HH24:MI:SS ')
|| split_part(m[4], ' ',2))::timestamp with time zone,
m[5], m[6]::smallint, (case m[7] when '-' then '0' else m[7] end)::integer, m[8], m[9] from (
select regexp_matches(record,
E'(.*) (.*) (.*) \\[(.*)\\] "(.*)" (\\d+) (.*) "(.*)" "(.*)"')
as m from tmp_apache) s;
It takes raw log lines from the table tmp_apache and extracts the fields (using the regexp) into an array.

Answer 2

Here is my somewhat more complete solution.
The apache log file should not contain invalid characters or backslashes. If necessary, you can remove these from the log file with:
cat logfile | strings | grep -v '\\' > cleanedlogfile
Then copy and parse the log file into postgres (m[1] to m[7] correspond to the regex groups in regexp_matches function):
-- sql for postgres:
drop table if exists rawlog;
create table rawlog (record varchar);
-- import data from log file
copy rawlog from '/path/to/your/apache/cleaned/log/file';
-- parse the rawlog into table accesslog
drop table if exists accesslog;
create table accesslog as
(select m[1] as clientip,
(to_char(to_timestamp(m[4], 'DD/Mon/YYYY:HH24:MI:SS'), 'YYYY-MM-DD HH24:MI:SS ')
|| split_part(m[4], ' ',2))::timestamp with time zone as "time",
split_part(m[5], ' ', 1) as method,
split_part(split_part(m[5], ' ', 2), '?', 1) as uri,
split_part(split_part(m[5], ' ', 2), '?', 2) as query,
m[6]::smallint as status,
m[7]::bigint bytes
from
(select
regexp_matches(record, E'(.*) (.*) (.*) \\[(.*)\\] "(.*)" (\\d+) (\\d+)') as m
from rawlog) s);
-- optionally create indexes
create index accesslogclientipidx on accesslog(clientip);
create index accesslogtimeidx on accesslog(time);
create index accessloguriidx on accesslog(uri);