How to count the number of emails sent per single account EXIM? - exim

Question

My application needs to count in real time the number of emails and the time it was sent by exim, is it possible?
The connection is made by SMTP.

Answer 1

There are three ways to do that:
1 Parsing logs (worse approach).
2 RSyslog implementation plus Exim conf.
3 Exim with Mysql.
Rsyslog
Install syslog and syslog-mysql
[root#web ~]# yum install rsyslog rsyslog-mysql
Basic configuration
[root#web ~]# mysql
mysql> CREATE DATABASE Syslog;
mysql> USE Syslog;
mysql> CREATE TABLE `SmtpMailLog` (
`Id` bigint(11) unsigned NOT NULL AUTO_INCREMENT,
`Hostname` varchar(255) NOT NULL,
`EximID` varchar(16) NOT NULL,
`DateIn` datetime DEFAULT NULL,
`DateLastProcessed` datetime DEFAULT NULL,
`DateCompleted` datetime DEFAULT NULL,
`FromAddr` varchar(100) DEFAULT NULL,
`FromAddrHost` varchar(100) DEFAULT NULL,
`FirstToAddr` varchar(100) DEFAULT NULL,
`AdditionalToAddr` text,
`HostFrom` varchar(100) DEFAULT NULL,
`FirstHostTo` varchar(100) DEFAULT NULL,
`Size` int(11) DEFAULT NULL,
`Subject` varchar(255) DEFAULT NULL,
`Notes` varchar(255) DEFAULT NULL,
PRIMARY KEY (`Id`),
UNIQUE KEY `EximID` (`EximID`)
) ENGINE=MyISAM AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COMMENT='--';
mysql> exit
[root#web ~]# echo "USE mysql; CREATE USER rsyslog; FLUSH PRIVILEGES;" | mysql
[root#web ~]# echo "USE mysql; GRANT ALL PRIVILEGES ON Syslog.* TO 'rsyslog'#'127.0.0.1' IDENTIFIED BY 'rsysl0g'; FLUSH PRIVILEGES;" | mysql
[root#web ~]# echo "USE mysql; SET PASSWORD FOR 'rsyslog'#'127.0.0.1' = PASSWORD('rsysl0g'); FLUSH PRIVILEGES;" | mysql
[root#web ~]# /bin/cat << EOF > /etc/rsyslog.conf
# Modules --------------------------------------------------------------------
# Input
$ModLoad imuxsock.so # Unix sockets
# Output
$ModLoad ommysql.so # Log to MySQL
# Globals --------------------------------------------------------------------
# There are many more - see docs
# Files and dirs are created as needed (dirs only for "dynamic" files)
$umask 0000
$DirCreateMode 0640
$FileCreateMode 0640
#$FileOwner rsyslog
#$FileGroup rsyslog
#$DirOwner rsyslog
#$DirGroup rsyslog
$RepeatedMsgReduction on
# Include package specific logs (including rsyslog itself)
$IncludeConfig /etc/rsyslog.d/*.conf
# Log to the console
*.* -/var/log/exim/main.log
& ~
EOF
Parser data Configuration
[root#web ~]# /bin/cat << EOF > /etc/rsyslog.d/20-mail.conf
# ###############################################################
# Mail system logging
# Exim, Spam Assassin, SA-Exim, ClamAV
# /etc/rsyslog.d/20-mail.conf
# ###############################################################
# NOTES
# Careful with quotes in if clauses
# seems to need ' and not " (JG 11 Jun 2009)
# Multi line logging from Exim "detector":
# :msg, regex, " \[[0-9]{1,3}[\\/][0-9]{1,3}\]" ~
# email address finder:
# %msg:R,ERE,0,ZERO:[A-Za-z0-9._%+-]+#[A-Za-z0-9.-]+\.[A-Za-z]{2,4}--end%
# Exim ID finder:
# %msg:R,ERE,0,ZERO:[A-Za-z0-9]{6}-[A-Za-z0-9]{6}-[A-Za-z0-9]{2}--end%
# Easier to read log format:
# $template Mail-Exim-File-Format,"%timegenerated:1:10:date-rfc3339% %timegenerated:12:19:date-rfc3339% %hostname% %syslogtag%%msg%\n"
#########################################################
# Syslog style to support OSSEC (JG 26 AUg 2009)
$template Mail-Exim-File-Format,"%timegenerated% %HOSTNAME% %syslogtag%%msg%\n"
#########################################################
# Amalgamated logging templates
# The log entry is built up an initial entry from ClamAV followed by successive updates from the vaious components, in the order
# of the templates here. The EximID is used to look up the entry except for SA-Exim (which uses the msgid).
# <= - In
# Local:
# Sep 15 09:06:17 loghost exim[20787]: 1MnT3J-0005PH-2y <= nagios#example.com U=nagios P=local S=794 T="** PROBLEM Service Alert: host-name/NTP-peer is CRITICAL **"
# Sep 22 10:40:59 portal exim[12557]: 1Mq1rn-0003GX-MZ <= root#blueloop.net U=root P=local S=516 T="test message"
# Relayed:
# Sep 15 09:03:38 loghost exim[20078]:
# 1MnT0g-0005Dq-BC <= user#example.com H=host.example.com [192.168.100.100] P=esmtp S=8690192 id=4AAF585B020000AA0004ED5B#port.blueloop.net T="Subject line from message"
# If an arg to CONCAT is NULL then the whole output is NULL
$template Mail-Exim-In-Amalgamated,"REPLACE INTO SmtpMailLog \
( \
Hostname, \
EximID, \
DateIn, \
DateLastProcessed, \
FirstToAddr, \
FromAddr, \
FromAddrHost, \
AdditionalToAddr, \
HostFrom, \
Size, \
Subject, \
FirstHostTo \
) \
VALUES \
( \
'%hostname%', \
'%msg:R,ERE,0,ZERO:[A-Za-z0-9]{6}-[A-Za-z0-9]{6}-[A-Za-z0-9]{2}--end%', \
'%timereported:::date-mysql%', \
'%timereported:::date-mysql%', \
'%msg:R,ERE,0,ZERO:([A-Za-z0-9._%+-]+#[A-Za-z0-9.-]+\.[A-Za-z]{2,4}$)--end%', \
'%msg:R,ERE,0,ZERO:[A-Za-z0-9._%+-]+#[A-Za-z0-9.-]+\.[A-Za-z]{2,4}--end%', \
substring_index('%msg:R,ERE,0,ZERO:[A-Za-z0-9._%+-]+#[A-Za-z0-9.-]+\.[A-Za-z]{2,4}--end%', '#', -1), \
'', \
SUBSTRING('%msg:R,ERE,0,ZERO:H=.*\[[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}]--end%' FROM 3), \
SUBSTRING('%msg:R,ERE,0,ZERO:S=[0-9]{1,}--end%' FROM 3), \
SUBSTRING('%msg:R,ERE,0,ZERO:T=.*--end%' FROM 3), \
'pending' \
) \
",SQL
# ** - Failed
$template Mail-Exim-Fail-Amalgamated,"UPDATE SmtpMailLog \
SET \
DateLastProcessed = '%timereported:::date-mysql%', \
FirstToAddr = 'Failed - see notes', \
FirstHostTo = 'Failed - see notes', \
Notes = '%msg%' \
WHERE EximID = '%msg:R,ERE,0,ZERO:[A-Za-z0-9]{6}-[A-Za-z0-9]{6}-[A-Za-z0-9]{2}--end%' \
",SQL
# => - Out
$template Mail-Exim-Out-Amalgamated, "UPDATE SmtpMailLog \
SET \
FirstToAddr = '%msg:R,ERE,0,ZERO:[A-Za-z0-9._%+-]+#[A-Za-z0-9.-]+\.[A-Za-z]{2,4}--end%', \
FirstHostTo = SUBSTRING('%msg:R,ERE,0,ZERO:H=.*]--end%' FROM 3), \
DateLastProcessed = '%timereported:::date-mysql%', \
Notes = 'Out' \
WHERE EximID = '%msg:R,ERE,0,ZERO:[A-Za-z0-9]{6}-[A-Za-z0-9]{6}-[A-Za-z0-9]{2}--end%' \
",SQL
# -> - additional deliveries
$template Mail-Exim-Add-Amalgamated, "UPDATE SmtpMailLog \
SET \
AdditionalToAddr = CONCAT_WS(' ',AdditionalToAddr,'%msg:R,ERE,0,ZERO:[A-Za-z0-9._%+-]+#[A-Za-z0-9.-]+\.[A-Za-z]{2,4}--end%'), \
DateLastProcessed = '%timereported:::date-mysql%', \
Notes = 'Additional delivery' \
WHERE EximID = '%msg:R,ERE,0,ZERO:[A-Za-z0-9]{6}-[A-Za-z0-9]{6}-[A-Za-z0-9]{2}--end%' \
",SQL
# Completed
$template Mail-Exim-Completed-Amalgamated,"UPDATE SmtpMailLog \
SET \
DateCompleted = '%timereported:::date-mysql%', \
DateLastProcessed = '%timereported:::date-mysql%', \
Notes = 'Completed' \
WHERE EximID = '%msg:R,ERE,0,ZERO:[A-Za-z0-9]{6}-[A-Za-z0-9]{6}-[A-Za-z0-9]{2}--end%' \
",SQL
#########################################################
# Full Exim log (bar the bits that are filtered out above) - file
if $programname == 'exim' then /var/log/exim/main.log;Mail-Exim-File-Format
###################################
# Amalgamated Mail log - single line per mail, some details lost - DB
#if $programname == 'exim' \
# and $msg contains 'dovecot_login' \
#then :ommysql:127.0.0.1,Syslog,rsyslog,rsysl0g;Mail-Exim-New-Amalgamated
if $programname == 'exim' \
and $msg contains '<=' \
then :ommysql:127.0.0.1,Syslog,rsyslog,rsysl0g;Mail-Exim-In-Amalgamated
if $programname == 'exim' \
and $msg contains '=>' \
then :ommysql:127.0.0.1,Syslog,rsyslog,rsysl0g;Mail-Exim-Out-Amalgamated
if $programname == 'exim' \
and $msg contains '->' \
then :ommysql:127.0.0.1,Syslog,rsyslog,rsysl0g;Mail-Exim-Add-Amalgamated
if $programname == 'exim' \
and $msg contains '**' \
then :ommysql:127.0.0.1,Syslog,rsyslog,rsysl0g;Mail-Exim-Fail-Amalgamated
if $programname == 'exim' \
and $msg contains 'Completed' \
then :ommysql:127.0.0.1,Syslog,rsyslog,rsysl0g;Mail-Exim-Completed-Amalgamated
##################################
# Dump Exim messages
if $programname == 'exim' then ~
EOF
Adjust exim log selector:
[root#web ~]# vi /etc/exim/exim.conf
log_selector = +incoming_port +smtp_connection +all_parents +retry_defer +subject +arguments +received_recipients
--
Exim Mysql
Install dependencies.
[root#web ~]# yum install exim-mysql
Add exim mysql connection.
[root#web ~]# vi /etc/exim/exim.conf
hide mysql_servers = 127.0.0.1/{DATABASE}/{USER}/{PASSWORD}
Is possible to use the same table structure as used on Rsyslog instalation.
On acl_smtp_data section, add some like that:
acl_smtp_data:
warn
continue = ${lookup mysql{INSERT INTO SmtpMailLog \
(\
AdditionalToAddr \
)\
values \
(\
'${quote_mysql:$recipients}' \
)}}