#!/usr/bin/perl
#
#   mail-report v0.1, Copyright (C) 2008 Mike Cathey
# 
#  This program is free software; you can redistribute it and/or modify
#  it under the terms of the GNU General Public License as published by
#  the Free Software Foundation; either version 2 of the License, or
#  (at your option) any later version.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.
#
#  You should have received a copy of the GNU General Public License
#  along with this program; if not, write to the Free Software
#  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
#

my $DEBUG = 0;
# log lines missed by the parser
my $missed = "";

# amavis stuff
my $spam_without_sare = 0;
my $spam_with_sare = 0;
my $spam = 0;
my $not_spam = 0;
# sum of scores for all spam messages
my $spam_score_total = 0;
# sum of scores for all ham messages
my $ham_score_total = 0;
# sum of scores for all messages
my $score_total = 0;
# total number of messages that amavis sees
my $messages_total = 0;
# count of all messages rejected in the smtp session (rbls, spf, relay checks, etc)
my $smtp_rejects = 0;
# this is in milliseconds
my $processing_time = 0;

# stuff bigger than amavis's message size limit
my $no_scan = 0;

# virus messages
my $virus_emails = 0;

# local deliveries
my $local_deliveries = 0;

# Sender Reject messages
my $sender_rejects = 0;
my %sender_rejects = {};

# Client host rejects (from access maps)
$client_rejects = 0;

# Relay access denied
$relay_rejects = 0;

# Recipient Reject messages
my $recipient_rejects = 0;
# we only have SPF rejects in this category
my $spf_rejects = 0;

# helo rejects
my $helo_rejects = 0;
my %helo_rejects = {};

# RBLs
my $rbl_blocks = 0;
my %rbls = {};

# from/to addresses/domains
my %sender = {};
my %recipient = {};
my %sender_domains = {};
my %recipient_domains = {};

# temp variables
my $from = "";
my $from_domain = "";
my $to = "";
my $to_domain = "";
my $amavis_log_line = "";

while ( <STDIN> )
{
	# just in case
	chomp;

	# local deliveries
	# FIXME FIXME FIXME you _WILL_ need to change this to match your system
	# Mar 29 18:46:47 sid postfix/lmtp[15874]: CFB7B4403F3:
	# to=<blah@foo.com>, relay=sid.cathey.us[/var/run/cyrus/socket/lmtp],
	# delay=0.09, delays=0.02/0.02/0.04/0.01, dsn=2.1.5, status=sent (250 2.1.5 Ok)
	#
	if ( $_ =~ /postfix\/lmtp.*relay=sid.cathey.us\[.*socket\/lmtp\].*/i )
	{
		$local_deliveries++;
	}

	# Client host rejects (from access maps)
	# Mar 29 20:50:58 sid postfix/smtpd[19138]: NOQUEUE: reject: RCPT from
	# unknown[193.200.50.5]: 554 5.7.1 <unknown[193.200.50.5]>: Client host
	# rejected: "CUSTOM REJECT MESSAGE HERE"; from=<blah@blah.com>
	# to=<blah@blah.com> proto=SMTP helo=<74.205.133.5>
	#
	elsif ( $_ =~ /postfix\/smtpd.*NOQUEUE\:.*Client host rejected\:.*/i )
	{
		$client_rejects++;
		$smtp_rejects++;
	}

	# Recipient rejects
	# Mar 30 04:24:38 sid postfix/smtpd[4465]: NOQUEUE: reject: RCPT from
	# mail2.cathey.us[72.4.203.54]: 554 5.7.1 <foo@blah.net>: Relay
	# access denied; from=<blah@foo.com> to=<foo@blah.net>
	# proto=ESMTP helo=<mail2.cathey.us> 
	#
	elsif ( $_ =~ /postfix\/smtpd.*NOQUEUE\:.*Relay access denied\;.*/i )
	{
		$relay_rejects++;
		$smtp_rejects++;
	}

	# Recipient rejects
	# Mar 30 04:53:35 sid postfix/smtpd[6013]: NOQUEUE: reject: RCPT from
	# unknown[216.6.0.18]: 554 5.7.1 <foo@blah.com>: Recipient address
	# rejected: Please see
	# http://www.openspf.org/why.html?sender=foo%40blah.com&ip=216.6.0.18&receiver=sid.cathey.us;
	# from=<foo@blaha.com> to=<foo@blah.com> proto=ESMTP
	# helo=<[216.6.0.18]>

	# spf is the only one we have right now
	elsif ( $_ =~ /postfix\/smtpd.*NOQUEUE\:.*Recipient address rejected:\sPlease see.*openspf\.org.*\;/i )
	{
		$spf_rejects++;
		$recipient_rejects++;
		$smtp_rejects++;
	}

	# Sender rejects
	# Mar 30 05:53:39 sid postfix/smtpd[8895]: NOQUEUE: reject: RCPT from ns2.ascio.com[80.237.206.33]: 450 4.1.8 <foo@blah.com>: Sender address rejected: Domain not found; from=<foo@blah.com> to=<foo@blah.com> proto=ESMTP helo=<cgn1srv3.ascio.net>
	elsif ( $_ =~ /postfix\/smtpd.*NOQUEUE\:.*Sender address rejected:\s([a-z0-9\.\-\ ]+)\;/i )
	{
		my $sender_reject = $1;
		$sender_rejects{$sender_reject}++;
		$sender_rejects++;
		$smtp_rejects++;
		
	}

	# HELO rejections
	# Mar 30 06:28:49 sid postfix/smtpd[9329]: NOQUEUE: reject: RCPT from
	# 82.158.90.160.dyn.user.ono.com[82.158.90.160]: 504 5.5.2 <usuario1>:
	# Helo command rejected: need fully-qualified hostname;
	# from=<foo@blahand.com> to=<foo@blah.com>
	# proto=SMTP helo=<usuario1>
	elsif ( $_ =~ /postfix\/smtpd.*NOQUEUE\:.*Helo command rejected\:\s([a-z0-9\.\-\ ]+)\;.*/i )
	{
		my $helo_reject = $1;
		$helo_rejects{$helo_reject}++;
		$helo_rejects++;
		$smtp_rejects++;
		
	}

	# RBLs
	# Mar 30 06:44:41 sid postfix/smtpd[9228]: NOQUEUE: reject: RCPT from
	# unknown[124.13.97.131]: 554 5.7.1 Service unavailable; Client host
	# [124.13.97.131] blocked using zen.spamhaus.org;
	# http://www.spamhaus.org/query/bl?ip=124.13.97.131;
	# from=<foo@blah.org> to=<foo@blah.us> proto=ESMTP
	# helo=<[124.13.97.131]>
	#
	elsif ( $_ =~ /postfix\/smtpd.*NOQUEUE\:.*blocked using\s([a-z0-9\.\-]+)\;.*/i )
	{
		my $rbl = $1;
		$rbl_blocks++;
		$rbls{$rbl}++;
		$smtp_rejects++;
	}

	# stuff too big for amavis
	# Mar 30 11:47:47 sid amavis[22357]: (22357-13) spam_scan: not wasting
	# time on SA, message longer than 204800 bytes: 1799+313948
	#
	elsif ( $_ =~ /\samavis.*message longer than [0-9]+ bytes\:\s(.*)/i )
	{
		$no_scan++;
	}
	

	# Virus infected messages
	# 
	#
	#
	elsif ( $_ =~ /\samavis.*\sINFECTED\s/i )
	{
		$virus_emails++;
		$messages_total++;
	}

	# amavis stuff
	elsif ($_ =~ /\samavis\[/ )
	{

		# grab from/to data
		$amavis_log_line = $_;
		if ( $amavis_log_line =~ /\<([a-z0-9\.\-\@]*)\>\s\-\>\s\<([a-z0-9\.\-\@]+)\>/i ) 
		{
			$from = $1;
			if ($from ne "" )
			{
				$from_domain = $from;
				$from_domain =~ s/.*\@//; 
				$sender{$from}++;
				$sender_domains{$from_domain}++;
			}

			$to = $2;
			if ($to ne "" )
			{
				$to_domain = $to;
				$to_domain =~ s/.*\@//; 
				$recipient{$to}++;
				$recipient_domains{$to_domain}++;
			}
			# print "MATCHED: from = $from; from_domain = $from_domain; to = $to; to_domain = $to_domain\n";
		}

		# get timing information
		# amavis[10251]: (10251-04) TIMING [total 859 ms] - 
		if ( $amavis_log_line =~ /TIMING\s\[total\s([0-9]+)\sms\]/i ) 
		{
			$processing_time += $1;
		}

		# get the spam/nonspam counts
		my @tokens = split(/\ /, $_);
		my $score = 0;
		my $sare_contribution = 0;
		my $is_spam = 0;
		foreach my $token (@tokens)
		{

			# Yes,
			if ( $token =~ /Yes,/ )
			{ 
				$spam++;
				$is_spam = 1;
				$messages_total++;
			}
			elsif ( $token =~ /No,/ )
			{
				$not_spam++;
				$messages_total++;
			}
		
			# score=20.111
			if ( $token =~ /score=([0-9\.]+)/ )
			{ 
				$score = $1;
				$score_total += $score;

				if ( $is_spam == 1 )
				{
					$spam_score_total += $score;
				}
				else 
				{
					$ham_score_total += $score;
				}
			}
		
			# SARE_MSGID_DDDASH=1.666,
			if ( $token =~ /SARE_[a-z0-9\_]+=([0-9\.]+)\,/i )
			{ 
				$sare_contribution += $1;
			}
		}
	
		if ($is_spam == 1)
		{
			print "SPAM:\tSCORE: $score; SARE: $sare_contribution; " if $DEBUG > 0;
			if ( ( $score - $sare_contribution ) > 6.31 )
			{ 
				print "NO" if $DEBUG > 0;
				$spam_without_sare++;
			}
			else
			{
				print "YES" if $DEBUG > 0;
				$spam_with_sare++;
			}
			print "\n" if $DEBUG > 0;
		}
		else 
		{
			print "HAM:\tSCORE: $score; SARE: $sare_contribution\n" if $DEBUG > 0;
		}
	}
	
	# ignores
	# FIXME FIXME FIXME you _WILL_ need to change this to match your system
	elsif ( $_ =~ /postfix\/cleanup/ 
			|| $_ =~ /last message repeated [0-9]+ times/i
			|| $_ =~ /postfix\/qmgr/i
			|| $_ =~ /postfix\/master/i
			|| $_ =~ /postfix\/postfix-script/i
			|| $_ =~ /postfix\/scache/i
			|| $_ =~ /postfix\/pickup/i
			|| $_ =~ /postfix\/bounce/i
			|| $_ =~ /postfix\/smtp/i
			|| $_ =~ /postfix\/policy-spf/i
			|| $_ =~ /postfix\/anvil/i
			|| $_ =~ /postfix\/smtpd.*connect from/i
			|| $_ =~ /postfix\/smtpd.*lost connection/i
			|| $_ =~ /cyrus\/[a-z0-9\_\-]+\[[0-9]+\].*/i
			|| $_ =~ /fetchmail\[[0-9]+\].*/i
			|| $_ =~ /in.imapproxyd\[[0-9]+\].*/i
		) 
	{
		# do nothing
	}

	# what did we miss?
	else
	{
		$missed .= $_ . "\n";
	}
}

#
# sum of smtp rejections and amavis processed/unprocessed messages
my $total = $messages_total + $smtp_rejects  + $no_scan;
#
print "\n";
print "\n";
print "========================================================================\n";
print "Summary:\n";
print "========================================================================\n";
print "\n";
print "Local deliveries (includes spam below Amavis kill2 threshold): $local_deliveries\n";
print "\n";
print "Total message delivery attempts: $total\n";
print "SMTP Rejections = ($smtp_rejects) " . sprintf("%.2f",($smtp_rejects / $total * 100)) . "%\n";
print "Spam = ($spam) " . sprintf("%.2f",($spam / $total * 100)) . "%\n";
print "Ham (clean messages) = ($not_spam) " . sprintf("%.2f",(  $not_spam / $total * 100)) . "%\n";
print "Virus infected = ($virus_emails) " . sprintf("%.2f",($virus_emails / $total * 100)) . "%\n";
print "Unprocessed = ($no_scan) " . sprintf("%.2f",($no_scan / $total * 100)) . "%\n";
print "\n";
#
print "========================================================================\n";
print "Detailed Numbers\n";
print "========================================================================\n";
print "\n";
print "SpamAssassin/Amavis statistics:\n";
print "\tTotal processed: $messages_total\n";
print "\tNot scanned due to size: $no_scan\n";
print "\n";
print "\tVirus Infected: $virus_emails (" . sprintf("%.2f",($virus_emails / $messages_total * 100)) . "%\)\n";
print "\tSpam: $spam (" . sprintf("%.2f",($spam / $messages_total * 100)) . "%\)\n";
print "\tHam: $not_spam (" . sprintf("%.2f",( $not_spam / $messages_total * 100 )) . "%\)\n";
print "\n";
print "\tAverage message score: " . sprintf("%.2f",($score_total / $messages_total))  . "\n";
print "\tAverage spam score: " . sprintf("%.2f",($spam_score_total / $spam))  . "\n";
print "\tAverage non-spam score: " . sprintf("%.2f",($ham_score_total / $not_spam))  . "\n";
print "\tAverage scan time: " . sprintf("%.3f",($processing_time / $messages_total / 1000 ))  . " seconds\n";
#
print "\tSPAM without SARE: $spam_without_sare\n";
print "\tSARE Contribution: $spam_with_sare\n";
#
print "\n";
print "SMTP-level rejects: $smtp_rejects\n";
print "\n";
print "\tClient host rejects (from access maps): $client_rejects\n";
print "\n";
print "\tRelay denied: $relay_rejects\n";
print "\n";
print "\tSPF rejects: $spf_rejects\n";
print "\n";
print "\tRBL Blocks: $rbl_blocks\n";
foreach my $rbl_name ( keys %rbls )
{
	# I kept getting an empty hash element...didn't feel like debugging it
	if ($rbls{$rbl_name} > 0 )
	{
		print "\t\t$rbl_name: $rbls{$rbl_name}\n";
	}
}
#
print "\n";
#
print "\tHELO rejections: $helo_rejects\n";
foreach my $helo_reject_message ( keys %helo_rejects )
{
	# I kept getting an empty hash element...didn't feel like debugging it
	if ($helo_rejects{$helo_reject_message} > 0 )
	{
		print "\t\t$helo_reject_message: $helo_rejects{$helo_reject_message}\n";
	}
}
#
print "\n";
#
print "\tSender Rejects: $sender_rejects\n";
foreach my $sender_reject_message ( keys %sender_rejects )
{
	# I kept getting an empty hash element...didn't feel like debugging it
	if ($sender_rejects{$sender_reject_message} > 0 )
	{
		print "\t\t$sender_reject_message: $sender_rejects{$sender_reject_message}\n";
	}
}
#
print "\n";
#
my $counter = 0;
print "Top 20 Recipients\n";
foreach $key (sort { $recipient {$b} <=> $recipient {$a}} keys %recipient )
{
	print "\t$key $recipient{$key}\n";
 	$counter++;
 
 	if ( $counter == 20 )
	{
		last;
	}	

}
#
print "\n";
$counter = 0;
print "Top 20 Recipient Domains\n";
foreach $key (sort { $recipient_domains {$b} <=> $recipient_domains {$a}} keys %recipient_domains )
{
	print "\t$key $recipient_domains{$key}\n";
 	$counter++;
 
 	if ( $counter == 20 )
	{
		last;
	}	

}
#
print "\n";
$counter = 0;
print "Top 20 Senders\n";
foreach $key (sort { $sender{$b} <=> $senders {$a}} keys %sender )
{
	print "\t$key $sender{$key}\n";
 	$counter++;
 
 	if ( $counter == 20 )
	{
		last;
	}	

}
#
print "\n";
$counter = 0;
print "Top 20 Sender Domains\n";
foreach $key (sort { $sender_domains {$b} <=> $sender_domains {$a}} keys %sender_domains )
{
	print "\t$key $sender_domains{$key}\n";
 	$counter++;
 
 	if ( $counter == 20 )
	{
		last;
	}	

}

print "\n";
print "========================================================================\n";
print "Potentially interesting things missed by this report parser\n";
print "========================================================================\n";
print "\n";
print $missed;
print "========================================================================\n";
print "END\n";
print "========================================================================\n";