#!/usr/bin/perl
# log-analysis.pl written by John Borwick 2003 for Wake Forest University.
# uses "heuristics" (in a loose sense of the word) to judge which messages
# from a log should be reported.
use strict;
use warnings;
use File::Find;
use POSIX qw(strftime);

# quick exit: don't even run startup commands if initial conditions aren't met
die "usage: $0 [/path/to/log/dir [/path/to/log/dir2 ... ]]\n"
  unless @ARGV>0;

foreach (@ARGV) {
  die "Directory '$_' doesn't exist!\n" unless -d $_;
}

#  INITIALIZATION
# regex strings you always want to ignore, no matter what.
our $ACKNOWLEDGED_BAD_STRING_REGEX =
  '\b(' . (join '|', split /\n/, <<'_ACKNOWLEDGED_BAD_STRINGS_') . ')\b';
enabledProcessor #\d+ Unknown CPU
_ACKNOWLEDGED_BAD_STRINGS_

# 'bad' regex strings; you will see errors unless they are explicitly ignored
our $BAD_STRING_REGEX =
  '\b(' . ( join '|', (split /\s+/, <<'_BAD_STRINGS_'),
fail\w* abort\w* warn\w* unknown wrong fatal error unsafe cannot
_BAD_STRINGS_
	    ('no free leases', 'not configured')
          ) . ')\b';

# 'good' regex strings; you will never see these messages
our $GOOD_STRING_REGEX =
  '\b(' . ( join '|', (split /\s+/, <<'_GOOD_STRINGS_'),
accept\w* authenticate\w* \w*connect\w* exiting open\w* rebuilt relay
\w*start\w* running creat\w* succeed\w* clos\w* terminat\w* register\w*
success\w* cancelled
_GOOD_STRINGS_
	    ('no more authentication methods')
	  ) . ')\b';

# services you don't want to hear about unless there was an error
our $SERVICE_ONLY_BAD_REGEX =
  '\b(' . ( join '|', split /\s+/, <<'_SERVICE_ONLY_BAD_' ) . ')\b';
kernel crond crontab anacron dd sysctl gconfd ifup ifdown xfs
sendmail firstboot dhcpd dhclient info
_SERVICE_ONLY_BAD_

# $YESTERDAY is printed out at the beginning of the log
our $YESTERDAY = strftime("%b %d", localtime( time() - 86400 ) );
our $RELEVANT_DATE_REGEX = '\b(' . $YESTERDAY . ')\b';

# $MONTH_REGEX is helps guess whether a file is a message log
our $MONTH_REGEX = '\b(' . (join '|', qw(Jan Feb Mar Apr May Jun
                                         Jul Aug Sep Oct Nov Dec)) . ')\b';

#  RUN.  begin the clock
our $START_TIME = time();

our @LOGS;			# list of log files to process
find( \&search, @ARGV);		# populates @LOGS

print <<"_HEAD_";
Log analysis running... finding entries for $YESTERDAY.
_HEAD_

process_message_logs( @LOGS);	# do the actual reporting

print "Total analysis time ".( time() - $START_TIME ). " seconds\n";

sub search {
  # called by find()
  return unless -f;
  warn "I can't read from $File::Find::name\n" and return
    unless -r;
  if ( -T and log_type( $_ ) eq 'message' ) { # -T tests for type=text
    push @LOGS, $File::Find::name; # The full file path, not just $_
  }
}

sub log_type {
  # called by search().  could be extended
  my $file = shift;
  open LOG, $file
    or die "Can't open '$file': $!\n";
  if ( <LOG> =~ /^\W*$MONTH_REGEX\b/i ) {
    return 'message';
  } else {
    return 'unknown';
  }
}

sub process_message_logs {
  # called by main function.
  my (%LINES,			# unique lines to be printed. value=line count
      %PRIORITY,		# the 'rating' for %LINES keys
      $last_key);		# the last key put into %LINES

  my $lines_seen = 0;		# counting variable, just for statistics

  foreach my $file ( @LOGS ) {
    open F, $file or die "Can't open '$file': $!\n";

    while(<F>) {
      $lines_seen++;
      chomp;

      # turn 'Oct 24 10:00:00 xyz: bo' into 'Oct 24 10:00:00' and 'xyz: bo'
      my ($date, $message) = split_time( $_);

      # a line doesn't look like a date even though the file looks like a
      # message log, just ignore the line.  replace with an error?
      next unless defined $date;
      next unless $date =~ /^$RELEVANT_DATE_REGEX/;

      # turn 'xyz: bo' into 'xyz' and 'bo'
      my ($service, $string) = split_message( $message);

      # if $service isn't defined, check to see whether the message is a
      # 'special' message.  otherwise, bail.
      if ( ! $service
	   and $message =~ /\blast message repeated (\d+) times?$/) {
	$LINES{$last_key}+=$1 if defined $last_key;
	next;
      } elsif ( ! $service ) {
	warn "?? $_\n";
	next;
      }

      # this is definitely a real message.  assign it a tentative priority.
      my $priority = 0;
      $last_key = undef;

      # if it's a 'bad' message
      if ( $string =~ /$BAD_STRING_REGEX/io ) {
	# and we don't ignore the bad messages'
	next if ( $string =~ /$ACKNOWLEDGED_BAD_STRING_REGEX/io );
	# give it a high priority
	$priority += 50;

      } else {
	# if this service doesn't print out a lot of messages
	next if $service =~ /$SERVICE_ONLY_BAD_REGEX/io;

	# and the message coming is not a 'good' message
	if ( $string =~ /$GOOD_STRING_REGEX/io) {
	  next;
	}

	# assign it a low priority, but eventually print it
	$priority -=50;
      }

      # fixup weird service names like 'sshd2[12345]'
      service_fixup( $service);
      # fixup strings, e.g. 'aoinasoijf23: message' => 'message'
      string_fixup( $string);
      # store the message.
      $last_key = "$service: $string";
      $LINES{$last_key}++;
      $PRIORITY{$last_key} ||= $priority;
    }
  }

  my $pri_register;		# remember the priority we're processing
  my $messages_processed = 0;	# for statistics, number of *syslog messages*
				# 'last message repeated 1000 times' => +1000

  foreach my $line
    ( map { $_->[0] }		# Schwartzian transform
      sort { $b->[1] <=> $a->[1]            # first sort by priority
	       or $a->[2] cmp $b->[2]       # then service name
		 or $a->[0] cmp $b->[0]     # then message
	     }
      map { [ $_, $PRIORITY{$_}, /^(?:\S+\s+)?([^:]+)/ ] }
      keys %LINES
    ) {
      $messages_processed+=$LINES{$line};

      # print out some phatty '----' priority separator messages
      if ( ! defined $pri_register
	   or $PRIORITY{$line} != $pri_register) {
	$pri_register = $PRIORITY{$line};
	my $pri_str = " PRI $pri_register ";
	my $dash_count = (72 -length($pri_str)) / 2;
	print '-'x$dash_count, $pri_str, '-'x$dash_count,"\n";
      }
      print $line;
      # print out a repeat number if this message has been seen more than once
      print " (x$LINES{$line})"
	if $LINES{$line}>1;
      print "\n";
  }
  print '-' x 72, "\n";

  print "Total lines seen: $lines_seen  Messages processed: $messages_processed\n";
}

sub service_fixup {
  # remove [123] (process ID) from service names
  foreach (@_) {
    s/\[\d+\]//;
  }
}

sub string_fixup {
  # remove a lot of unimportant unique string stuff, to reduce 100 similar
  # messages into one message w/ count of 100
  foreach (@_) {
    s/\blogin on tty\d+\s+//i;
    s/^(\w{12,16}: )+//; #sendmail (maybe others) uses 12-char IDs w/ colon
    s/(failed login )\d* from \([^)]+\) (for)/lc($1 .$2)/ei;
    s/^.*(authentication failure)s?; logname=(\S+).*$/$1: $2/i;
    s/ -- renewal in \d+ .*//;
    s/, delay=[\d:+]+,.*?stat=/, stat=/;
  }
}

sub split_time {
  # called by process_message_logs, split into 2 fields: date and the rest
  $_[0] =~ /^(.*?:\d{2}(?: [AP]M)?) (.*)/;
}

sub split_message {
  # called by process_message_logs, split into 2 fields: service and the rest
  $_[0] =~ /([^:]+): (.*)/;
}