#!/usr/bin/perl
#
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Library General Public License for more details.
#
# You should have received a copy of the GNU General Public
# License along with this library; if not, write to the Free
# Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
#
# Dumps ipfm (IP flowmeter) traffic flow data into a PostgreSQL database
#
# Version: $Id: ipfm2psql,v 1.15 2004/05/26 20:57:19 als Exp $
#
# Author: Alexander Schreiber <als@thangorodrim.de>
#
#
# SQL to create the needed table:
# CREATE TABLE trafficlog(id serial, date date, time time, 
# host varchar, incoming numeric(12,0), outgoing numeric(12,0));
#
# version number: 0.5
#


use DBI;


$CONFIG_FILE = '/etc/ipfm2psql.conf';

sub db_connect {

    my $dsn = shift;
    my $user = shift;
    my $password = shift;

    my $dbh;

    $dbh = DBI->connect($dsn, $user, $password,
                        { 
                            RaiseError => 0, 
                            PrintError => 1,
                            ShowErrorStatement => 1,
                            AutoCommit => 1 
                            });
    
    if ( not defined ($dbh) ) {
        print STDERR "Failed to connect to database ($dsn)!\n";
        return -1;
    } else {
        $dbh->{HandleError} = sub {
            print STDERR "database error: $DBI::errstr\n";
            unlink($Config{'LockFile'});
        };
        return $dbh;
    }
}


sub insert_row {
# inserts one row of data
# parameters: conn, host (FQDN/IP), timestamp ("YYYY-MM-YY HH:MM:SS") in_byte,
#             out_byte
# returns 0 on success, -1 on failure

    my $conn = shift;
    my $host = shift;
    my $timestamp = shift;
    my $in_byte = shift;
    my $out_byte = shift;

    my $query;
    my $result;
    my $result_status;
    my $sth;


    my $date;
    my $time;

    ($date, $time) = split(/\s/, $timestamp);

# first, try to insert the host
    $query = "INSERT INTO hosts (hostname) values ('$host')";
    $sth = $conn->prepare($query);
    $sth->execute;

    $query  = "INSERT INTO trafficentries ";
    $query .= "(date, time, host, incoming, outgoing) ";
    $query .= " VALUES ('$date', '$time', ";
    $query .= "(select id from hosts where hostname = '$host'), ";
    $query .= "$in_byte, $out_byte)";

    $sth = $conn->prepare($query);
    $sth->execute;
}


sub process_file {
# process the IPFM log
# parameters: conn (DB handler), filename
# returns 0 on success, -1 on failure

    my $conn = shift;
    my $filename = shift;

    my $line;
    my $timestamp;
    my ($host, $in, $out, $total);
    my $result;
    my $error_msg;
    
    my $error = 0;

    $filename =~ /(\d\d\d\d-\d\d-\d\d)-T-(\d\d)-(\d\d)-(\d\d)/;
    $timestamp = "$1 $2:$3:$4";
    open(IPFM, "<$filename") or $error = 1;
    if ( $error == 1 ) {
        print STDERR "failed to open $filename for reading\n";
        return -1;
    }
# wrap writing the data of the file into a transaction - both for 
# atomicity of the log entries for this file and for a little speedup
# result: we either dump _all_ or _no_ log entries of this file into
# the database
# 
# That was the original approach. Unfortunately, due to sometimes 
# processing incomplete logs (usually the last 3 columns of data would be
# missing), this resulted logically in entire datasets not being dumped
# into the database. Now we complain to STDERR if we can't match a line,
# then ignore this line and carry on with the rest - which is normally
# the empty set, since these corrupted entries usually are the last line
# in the file.
    $conn->begin_work or die "Failed to setup transaction!\n";
 
    while ( $line = <IPFM> ) {
        unless ( $line =~ /^#/ ) { # skip comment lines
            $line =~ s/\n//;
            if ( $line =~ /(\S+)\s+(\d+)\s+(\d+)\s+(\d+)$/ ) {
                ($host, $in, $out, $total) = ($1, $2, $3, $4);
                $error = &insert_row($conn, $host, $timestamp, $in, $out);
                if ( $error == -1 ) {
                     $error_msg  = "failed to insert ";
                     $error_msg .= "|$host|$timestamp|$in|$out|\n";
                     print STDERR $error_msg;
                     $result = $conn->rollback;
                     return -1;
                }
            } else {
                print STDERR "ignoring invalid line |$line|\n"; 
            }
        }
    }

    $conn->commit or die "Failed to commit transaction!\n";
 
    close(IPFM);

    return 0;
}

sub process_directory {
# processes directory: reads in the directory file, drops all files that
# don't match the expected pattern, sorts whats left, feeds them one by one
# to process_file() and optionally deletes them after successfull processing
# parameters: db handle, directory
# returns 0 on success, -1 on failure

    my $conn = shift;
    my $directory = shift;

    my $dir_entry;
    my @file_list;
    my $work;
    my $result;
    my $filename;

    my $error = 0;
    

    opendir(LOGDIR, $directory) or $error = 1;
    if ( $error == 1 ) {
        print STDERR "failed to open logdir $directory!\n";
        return -1;
    }

    while ( $dir_entry = readdir(LOGDIR) ) {
        if ( $dir_entry =~ /^ipfm-\d\d\d\d-\d\d-\d\d-T-\d\d-\d\d-\d\d$/ ) {
            push(@file_list, $dir_entry);
        }
    }
    @file_list = sort(@file_list);

    foreach $work ( @file_list) {
        $filename = "$directory/$work";
        $result = &process_file($conn, $filename);
        if ( $result == -1 ) {
            print STDERR "failed to process file $work!\n";
            return -1;
        } else { 
            if ( $Config{'DeleteAfterProcess'} eq 'yes' ) {
                $result = unlink($filename);
                unless ( $result == 1 ) {
                    print STDERR "failed to delete $filename!\n";
                }
            }
        }
    }

    return 0;
}



sub load_config {
# loads configuration from file
# no parameters, uses fixed value CONFIG_FILE
# return 0 on success, -1 on failure
# loads configuration into global hash %Config

    my $line;
    my $parameter;
    my $value;

    my $error = 0;

    open(CONFIG, "<$CONFIG_FILE") or $error = 1;
    if ( $error == 1 ) {
        print STDERR "Failed to load configfile $CONFIG_FILE!\n";
        return -1;
    }
 
    while ( $line = <CONFIG> ) {
        unless ( $line =~ /^#/ ) { # skip over comment lines
            $line =~ s/\n//;
            $line =~ /(\S+)\s*=\s*(\S+)/;
            $parameter = $1;
            $value = $2;
            $Config{$parameter} = $value;
        }
    }
    

    close(CONFIG);

    return 0;
}

sub get_ISO8601_timestamp {
# creates an ISO8601 timestamp
# parameters: $time (optional, as seconds since the epoch (time_t)
#             if empty, NOW is assumed
# call: $time_stamp = &get_ISO8601_timestamp() 
#   or  $time_stamp = &get_ISO8601_timestamp($time_t)
# returns: current time as ISO8601 timestamp

    my $time = shift;

    my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst);
    my $timestamp;

    unless ( defined($time)) {
        $time = '';
    }

    if ( $time eq '' ) {
        ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) =
                                                             localtime(time);
    } else {
        ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) =
                                                             localtime($time);
    }

    $year += 1900;
    $mon++;

    $timestamp = sprintf("%04d-%02d-%02dT%02d:%02d:%02d", $year, $mon,
                         $mday, $hour, $min, $sec);

    return $timestamp;
}


sub try_lock {
# tries to get the lock for running ipfm2psql
# parameters: none
# on success: returns 0
# on failure: returns -1, prints error message to stderr

    my $success;
    my $timestamp;
    my $line;
    my ($pid, $time);
    my $ret_val;

    $success = 1;
    open(LOCKFILE, "<$Config{'LockFile'}") or $success = 0;
    if ( $success == 1 ) {    # lock exists
        $line = <LOCKFILE>;
        chomp($line);
        ($pid, $time) = split('|', $line);
        print STDERR "lock still held by PID $pid, time $time\n";
        close(LOCKFILE);
        return -1;
    } else {                  # lock does not exist
        $success = 1;
        open(LOCKFILE, ">$Config{'LockFile'}") or $success = 0;
        if ( $success == 1 ){ # created lockfile
            $timestamp = &get_ISO8601_timestamp();
            print LOCKFILE "$$|$timestamp\n";
            close(LOCKFILE);
            return 0;
        } else {              # failed to create lockfile
            print STDERR "failed to create lockfile $Config{'LockFile'}!\n";
            return -1;
        }
    }
}



# ##MAIN##

my $db_conn;
my $dsn;
my $lock_state;

&load_config();

$lock_state = &try_lock;
if ( $lock_state == -1 ) {
    print STDERR "unable to aquire lock, exiting\n";
    exit(2);
}

$dsn = "dbi:Pg:dbname=$Config{DBName} host=$Config{'DBHost'}";
$db_conn = &db_connect($dsn, $Config{'DBUser'}, $Config{'DBPassword'});


if ( $db_conn == -1 ) {
    print "db connect failed!\n";
    exit(1);
}


# we always process the entire directory in one run
&process_directory($db_conn, $Config{'LogDir'});
$db_conn->disconnect;

unlink($Config{'LockFile'});

