2020-01-24 12:00:49 +01:00
package logparser
2020-01-27 16:07:09 +01:00
import (
"fmt"
2020-02-03 16:00:00 +01:00
"html/template"
2020-02-07 15:55:30 +01:00
"io/ioutil"
2020-01-30 17:31:47 +01:00
"log"
"math"
"os"
2020-01-31 11:44:01 +01:00
"path/filepath"
2020-01-27 16:07:09 +01:00
"regexp"
"strconv"
2020-01-30 17:31:47 +01:00
"strings"
2020-01-27 16:07:09 +01:00
"time"
"github.com/gomodule/redigo/redis"
2020-01-30 17:31:47 +01:00
"gonum.org/v1/plot"
"gonum.org/v1/plot/plotter"
"gonum.org/v1/plot/plotutil"
"gonum.org/v1/plot/vg"
2020-01-27 16:07:09 +01:00
)
2020-01-24 12:00:49 +01:00
// SshdParser Holds a struct that corresponds to a sshd log line
// and the redis connection
type SshdParser struct {
2020-02-05 10:45:05 +01:00
// Write
2020-01-30 17:31:47 +01:00
r1 * redis . Conn
2020-02-05 10:45:05 +01:00
// Read
2020-01-30 17:31:47 +01:00
r2 * redis . Conn
2020-01-24 12:00:49 +01:00
}
2020-01-28 10:48:19 +01:00
// Set set the redic connection to this parser
2020-01-30 17:31:47 +01:00
func ( s * SshdParser ) Set ( rconn1 * redis . Conn , rconn2 * redis . Conn ) {
s . r1 = rconn1
s . r2 = rconn2
2020-01-24 12:00:49 +01:00
}
2020-02-05 10:45:05 +01:00
// Flush recomputes statistics and recompile HTML output
func ( s * SshdParser ) Flush ( ) error {
log . Println ( "Flushing" )
r1 := * s . r1
r0 := * s . r2
// writing in database 1
if _ , err := r1 . Do ( "SELECT" , 1 ) ; err != nil {
r0 . Close ( )
r1 . Close ( )
return err
}
// flush stats DB
if _ , err := r1 . Do ( "FLUSHDB" ) ; err != nil {
r0 . Close ( )
r1 . Close ( )
return err
}
log . Println ( "Statistics Database Flushed" )
// reading from database 0
if _ , err := r0 . Do ( "SELECT" , 0 ) ; err != nil {
r0 . Close ( )
r1 . Close ( )
return err
}
// Compile statistics / html output for each line
keys , err := redis . Strings ( r0 . Do ( "KEYS" , "*" ) )
if err != nil {
r0 . Close ( )
r1 . Close ( )
return err
}
for _ , v := range keys {
dateHost := strings . Split ( v , ":" )
kkeys , err := redis . StringMap ( r0 . Do ( "HGETALL" , v ) )
if err != nil {
r0 . Close ( )
r1 . Close ( )
return err
}
dateInt , err := strconv . ParseInt ( dateHost [ 0 ] , 10 , 64 )
if err != nil {
r0 . Close ( )
r1 . Close ( )
return err
}
parsedTime := time . Unix ( dateInt , 0 )
err = compileStats ( s , parsedTime , kkeys [ "src" ] , kkeys [ "username" ] , dateHost [ 1 ] )
if err != nil {
r0 . Close ( )
r1 . Close ( )
return err
}
}
return nil
}
2020-01-24 12:00:49 +01:00
// Parse parses a line of sshd log
2020-01-27 16:07:09 +01:00
func ( s * SshdParser ) Parse ( logline string ) error {
2020-01-30 17:31:47 +01:00
r := * s . r1
2020-02-03 09:39:37 +01:00
re := regexp . MustCompile ( ` ^(?P<date>[[:alpha:]] { 3} { 1,2}\d { 1,2}\s\d { 2}:\d { 2}:\d { 2}) (?P<host>[^ ]+) sshd\[[[:alnum:]]+\]: Invalid user (?P<username>.*) from (?P<src>.*$) ` )
2020-01-27 16:07:09 +01:00
n1 := re . SubexpNames ( )
r2 := re . FindAllStringSubmatch ( logline , - 1 ) [ 0 ]
// Build the group map for the line
md := map [ string ] string { }
for i , n := range r2 {
// fmt.Printf("%d. match='%s'\tname='%s'\n", i, n, n1[i])
md [ n1 [ i ] ] = n
}
// Assumes the system parses logs recorded during the current year
md [ "date" ] = fmt . Sprintf ( "%v %v" , md [ "date" ] , time . Now ( ) . Year ( ) )
2020-02-03 09:39:37 +01:00
// TODO Make this automatic or a config parameter
2020-01-27 16:07:09 +01:00
loc , _ := time . LoadLocation ( "Europe/Luxembourg" )
2020-02-03 09:39:37 +01:00
parsedTime , _ := time . ParseInLocation ( "Jan 2 15:04:05 2006" , md [ "date" ] , loc )
2020-01-27 16:07:09 +01:00
md [ "date" ] = string ( strconv . FormatInt ( parsedTime . Unix ( ) , 10 ) )
2020-01-28 10:48:19 +01:00
// Pushing loglines in database 0
if _ , err := r . Do ( "SELECT" , 0 ) ; err != nil {
r . Close ( )
return err
}
2020-02-05 10:45:05 +01:00
// Writing logs
2020-01-28 10:48:19 +01:00
_ , err := redis . Bool ( r . Do ( "HSET" , fmt . Sprintf ( "%v:%v" , md [ "date" ] , md [ "host" ] ) , "username" , md [ "username" ] , "src" , md [ "src" ] ) )
2020-01-27 16:07:09 +01:00
if err != nil {
2020-01-28 10:48:19 +01:00
r . Close ( )
return err
}
2020-02-05 10:45:05 +01:00
err = compileStats ( s , parsedTime , md [ "src" ] , md [ "username" ] , md [ "host" ] )
if err != nil {
r . Close ( )
return err
}
return nil
}
func compileStats ( s * SshdParser , parsedTime time . Time , src string , username string , host string ) error {
r := * s . r1
2020-01-28 10:48:19 +01:00
// Pushing statistics in database 1
if _ , err := r . Do ( "SELECT" , 1 ) ; err != nil {
r . Close ( )
return err
2020-01-27 16:07:09 +01:00
}
2020-01-31 14:44:27 +01:00
// Daily
2020-02-03 09:54:39 +01:00
dstr := fmt . Sprintf ( "%v%v%v" , parsedTime . Year ( ) , fmt . Sprintf ( "%02d" , int ( parsedTime . Month ( ) ) ) , fmt . Sprintf ( "%02d" , int ( parsedTime . Day ( ) ) ) )
2020-02-03 14:47:44 +01:00
// Check current entry date as oldest if older than the current
2020-02-05 10:45:05 +01:00
if oldest , err := redis . String ( r . Do ( "GET" , "oldest" ) ) ; err == redis . ErrNil {
2020-02-03 14:47:44 +01:00
r . Do ( "SET" , "oldest" , dstr )
} else if err != nil {
r . Close ( )
return err
} else {
// Check if dates are the same
if oldest != dstr {
// Check who is the oldest
2020-02-03 16:00:00 +01:00
parsedOldest , _ := time . Parse ( "20060102" , oldest )
2020-02-03 14:47:44 +01:00
if parsedTime . Before ( parsedOldest ) {
r . Do ( "SET" , "oldest" , dstr )
}
}
}
// Check current entry date as oldest if older than the current
2020-02-05 10:45:05 +01:00
if newest , err := redis . String ( r . Do ( "GET" , "newest" ) ) ; err == redis . ErrNil {
2020-02-03 14:47:44 +01:00
r . Do ( "SET" , "newest" , dstr )
} else if err != nil {
r . Close ( )
return err
} else {
// Check if dates are the same
if newest != dstr {
// Check who is the newest
2020-02-03 16:00:00 +01:00
parsedNewest , _ := time . Parse ( "20060102" , newest )
2020-02-03 14:47:44 +01:00
if parsedTime . After ( parsedNewest ) {
r . Do ( "SET" , "newest" , dstr )
}
}
}
2020-02-05 10:45:05 +01:00
err := compileStat ( s , dstr , "daily" , src , username , host )
2020-01-27 16:07:09 +01:00
if err != nil {
2020-01-28 10:48:19 +01:00
r . Close ( )
return err
2020-01-27 16:07:09 +01:00
}
2020-01-31 14:44:27 +01:00
// Monthly
2020-02-03 09:54:39 +01:00
mstr := fmt . Sprintf ( "%v%v" , parsedTime . Year ( ) , fmt . Sprintf ( "%02d" , int ( parsedTime . Month ( ) ) ) )
2020-02-05 10:45:05 +01:00
err = compileStat ( s , mstr , "daily" , src , username , host )
2020-01-27 16:07:09 +01:00
if err != nil {
2020-01-28 10:48:19 +01:00
r . Close ( )
return err
2020-01-27 16:07:09 +01:00
}
2020-01-31 14:44:27 +01:00
// Yearly
ystr := fmt . Sprintf ( "%v" , parsedTime . Year ( ) )
2020-02-05 10:45:05 +01:00
err = compileStat ( s , ystr , "daily" , src , username , host )
2020-01-27 16:07:09 +01:00
if err != nil {
2020-01-28 10:48:19 +01:00
r . Close ( )
return err
2020-01-27 16:07:09 +01:00
}
2020-01-31 14:44:27 +01:00
return nil
}
2020-02-05 10:45:05 +01:00
func compileStat ( s * SshdParser , datestr string , mode string , src string , username string , host string ) error {
2020-01-31 14:44:27 +01:00
r := * s . r1
_ , err := redis . String ( r . Do ( "ZINCRBY" , fmt . Sprintf ( "%v:%v" , datestr , "statssrc" ) , 1 , src ) )
2020-01-30 17:31:47 +01:00
if err != nil {
r . Close ( )
return err
}
2020-01-31 14:44:27 +01:00
_ , err = redis . String ( r . Do ( "ZINCRBY" , fmt . Sprintf ( "%v:%v" , datestr , "statsusername" ) , 1 , username ) )
2020-01-30 17:31:47 +01:00
if err != nil {
r . Close ( )
return err
}
2020-01-31 14:44:27 +01:00
_ , err = redis . String ( r . Do ( "ZINCRBY" , fmt . Sprintf ( "%v:%v" , datestr , "statshost" ) , 1 , host ) )
if err != nil {
r . Close ( )
return err
}
_ , err = redis . Int ( r . Do ( "SADD" , fmt . Sprintf ( "toupdate:%v" , mode ) , fmt . Sprintf ( "%v:%v" , datestr , "statssrc" ) ) )
if err != nil {
r . Close ( )
return err
}
_ , err = redis . Int ( r . Do ( "SADD" , fmt . Sprintf ( "toupdate:%v" , mode ) , fmt . Sprintf ( "%v:%v" , datestr , "statsusername" ) ) )
if err != nil {
r . Close ( )
return err
}
_ , err = redis . Int ( r . Do ( "SADD" , fmt . Sprintf ( "toupdate:%v" , mode ) , fmt . Sprintf ( "%v:%v" , datestr , "statshost" ) ) )
2020-01-30 17:31:47 +01:00
if err != nil {
r . Close ( )
return err
}
return nil
}
// Compile create graphs of the results
func ( s * SshdParser ) Compile ( ) error {
r := * s . r2
// Pulling statistics from database 1
if _ , err := r . Do ( "SELECT" , 1 ) ; err != nil {
r . Close ( )
return err
}
2020-01-31 14:44:27 +01:00
// List days for which we need to update statistics
toupdateD , err := redis . Strings ( r . Do ( "SMEMBERS" , "toupdate:daily" ) )
2020-01-30 17:31:47 +01:00
if err != nil {
r . Close ( )
return err
}
2020-01-31 14:44:27 +01:00
// Plot statistics for each day to update
for _ , v := range toupdateD {
err = plotStats ( s , v )
2020-01-30 17:31:47 +01:00
if err != nil {
r . Close ( )
return err
}
2020-01-31 14:44:27 +01:00
}
2020-01-30 17:31:47 +01:00
2020-01-31 14:44:27 +01:00
// List months for which we need to update statistics
toupdateM , err := redis . Strings ( r . Do ( "SMEMBERS" , "toupdate:monthly" ) )
if err != nil {
r . Close ( )
return err
}
2020-01-30 17:31:47 +01:00
2020-01-31 14:44:27 +01:00
// Plot statistics for each month to update
for _ , v := range toupdateM {
err = plotStats ( s , v )
2020-01-30 17:31:47 +01:00
if err != nil {
2020-01-31 14:44:27 +01:00
r . Close ( )
return err
2020-01-30 17:31:47 +01:00
}
2020-01-31 14:44:27 +01:00
}
2020-01-30 17:31:47 +01:00
2020-01-31 14:44:27 +01:00
// List years for which we need to update statistics
toupdateY , err := redis . Strings ( r . Do ( "SMEMBERS" , "toupdate:yearly" ) )
if err != nil {
r . Close ( )
return err
}
2020-01-30 17:31:47 +01:00
2020-01-31 14:44:27 +01:00
// Plot statistics for each year to update
for _ , v := range toupdateY {
err = plotStats ( s , v )
2020-01-30 17:31:47 +01:00
if err != nil {
2020-01-31 14:44:27 +01:00
r . Close ( )
2020-01-30 17:31:47 +01:00
return err
}
2020-01-31 14:44:27 +01:00
}
2020-02-03 16:00:00 +01:00
// Get oldest / newest entries
var newest string
var oldest string
if newest , err = redis . String ( r . Do ( "GET" , "newest" ) ) ; err == redis . ErrNil {
r . Close ( )
return err
}
if oldest , err = redis . String ( r . Do ( "GET" , "oldest" ) ) ; err == redis . ErrNil {
r . Close ( )
return err
}
parsedOldest , _ := time . Parse ( "20060102" , oldest )
parsedNewest , _ := time . Parse ( "20060102" , newest )
parsedOldestStr := parsedOldest . Format ( "2006-01-02" )
parsedNewestStr := parsedNewest . Format ( "2006-01-02" )
2020-02-05 16:37:01 +01:00
// Gettings list of years for which we have statistics
reply , err := redis . Values ( r . Do ( "SCAN" , "0" , "MATCH" , "????:*" , "COUNT" , 1000 ) )
if err != nil {
r . Close ( )
return err
}
var cursor int64
var items [ ] string
_ , err = redis . Scan ( reply , & cursor , & items )
if err != nil {
r . Close ( )
return err
}
var years [ ] string
for _ , v := range items {
yearSplit := strings . Split ( v , ":" )
found := false
for _ , y := range years {
if y == yearSplit [ 0 ] {
found = true
}
}
if ! found {
years = append ( years , yearSplit [ 0 ] )
}
}
// Gettings list of months for which we have statistics
months := make ( map [ string ] [ ] string )
for _ , v := range years {
var mraw [ ] string
reply , err = redis . Values ( r . Do ( "SCAN" , "0" , "MATCH" , v + "??:*" , "COUNT" , 1000 ) )
2020-02-03 16:00:00 +01:00
if err != nil {
2020-02-05 16:37:01 +01:00
r . Close ( )
return err
}
_ , err = redis . Scan ( reply , & cursor , & mraw )
if err != nil {
r . Close ( )
return err
}
for _ , m := range mraw {
m = strings . TrimPrefix ( m , v )
monthSplit := strings . Split ( m , ":" )
found := false
for _ , y := range months [ v ] {
if y == monthSplit [ 0 ] {
found = true
}
}
if ! found {
months [ v ] = append ( months [ v ] , monthSplit [ 0 ] )
}
2020-02-03 16:00:00 +01:00
}
}
2020-02-05 16:37:01 +01:00
2020-02-07 15:55:30 +01:00
// Parse Template
t , err := template . ParseFiles ( filepath . Join ( "logparser" , "sshd" , "statistics.gohtml" ) )
2020-02-05 16:37:01 +01:00
if err != nil {
r . Close ( )
return err
}
2020-02-03 16:00:00 +01:00
2020-02-07 15:55:30 +01:00
daily := struct {
Title string
Current string
MinDate string
MaxDate string
CurrentTime string
} {
Title : "sshd failed logins - daily statistics" ,
MinDate : parsedOldestStr ,
MaxDate : parsedNewestStr ,
Current : newest ,
CurrentTime : parsedNewestStr ,
}
monthly := struct {
Title string
MonthList map [ string ] [ ] string
CurrentTime string
Current string
2020-02-03 16:00:00 +01:00
} {
2020-02-07 15:55:30 +01:00
Title : "sshd failed logins - monthly statistics" ,
MonthList : months ,
CurrentTime : years [ 0 ] + months [ years [ 0 ] ] [ 0 ] ,
Current : years [ 0 ] + months [ years [ 0 ] ] [ 0 ] ,
}
yearly := struct {
Title string
YearList [ ] string
Current string
CurrentTime string
} {
Title : "sshd failed logins - yearly statistics" ,
YearList : years ,
Current : years [ 0 ] ,
CurrentTime : years [ 0 ] ,
}
// Create folder to store resulting files
if _ , err := os . Stat ( "data" ) ; os . IsNotExist ( err ) {
err := os . Mkdir ( "data" , 0700 )
if err != nil {
r . Close ( )
return err
}
}
if _ , err := os . Stat ( filepath . Join ( "data" , "sshd" ) ) ; os . IsNotExist ( err ) {
err := os . Mkdir ( filepath . Join ( "data" , "sshd" ) , 0700 )
if err != nil {
r . Close ( )
return err
}
}
_ = os . Remove ( filepath . Join ( "data" , "sshd" , "dailystatistics.html" ) )
_ = os . Remove ( filepath . Join ( "data" , "sshd" , "monthlystatistics.html" ) )
_ = os . Remove ( filepath . Join ( "data" , "sshd" , "yearlystatistics.html" ) )
f , err := os . OpenFile ( filepath . Join ( "data" , "sshd" , "dailystatistics.html" ) , os . O_RDWR | os . O_CREATE , 0666 )
defer f . Close ( )
// err = t.Execute(f, daily)
err = t . ExecuteTemplate ( f , "headertpl" , daily )
err = t . ExecuteTemplate ( f , "dailytpl" , daily )
err = t . ExecuteTemplate ( f , "footertpl" , daily )
if err != nil {
r . Close ( )
return err
}
f , err = os . OpenFile ( filepath . Join ( "data" , "sshd" , "monthlystatistics.html" ) , os . O_RDWR | os . O_CREATE , 0666 )
defer f . Close ( )
// err = t.Execute(f, monthly)
err = t . ExecuteTemplate ( f , "headertpl" , monthly )
err = t . ExecuteTemplate ( f , "monthlytpl" , monthly )
err = t . ExecuteTemplate ( f , "footertpl" , monthly )
if err != nil {
r . Close ( )
return err
}
f , err = os . OpenFile ( filepath . Join ( "data" , "sshd" , "yearlystatistics.html" ) , os . O_RDWR | os . O_CREATE , 0666 )
2020-02-03 16:00:00 +01:00
defer f . Close ( )
2020-02-07 15:55:30 +01:00
// err = t.Execute(f, yearly)
err = t . ExecuteTemplate ( f , "headertpl" , yearly )
err = t . ExecuteTemplate ( f , "yearlytpl" , yearly )
err = t . ExecuteTemplate ( f , "footertpl" , yearly )
2020-02-05 16:37:01 +01:00
if err != nil {
r . Close ( )
return err
}
2020-02-03 16:00:00 +01:00
2020-02-07 15:55:30 +01:00
// Copy js asset file
input , err := ioutil . ReadFile ( filepath . Join ( "logparser" , "sshd" , "load.js" ) )
if err != nil {
log . Println ( err )
}
err = ioutil . WriteFile ( filepath . Join ( "data" , "sshd" , "load.js" ) , input , 0644 )
if err != nil {
log . Println ( err )
}
2020-01-31 14:44:27 +01:00
return nil
}
2020-01-30 17:31:47 +01:00
2020-01-31 14:44:27 +01:00
func plotStats ( s * SshdParser , v string ) error {
r := * s . r2
zrank , err := redis . Strings ( r . Do ( "ZRANGEBYSCORE" , v , "-inf" , "+inf" , "WITHSCORES" ) )
if err != nil {
r . Close ( )
return err
}
2020-01-30 17:31:47 +01:00
2020-01-31 14:44:27 +01:00
// Split keys and values - keep these ordered
values := plotter . Values { }
keys := make ( [ ] string , 0 , len ( zrank ) / 2 )
2020-01-31 11:44:01 +01:00
2020-01-31 14:44:27 +01:00
for k , v := range zrank {
// keys
if ( k % 2 ) == 0 {
keys = append ( keys , zrank [ k ] )
// values
} else {
fv , _ := strconv . ParseFloat ( v , 64 )
values = append ( values , fv )
2020-01-31 11:44:01 +01:00
}
2020-01-31 14:44:27 +01:00
}
2020-01-31 11:44:01 +01:00
2020-01-31 14:44:27 +01:00
p , err := plot . New ( )
if err != nil {
2020-02-05 16:37:01 +01:00
r . Close ( )
return err
2020-01-31 14:44:27 +01:00
}
stype := strings . Split ( v , ":" )
switch stype [ 1 ] {
case "statsusername" :
p . Title . Text = "Usernames"
case "statssrc" :
p . Title . Text = "Source IP"
case "statshost" :
p . Title . Text = "Host"
default :
p . Title . Text = ""
log . Println ( "We should not reach this point, open an issue." )
}
p . Y . Label . Text = "Count"
w := 0.5 * vg . Centimeter
bc , err := plotter . NewBarChart ( values , w )
bc . Horizontal = true
if err != nil {
return err
}
bc . LineStyle . Width = vg . Length ( 0 )
bc . Color = plotutil . Color ( 0 )
p . Add ( bc )
p . NominalY ( keys ... )
// Create folder to store plots
if _ , err := os . Stat ( "data" ) ; os . IsNotExist ( err ) {
err := os . Mkdir ( "data" , 0700 )
if err != nil {
2020-02-05 16:37:01 +01:00
r . Close ( )
2020-01-31 14:44:27 +01:00
return err
2020-01-30 17:31:47 +01:00
}
2020-01-31 14:44:27 +01:00
}
2020-01-30 17:31:47 +01:00
2020-02-03 16:00:00 +01:00
if _ , err := os . Stat ( filepath . Join ( "data" , "sshd" ) ) ; os . IsNotExist ( err ) {
err := os . Mkdir ( filepath . Join ( "data" , "sshd" ) , 0700 )
if err != nil {
2020-02-05 16:37:01 +01:00
r . Close ( )
2020-02-03 16:00:00 +01:00
return err
}
}
if _ , err := os . Stat ( filepath . Join ( "data" , "sshd" , stype [ 0 ] ) ) ; os . IsNotExist ( err ) {
err := os . Mkdir ( filepath . Join ( "data" , "sshd" , stype [ 0 ] ) , 0700 )
2020-01-31 14:44:27 +01:00
if err != nil {
2020-02-05 16:37:01 +01:00
r . Close ( )
2020-01-30 17:31:47 +01:00
return err
}
2020-01-31 14:44:27 +01:00
}
2020-01-30 17:31:47 +01:00
2020-01-31 14:44:27 +01:00
xsize := 3 + vg . Length ( math . Round ( float64 ( len ( keys ) / 2 ) ) )
2020-02-03 16:00:00 +01:00
if err := p . Save ( 15 * vg . Centimeter , xsize * vg . Centimeter , filepath . Join ( "data" , "sshd" , stype [ 0 ] , fmt . Sprintf ( "%v.svg" , v ) ) ) ; err != nil {
2020-02-05 16:37:01 +01:00
r . Close ( )
2020-01-31 14:44:27 +01:00
return err
2020-01-30 17:31:47 +01:00
}
2020-01-24 12:00:49 +01:00
return nil
}