From 6746f5ede5b7ffb1b0bf03f5fb7a64b5a1582a6e Mon Sep 17 00:00:00 2001 From: Jean-Louis Huynen Date: Thu, 30 Jan 2020 17:31:47 +0100 Subject: [PATCH] chg: [sshd] svg graph generation --- .gitignore | 3 + conf.sample/redis_d4 | 1 + conf.sample/redis_parsers | 1 + go.mod | 3 + go.sum | 47 +++++++++++++++ logparser/parser.go | 3 +- logparser/sshd.go | 121 ++++++++++++++++++++++++++++++++++++-- main.go | 59 +++++++++++-------- 8 files changed, 210 insertions(+), 28 deletions(-) create mode 100644 conf.sample/redis_d4 create mode 100644 conf.sample/redis_parsers diff --git a/.gitignore b/.gitignore index 059e2d2..6442ccc 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,6 @@ analyzer-d4-log # Results data* + +# Redis backups +*.rdb diff --git a/conf.sample/redis_d4 b/conf.sample/redis_d4 new file mode 100644 index 0000000..5d6f103 --- /dev/null +++ b/conf.sample/redis_d4 @@ -0,0 +1 @@ +localhost:6380/2 diff --git a/conf.sample/redis_parsers b/conf.sample/redis_parsers new file mode 100644 index 0000000..1e8d1ba --- /dev/null +++ b/conf.sample/redis_parsers @@ -0,0 +1 @@ +localhost:6381/16 diff --git a/go.mod b/go.mod index 134cdf0..decf82a 100644 --- a/go.mod +++ b/go.mod @@ -5,4 +5,7 @@ go 1.13 require ( github.com/D4-project/d4-golang-utils v0.0.0-20200108150548-740f16240125 github.com/gomodule/redigo v2.0.0+incompatible + golang.org/x/image v0.0.0-20200119044424-58c23975cae1 // indirect + gonum.org/v1/netlib v0.0.0-20191229114700-bbb4dff026f8 // indirect + gonum.org/v1/plot v0.0.0-20200111075622-4abb28f724d5 ) diff --git a/go.sum b/go.sum index 5a928c3..dd2bdbe 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,52 @@ +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/D4-project/d4-golang-utils v0.0.0-20200108150548-740f16240125 h1:iv+hcdT+M0XJIDEoCtvk9HVvI8PgvbQNBtbEfCczCRI= github.com/D4-project/d4-golang-utils v0.0.0-20200108150548-740f16240125/go.mod h1:2rq8KBQnNNDocwc/49cnpaqoQA/komoSHKom7ynvqJc= +github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af h1:wVe6/Ea46ZMeNkQjjBW6xcqyQA/j5e0D6GytH95g0gQ= +github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= +github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= +github.com/fogleman/gg v1.3.0 h1:/7zJX8F6AaYQc57WQCyN9cAIz+4bCJGO9B+dyW29am8= +github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 h1:DACJavvAHhabrF08vX0COfcOBJRhZ8lUbR+ZWIs0Y5g= +github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/gomodule/redigo v2.0.0+incompatible h1:K/R+8tc58AaqLkqG2Ol3Qk+DR/TlNuhuh457pBFPtt0= github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= +github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5 h1:PJr+ZMXIecYc1Ey2zucXdR73SMBtgjPgwa31099IMv0= +github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= +github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f h1:9kQ594xxPWRNKfTOnPjPcgrIJ19zM3ic57aI7PbMyAA= +golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190312203227-4b39c73a6495 h1:I6A9Ag9FpEKOjcKrRNjQkPHawoXIhKyTGfvvjFAiiAk= +golang.org/x/exp v0.0.0-20190312203227-4b39c73a6495/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20200119044424-58c23975cae1 h1:5h3ngYt7+vXCDZCup/HkCQgW5XwmSvR/nA2JmJ0RErg= +golang.org/x/image v0.0.0-20200119044424-58c23975cae1/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4 h1:nYxTaCPaVoJbxx+vMVnsFb6kw5+6aJCx52m/lmM/Vog= +gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= +gonum.org/v1/gonum v0.6.2 h1:4r+yNT0+8SWcOkXP+63H2zQbN+USnC73cjGUxnDF94Q= +gonum.org/v1/gonum v0.6.2/go.mod h1:9mxDZsDKxgMAuccQkewq682L+0eCu4dCN2yonUJTCLU= +gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= +gonum.org/v1/netlib v0.0.0-20191229114700-bbb4dff026f8 h1:kHY67jAKYewKUCz9YdNDa7iLAJ2WfNmoHzCCX4KnA8w= +gonum.org/v1/netlib v0.0.0-20191229114700-bbb4dff026f8/go.mod h1:2IgXn/sJaRbePPBA1wRj8OE+QLvVaH0q8SK6TSTKlnk= +gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= +gonum.org/v1/plot v0.0.0-20200111075622-4abb28f724d5 h1:edjGU2UC8RjriRJNETPkc6w4Y44FqlvPQEal6JdckTw= +gonum.org/v1/plot v0.0.0-20200111075622-4abb28f724d5/go.mod h1:+HbaZVpsa73UwN7kXGCECULRHovLRJjH+t5cFPgxErs= +modernc.org/cc v1.0.0/go.mod h1:1Sk4//wdnYJiUIxnW8ddKpaOJCF37yAdqYnkxUpaYxw= +modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk= +modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k= +modernc.org/strutil v1.1.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs= +modernc.org/xc v1.0.0/go.mod h1:mRNCo0bvLjGhHO9WsyuKVU4q0ceiDDDoEeWDJHrNx8I= +rsc.io/pdf v0.1.1 h1:k1MczvYDUvJBe93bYd7wrZLLUEcLZAuF824/I4e5Xr4= +rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= diff --git a/logparser/parser.go b/logparser/parser.go index 22f268b..98aa61a 100644 --- a/logparser/parser.go +++ b/logparser/parser.go @@ -8,7 +8,8 @@ type ( // Set to assign a redis connection to it // Parse to parse a line of log Parser interface { - Set(*redis.Conn) + Set(*redis.Conn, *redis.Conn) Parse(string) error + Compile() error } ) diff --git a/logparser/sshd.go b/logparser/sshd.go index c71d108..5fd529a 100644 --- a/logparser/sshd.go +++ b/logparser/sshd.go @@ -2,27 +2,37 @@ package logparser import ( "fmt" + "log" + "math" + "os" "regexp" "strconv" + "strings" "time" "github.com/gomodule/redigo/redis" + "gonum.org/v1/plot" + "gonum.org/v1/plot/plotter" + "gonum.org/v1/plot/plotutil" + "gonum.org/v1/plot/vg" ) // SshdParser Holds a struct that corresponds to a sshd log line // and the redis connection type SshdParser struct { - r *redis.Conn + r1 *redis.Conn + r2 *redis.Conn } // Set set the redic connection to this parser -func (s *SshdParser) Set(rconn *redis.Conn) { - s.r = rconn +func (s *SshdParser) Set(rconn1 *redis.Conn, rconn2 *redis.Conn) { + s.r1 = rconn1 + s.r2 = rconn2 } // Parse parses a line of sshd log func (s *SshdParser) Parse(logline string) error { - r := *s.r + r := *s.r1 re := regexp.MustCompile(`^(?P[[:alpha:]]{3}\s\d{2}\s\d{2}:\d{2}:\d{2}) (?P[^ ]+) sshd\[[[:alnum:]]+\]: Invalid user (?P[^ ]+) from (?P.*$)`) n1 := re.SubexpNames() r2 := re.FindAllStringSubmatch(logline, -1)[0] @@ -73,5 +83,108 @@ func (s *SshdParser) Parse(logline string) error { return err } + // Keeping track of which days we updated statistics for + _, err = redis.Int(r.Do("SADD", "toupdate", fmt.Sprintf("%v%v%v:statssrc", parsedTime.Year(), int(parsedTime.Month()), parsedTime.Day()))) + if err != nil { + r.Close() + return err + } + _, err = redis.Int(r.Do("SADD", "toupdate", fmt.Sprintf("%v%v%v:statsusername", parsedTime.Year(), int(parsedTime.Month()), parsedTime.Day()))) + if err != nil { + r.Close() + return err + } + _, err = redis.Int(r.Do("SADD", "toupdate", fmt.Sprintf("%v%v%v:statshost", parsedTime.Year(), int(parsedTime.Month()), parsedTime.Day()))) + if err != nil { + r.Close() + return err + } + + return nil +} + +// Compile create graphs of the results +func (s *SshdParser) Compile() error { + r := *s.r2 + + // Pulling statistics from database 1 + if _, err := r.Do("SELECT", 1); err != nil { + r.Close() + return err + } + + // List days for which we need to update statistic + toupdate, err := redis.Strings(r.Do("SMEMBERS", "toupdate")) + if err != nil { + r.Close() + return err + } + + // Query statistics dor each day to update + for _, v := range toupdate { + zrank, err := redis.Strings(r.Do("ZRANGEBYSCORE", v, "-inf", "+inf", "WITHSCORES")) + if err != nil { + r.Close() + return err + } + + // Split keys and values - keep these ordered + values := plotter.Values{} + keys := make([]string, 0, len(zrank)/2) + + for k, v := range zrank { + // keys + if (k % 2) == 0 { + keys = append(keys, zrank[k]) + // values + } else { + fv, _ := strconv.ParseFloat(v, 64) + values = append(values, fv) + } + } + + p, err := plot.New() + if err != nil { + panic(err) + } + + stype := strings.Split(v, ":") + switch stype[1] { + case "statsusername": + p.Title.Text = "Usernames" + case "statssrc": + p.Title.Text = "Source IP" + case "statshost": + p.Title.Text = "Host" + default: + p.Title.Text = "" + log.Println("We should not reach this point, open an issue.") + } + + p.Y.Label.Text = "Count" + w := 0.5 * vg.Centimeter + bc, err := plotter.NewBarChart(values, w) + bc.Horizontal = true + if err != nil { + return err + } + bc.LineStyle.Width = vg.Length(0) + bc.Color = plotutil.Color(0) + + p.Add(bc) + p.NominalY(keys...) + + // Create folder to store plots + if _, err := os.Stat(stype[0]); os.IsNotExist(err) { + os.Mkdir(stype[0], 0700) + } + + xsize := 3 + vg.Length(math.Round(float64(len(keys)/2))) + if err := p.Save(15*vg.Centimeter, xsize*vg.Centimeter, fmt.Sprintf("data/%v/%v.svg", stype[0], v)); err != nil { + return err + } + + } + return nil } diff --git a/main.go b/main.go index 9ca84b8..ccf5f81 100644 --- a/main.go +++ b/main.go @@ -1,6 +1,7 @@ package main import ( + "errors" "flag" "fmt" "log" @@ -11,8 +12,6 @@ import ( "sync" "time" - "bufio" - "github.com/D4-project/analyzer-d4-log/logparser" config "github.com/D4-project/d4-golang-utils/config" "github.com/gomodule/redigo/redis" @@ -45,12 +44,14 @@ var ( confdir = flag.String("c", "conf.sample", "configuration directory") all = flag.Bool("a", true, "run all parsers when set. Set by default") specific = flag.String("o", "", "run only a specific parser [sshd]") + debug = flag.Bool("d", false, "debug info in logs") redisD4 redis.Conn redisParsers *redis.Pool parsers = [1]string{"sshd"} - compilationTrigger = 10 + compilationTrigger = 200 wg sync.WaitGroup compiling comutex + torun = []logparser.Parser{} ) func main() { @@ -103,6 +104,11 @@ func main() { *confdir = strings.TrimSuffix(*confdir, "\\") } + // Debug log + if *debug { + log.SetFlags(log.LstdFlags | log.Lshortfile) + } + // Parse Redis D4 Config tmp := config.ReadConfigFile(*confdir, "redis_d4") ss := strings.Split(string(tmp), "/") @@ -146,23 +152,22 @@ func main() { // Line counter to trigger HTML compilation nblines := 0 - var torun = []logparser.Parser{} // Init parser depending on the parser flags: if *all { // Init all parsers for _, v := range parsers { switch v { case "sshd": - var sshdrcon, err = redisParsers.Dial() + sshdrcon1, err := redisParsers.Dial() if err != nil { - log.Fatal("Could not connect to Parser Redis") + log.Fatal("Could not connect to Line one Redis") } - _, err = sshdrcon.Do("PING") + sshdrcon2, err := redisParsers.Dial() if err != nil { - log.Fatal("Could connect to the Redis database") + log.Fatal("Could not connect to Line one Redis") } sshd := logparser.SshdParser{} - sshd.Set(&sshdrcon) + sshd.Set(&sshdrcon1, &sshdrcon2) torun = append(torun, &sshd) } } @@ -175,19 +180,19 @@ func main() { log.Fatalf("Error opening test file: %v", err) } defer f.Close() - scanner := bufio.NewScanner(f) - for scanner.Scan() { + // scanner := bufio.NewScanner(f) + // for scanner.Scan() { - // Pop D4 redis queue - //for { + // Pop D4 redis queue + for { - // err := errors.New("") - // logline, err := redis.String(redisD4.Do("LPOP", "analyzer:3:"+rd4.redisQueue)) - logline := scanner.Text() - // if err != nil { - // log.Fatal(err) - // } - // fmt.Println(logline) + err := errors.New("") + logline, err := redis.String(redisD4.Do("LPOP", "analyzer:3:"+rd4.redisQueue)) + // logline := scanner.Text() + if err != nil { + log.Fatal(err) + } + fmt.Println(logline) // Run the parsers for _, v := range torun { @@ -214,9 +219,17 @@ func compile() { compiling.mu.Lock() compiling.compiling = true wg.Add(1) - log.Println("I should probably be writing") - time.Sleep(500 * time.Millisecond) - log.Println("Writing") + + log.Println("Compiling") + + for _, v := range torun { + err := v.Compile() + if err != nil { + log.Fatal(err) + } + } + + log.Println("Done") compiling.compiling = false compiling.mu.Unlock() wg.Done()