chg: [sshd] svg graph generation
Jean-Louis Huynen
parent
05a8f493fd
commit
6746f5ede5
|
|
@ -6,3 +6,6 @@ analyzer-d4-log
|
|||
|
||||
# Results
|
||||
data*
|
||||
|
||||
# Redis backups
|
||||
*.rdb
|
||||
|
|
|
|||
|
|
@ -0,0 +1 @@
|
|||
localhost:6380/2
|
||||
|
|
@ -0,0 +1 @@
|
|||
localhost:6381/16
|
||||
3
go.mod
3
go.mod
|
|
@ -5,4 +5,7 @@ go 1.13
|
|||
require (
|
||||
github.com/D4-project/d4-golang-utils v0.0.0-20200108150548-740f16240125
|
||||
github.com/gomodule/redigo v2.0.0+incompatible
|
||||
golang.org/x/image v0.0.0-20200119044424-58c23975cae1 // indirect
|
||||
gonum.org/v1/netlib v0.0.0-20191229114700-bbb4dff026f8 // indirect
|
||||
gonum.org/v1/plot v0.0.0-20200111075622-4abb28f724d5
|
||||
)
|
||||
|
|
|
|||
47
go.sum
47
go.sum
|
|
@ -1,5 +1,52 @@
|
|||
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
|
||||
github.com/D4-project/d4-golang-utils v0.0.0-20200108150548-740f16240125 h1:iv+hcdT+M0XJIDEoCtvk9HVvI8PgvbQNBtbEfCczCRI=
|
||||
github.com/D4-project/d4-golang-utils v0.0.0-20200108150548-740f16240125/go.mod h1:2rq8KBQnNNDocwc/49cnpaqoQA/komoSHKom7ynvqJc=
|
||||
github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af h1:wVe6/Ea46ZMeNkQjjBW6xcqyQA/j5e0D6GytH95g0gQ=
|
||||
github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw=
|
||||
github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k=
|
||||
github.com/fogleman/gg v1.3.0 h1:/7zJX8F6AaYQc57WQCyN9cAIz+4bCJGO9B+dyW29am8=
|
||||
github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k=
|
||||
github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
|
||||
github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 h1:DACJavvAHhabrF08vX0COfcOBJRhZ8lUbR+ZWIs0Y5g=
|
||||
github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k=
|
||||
github.com/gomodule/redigo v2.0.0+incompatible h1:K/R+8tc58AaqLkqG2Ol3Qk+DR/TlNuhuh457pBFPtt0=
|
||||
github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4=
|
||||
github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5 h1:PJr+ZMXIecYc1Ey2zucXdR73SMBtgjPgwa31099IMv0=
|
||||
github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f h1:9kQ594xxPWRNKfTOnPjPcgrIJ19zM3ic57aI7PbMyAA=
|
||||
golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190312203227-4b39c73a6495 h1:I6A9Ag9FpEKOjcKrRNjQkPHawoXIhKyTGfvvjFAiiAk=
|
||||
golang.org/x/exp v0.0.0-20190312203227-4b39c73a6495/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
|
||||
golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs=
|
||||
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
|
||||
golang.org/x/image v0.0.0-20200119044424-58c23975cae1 h1:5h3ngYt7+vXCDZCup/HkCQgW5XwmSvR/nA2JmJ0RErg=
|
||||
golang.org/x/image v0.0.0-20200119044424-58c23975cae1/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
|
||||
golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
|
||||
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4 h1:nYxTaCPaVoJbxx+vMVnsFb6kw5+6aJCx52m/lmM/Vog=
|
||||
gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo=
|
||||
gonum.org/v1/gonum v0.6.2 h1:4r+yNT0+8SWcOkXP+63H2zQbN+USnC73cjGUxnDF94Q=
|
||||
gonum.org/v1/gonum v0.6.2/go.mod h1:9mxDZsDKxgMAuccQkewq682L+0eCu4dCN2yonUJTCLU=
|
||||
gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw=
|
||||
gonum.org/v1/netlib v0.0.0-20191229114700-bbb4dff026f8 h1:kHY67jAKYewKUCz9YdNDa7iLAJ2WfNmoHzCCX4KnA8w=
|
||||
gonum.org/v1/netlib v0.0.0-20191229114700-bbb4dff026f8/go.mod h1:2IgXn/sJaRbePPBA1wRj8OE+QLvVaH0q8SK6TSTKlnk=
|
||||
gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc=
|
||||
gonum.org/v1/plot v0.0.0-20200111075622-4abb28f724d5 h1:edjGU2UC8RjriRJNETPkc6w4Y44FqlvPQEal6JdckTw=
|
||||
gonum.org/v1/plot v0.0.0-20200111075622-4abb28f724d5/go.mod h1:+HbaZVpsa73UwN7kXGCECULRHovLRJjH+t5cFPgxErs=
|
||||
modernc.org/cc v1.0.0/go.mod h1:1Sk4//wdnYJiUIxnW8ddKpaOJCF37yAdqYnkxUpaYxw=
|
||||
modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk=
|
||||
modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k=
|
||||
modernc.org/strutil v1.1.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs=
|
||||
modernc.org/xc v1.0.0/go.mod h1:mRNCo0bvLjGhHO9WsyuKVU4q0ceiDDDoEeWDJHrNx8I=
|
||||
rsc.io/pdf v0.1.1 h1:k1MczvYDUvJBe93bYd7wrZLLUEcLZAuF824/I4e5Xr4=
|
||||
rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
|
||||
|
|
|
|||
|
|
@ -8,7 +8,8 @@ type (
|
|||
// Set to assign a redis connection to it
|
||||
// Parse to parse a line of log
|
||||
Parser interface {
|
||||
Set(*redis.Conn)
|
||||
Set(*redis.Conn, *redis.Conn)
|
||||
Parse(string) error
|
||||
Compile() error
|
||||
}
|
||||
)
|
||||
|
|
|
|||
|
|
@ -2,27 +2,37 @@ package logparser
|
|||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"math"
|
||||
"os"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/gomodule/redigo/redis"
|
||||
"gonum.org/v1/plot"
|
||||
"gonum.org/v1/plot/plotter"
|
||||
"gonum.org/v1/plot/plotutil"
|
||||
"gonum.org/v1/plot/vg"
|
||||
)
|
||||
|
||||
// SshdParser Holds a struct that corresponds to a sshd log line
|
||||
// and the redis connection
|
||||
type SshdParser struct {
|
||||
r *redis.Conn
|
||||
r1 *redis.Conn
|
||||
r2 *redis.Conn
|
||||
}
|
||||
|
||||
// Set set the redic connection to this parser
|
||||
func (s *SshdParser) Set(rconn *redis.Conn) {
|
||||
s.r = rconn
|
||||
func (s *SshdParser) Set(rconn1 *redis.Conn, rconn2 *redis.Conn) {
|
||||
s.r1 = rconn1
|
||||
s.r2 = rconn2
|
||||
}
|
||||
|
||||
// Parse parses a line of sshd log
|
||||
func (s *SshdParser) Parse(logline string) error {
|
||||
r := *s.r
|
||||
r := *s.r1
|
||||
re := regexp.MustCompile(`^(?P<date>[[:alpha:]]{3}\s\d{2}\s\d{2}:\d{2}:\d{2}) (?P<host>[^ ]+) sshd\[[[:alnum:]]+\]: Invalid user (?P<username>[^ ]+) from (?P<src>.*$)`)
|
||||
n1 := re.SubexpNames()
|
||||
r2 := re.FindAllStringSubmatch(logline, -1)[0]
|
||||
|
|
@ -73,5 +83,108 @@ func (s *SshdParser) Parse(logline string) error {
|
|||
return err
|
||||
}
|
||||
|
||||
// Keeping track of which days we updated statistics for
|
||||
_, err = redis.Int(r.Do("SADD", "toupdate", fmt.Sprintf("%v%v%v:statssrc", parsedTime.Year(), int(parsedTime.Month()), parsedTime.Day())))
|
||||
if err != nil {
|
||||
r.Close()
|
||||
return err
|
||||
}
|
||||
_, err = redis.Int(r.Do("SADD", "toupdate", fmt.Sprintf("%v%v%v:statsusername", parsedTime.Year(), int(parsedTime.Month()), parsedTime.Day())))
|
||||
if err != nil {
|
||||
r.Close()
|
||||
return err
|
||||
}
|
||||
_, err = redis.Int(r.Do("SADD", "toupdate", fmt.Sprintf("%v%v%v:statshost", parsedTime.Year(), int(parsedTime.Month()), parsedTime.Day())))
|
||||
if err != nil {
|
||||
r.Close()
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Compile create graphs of the results
|
||||
func (s *SshdParser) Compile() error {
|
||||
r := *s.r2
|
||||
|
||||
// Pulling statistics from database 1
|
||||
if _, err := r.Do("SELECT", 1); err != nil {
|
||||
r.Close()
|
||||
return err
|
||||
}
|
||||
|
||||
// List days for which we need to update statistic
|
||||
toupdate, err := redis.Strings(r.Do("SMEMBERS", "toupdate"))
|
||||
if err != nil {
|
||||
r.Close()
|
||||
return err
|
||||
}
|
||||
|
||||
// Query statistics dor each day to update
|
||||
for _, v := range toupdate {
|
||||
zrank, err := redis.Strings(r.Do("ZRANGEBYSCORE", v, "-inf", "+inf", "WITHSCORES"))
|
||||
if err != nil {
|
||||
r.Close()
|
||||
return err
|
||||
}
|
||||
|
||||
// Split keys and values - keep these ordered
|
||||
values := plotter.Values{}
|
||||
keys := make([]string, 0, len(zrank)/2)
|
||||
|
||||
for k, v := range zrank {
|
||||
// keys
|
||||
if (k % 2) == 0 {
|
||||
keys = append(keys, zrank[k])
|
||||
// values
|
||||
} else {
|
||||
fv, _ := strconv.ParseFloat(v, 64)
|
||||
values = append(values, fv)
|
||||
}
|
||||
}
|
||||
|
||||
p, err := plot.New()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
stype := strings.Split(v, ":")
|
||||
switch stype[1] {
|
||||
case "statsusername":
|
||||
p.Title.Text = "Usernames"
|
||||
case "statssrc":
|
||||
p.Title.Text = "Source IP"
|
||||
case "statshost":
|
||||
p.Title.Text = "Host"
|
||||
default:
|
||||
p.Title.Text = ""
|
||||
log.Println("We should not reach this point, open an issue.")
|
||||
}
|
||||
|
||||
p.Y.Label.Text = "Count"
|
||||
w := 0.5 * vg.Centimeter
|
||||
bc, err := plotter.NewBarChart(values, w)
|
||||
bc.Horizontal = true
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
bc.LineStyle.Width = vg.Length(0)
|
||||
bc.Color = plotutil.Color(0)
|
||||
|
||||
p.Add(bc)
|
||||
p.NominalY(keys...)
|
||||
|
||||
// Create folder to store plots
|
||||
if _, err := os.Stat(stype[0]); os.IsNotExist(err) {
|
||||
os.Mkdir(stype[0], 0700)
|
||||
}
|
||||
|
||||
xsize := 3 + vg.Length(math.Round(float64(len(keys)/2)))
|
||||
if err := p.Save(15*vg.Centimeter, xsize*vg.Centimeter, fmt.Sprintf("data/%v/%v.svg", stype[0], v)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
|
|
|||
59
main.go
59
main.go
|
|
@ -1,6 +1,7 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"log"
|
||||
|
|
@ -11,8 +12,6 @@ import (
|
|||
"sync"
|
||||
"time"
|
||||
|
||||
"bufio"
|
||||
|
||||
"github.com/D4-project/analyzer-d4-log/logparser"
|
||||
config "github.com/D4-project/d4-golang-utils/config"
|
||||
"github.com/gomodule/redigo/redis"
|
||||
|
|
@ -45,12 +44,14 @@ var (
|
|||
confdir = flag.String("c", "conf.sample", "configuration directory")
|
||||
all = flag.Bool("a", true, "run all parsers when set. Set by default")
|
||||
specific = flag.String("o", "", "run only a specific parser [sshd]")
|
||||
debug = flag.Bool("d", false, "debug info in logs")
|
||||
redisD4 redis.Conn
|
||||
redisParsers *redis.Pool
|
||||
parsers = [1]string{"sshd"}
|
||||
compilationTrigger = 10
|
||||
compilationTrigger = 200
|
||||
wg sync.WaitGroup
|
||||
compiling comutex
|
||||
torun = []logparser.Parser{}
|
||||
)
|
||||
|
||||
func main() {
|
||||
|
|
@ -103,6 +104,11 @@ func main() {
|
|||
*confdir = strings.TrimSuffix(*confdir, "\\")
|
||||
}
|
||||
|
||||
// Debug log
|
||||
if *debug {
|
||||
log.SetFlags(log.LstdFlags | log.Lshortfile)
|
||||
}
|
||||
|
||||
// Parse Redis D4 Config
|
||||
tmp := config.ReadConfigFile(*confdir, "redis_d4")
|
||||
ss := strings.Split(string(tmp), "/")
|
||||
|
|
@ -146,23 +152,22 @@ func main() {
|
|||
// Line counter to trigger HTML compilation
|
||||
nblines := 0
|
||||
|
||||
var torun = []logparser.Parser{}
|
||||
// Init parser depending on the parser flags:
|
||||
if *all {
|
||||
// Init all parsers
|
||||
for _, v := range parsers {
|
||||
switch v {
|
||||
case "sshd":
|
||||
var sshdrcon, err = redisParsers.Dial()
|
||||
sshdrcon1, err := redisParsers.Dial()
|
||||
if err != nil {
|
||||
log.Fatal("Could not connect to Parser Redis")
|
||||
log.Fatal("Could not connect to Line one Redis")
|
||||
}
|
||||
_, err = sshdrcon.Do("PING")
|
||||
sshdrcon2, err := redisParsers.Dial()
|
||||
if err != nil {
|
||||
log.Fatal("Could connect to the Redis database")
|
||||
log.Fatal("Could not connect to Line one Redis")
|
||||
}
|
||||
sshd := logparser.SshdParser{}
|
||||
sshd.Set(&sshdrcon)
|
||||
sshd.Set(&sshdrcon1, &sshdrcon2)
|
||||
torun = append(torun, &sshd)
|
||||
}
|
||||
}
|
||||
|
|
@ -175,19 +180,19 @@ func main() {
|
|||
log.Fatalf("Error opening test file: %v", err)
|
||||
}
|
||||
defer f.Close()
|
||||
scanner := bufio.NewScanner(f)
|
||||
for scanner.Scan() {
|
||||
// scanner := bufio.NewScanner(f)
|
||||
// for scanner.Scan() {
|
||||
|
||||
// Pop D4 redis queue
|
||||
//for {
|
||||
// Pop D4 redis queue
|
||||
for {
|
||||
|
||||
// err := errors.New("")
|
||||
// logline, err := redis.String(redisD4.Do("LPOP", "analyzer:3:"+rd4.redisQueue))
|
||||
logline := scanner.Text()
|
||||
// if err != nil {
|
||||
// log.Fatal(err)
|
||||
// }
|
||||
// fmt.Println(logline)
|
||||
err := errors.New("")
|
||||
logline, err := redis.String(redisD4.Do("LPOP", "analyzer:3:"+rd4.redisQueue))
|
||||
// logline := scanner.Text()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
fmt.Println(logline)
|
||||
|
||||
// Run the parsers
|
||||
for _, v := range torun {
|
||||
|
|
@ -214,9 +219,17 @@ func compile() {
|
|||
compiling.mu.Lock()
|
||||
compiling.compiling = true
|
||||
wg.Add(1)
|
||||
log.Println("I should probably be writing")
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
log.Println("Writing")
|
||||
|
||||
log.Println("Compiling")
|
||||
|
||||
for _, v := range torun {
|
||||
err := v.Compile()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
log.Println("Done")
|
||||
compiling.compiling = false
|
||||
compiling.mu.Unlock()
|
||||
wg.Done()
|
||||
|
|
|
|||
Loading…
Reference in New Issue