From 9be5c60b48fb4e31c0b7e7139bb44ac1361eb7e8 Mon Sep 17 00:00:00 2001 From: Jean-Louis Huynen Date: Fri, 24 Jan 2020 12:00:49 +0100 Subject: [PATCH] add: [init] parser logic - not functional --- logparser/parser.go | 12 +++++ logparser/sshd.go | 44 +++++++++++++++++++ main.go | 104 ++++++++++++++++++++++++++++++++++---------- 3 files changed, 138 insertions(+), 22 deletions(-) create mode 100644 logparser/sshd.go diff --git a/logparser/parser.go b/logparser/parser.go index cc77ae1..fe40b70 100644 --- a/logparser/parser.go +++ b/logparser/parser.go @@ -1 +1,13 @@ package logparser + +type ( + // Parser provides the interface for a Parser + // It should provide: + // Parse to parse a line of log + // GetAttributes to get list of attributes (map keys) + Parser interface { + Parse() error + Push() error + Pop() map[string]string + } +) diff --git a/logparser/sshd.go b/logparser/sshd.go new file mode 100644 index 0000000..e6891dd --- /dev/null +++ b/logparser/sshd.go @@ -0,0 +1,44 @@ +package logparser + +import "github.com/gomodule/redigo/redis" + +// Sshd is a struct that corresponds to a line +type Sshd struct { + Date string + Host string + User string + Src string +} + +// SshdParser Holds a struct that corresponds to a sshd log line +// and the redis connection +type SshdParser struct { + logs Sshd + r *redis.Conn +} + +// New Creates a new sshd parser +func New(rconn *redis.Conn) *SshdParser { + return &SshdParser{ + logs: Sshd{}, + r: rconn, + } +} + +// Parse parses a line of sshd log +func (s *SshdParser) Parse() error { + //TODO + return nil +} + +// Push pushed the parsed line into redis +func (s *SshdParser) Push() error { + //TODO + return nil +} + +// Pop returns the list of attributes +func (s *SshdParser) Pop() map[string]string { + //TODO + return nil +} diff --git a/main.go b/main.go index bbec9e2..eec1e8e 100644 --- a/main.go +++ b/main.go @@ -1,7 +1,6 @@ package main import ( - "errors" "flag" "fmt" "log" @@ -9,26 +8,39 @@ import ( "os/signal" "strconv" "strings" + "time" + "github.com/D4-project/analyzer-d4-log/logparser" config "github.com/D4-project/d4-golang-utils/config" "github.com/gomodule/redigo/redis" ) type ( - conf struct { + redisconfD4 struct { redisHost string redisPort string redisDB int redisQueue string - httpHost string - httpPort string + } + redisconfParsers struct { + redisHost string + redisPort string + redisDBCount int + } + conf struct { + httpHost string + httpPort string } ) // Setting up flags var ( - confdir = flag.String("c", "conf.sample", "configuration directory") - cr redis.Conn + confdir = flag.String("c", "conf.sample", "configuration directory") + all = flag.Bool("a", true, "run all parsers when set. Set by default") + specific = flag.String("o", "", "run only a specific parser [sshd]") + redisD4 redis.Conn + redisParsers *redis.Pool + parsers = [1]string{"sshd"} ) func main() { @@ -61,14 +73,17 @@ func main() { fmt.Printf("\n") fmt.Printf("The configuration directory should hold the following files\n") fmt.Printf("to specify the settings to use:\n\n") - fmt.Printf(" mandatory: redis - host:port/db\n") + fmt.Printf(" mandatory: redis_d4 - host:port/db\n") fmt.Printf(" mandatory: redis_queue - uuid\n") + fmt.Printf(" mandatory: redis_parsers - host:port/maxdb\n") fmt.Printf(" optional: http_server - host:port\n\n") fmt.Printf("See conf.sample for an example.\n") } // Config - c := conf{} + // c := conf{} + rd4 := redisconfD4{} + rp := redisconfParsers{} flag.Parse() if flag.NFlag() == 0 || *confdir == "" { flag.Usage() @@ -78,31 +93,76 @@ func main() { *confdir = strings.TrimSuffix(*confdir, "\\") } - // Parse Redis Config - tmp := config.ReadConfigFile(*confdir, "redis") + // Parse Redis D4 Config + tmp := config.ReadConfigFile(*confdir, "redis_d4") ss := strings.Split(string(tmp), "/") if len(ss) <= 1 { - log.Fatal("Missing Database in Redis config: should be host:port/database_name") + log.Fatal("Missing Database in Redis D4 config: should be host:port/database_name") } - c.redisDB, _ = strconv.Atoi(ss[1]) + rd4.redisDB, _ = strconv.Atoi(ss[1]) var ret bool ret, ss[0] = config.IsNet(ss[0]) if !ret { sss := strings.Split(string(ss[0]), ":") - c.redisHost = sss[0] - c.redisPort = sss[1] + rd4.redisHost = sss[0] + rd4.redisPort = sss[1] } - c.redisQueue = string(config.ReadConfigFile(*confdir, "redis_queue")) - initRedis(c.redisHost, c.redisPort, c.redisDB) - defer cr.Close() + rd4.redisQueue = string(config.ReadConfigFile(*confdir, "redis_queue")) + // Connect to D4 Redis + redisD4, err = redis.Dial("tcp", rd4.redisHost+":"+rd4.redisPort, redis.DialDatabase(rd4.redisDB)) + if err != nil { + log.Fatal(err) + } + defer redisD4.Close() + + // Parse Redis Parsers Config + tmp = config.ReadConfigFile(*confdir, "redis_parsers") + ss = strings.Split(string(tmp), "/") + if len(ss) <= 1 { + log.Fatal("Missing Database Count in Redis config: should be host:port/max number of DB") + } + rp.redisDBCount, _ = strconv.Atoi(ss[1]) + ret, ss[0] = config.IsNet(string(tmp)) + if !ret { + sss := strings.Split(string(ss[0]), ":") + rp.redisHost = sss[0] + rp.redisPort = sss[1] + } + + // Create a connection Pool + redisParsers = newPool(rp.redisHost+":"+rp.redisPort, rp.redisDBCount) + + // Init parser depending on the parser flags: + if *all { + // Init all parsers + var torun = []logparser.Parser{} + for _, v := range parsers { + switch v { + case "sshd": + var sshdrcon, err = redisParsers.Dial() + if err != nil { + log.Fatal("Could not connect to Parser Redis") + } + sshd := logparser.New(&sshdrcon) + torun = append(torun, sshd) + } + } + } else if *specific != "" { + log.Println("TODO should run specific parser here") + } + + // Run the parsers + log.Println("TODO should run the parsers here") log.Println("Exit") } -func initRedis(host string, port string, d int) { - err := errors.New("") - cr, err = redis.Dial("tcp", host+":"+port, redis.DialDatabase(d)) - if err != nil { - log.Fatal(err) +func newPool(addr string, maxconn int) *redis.Pool { + return &redis.Pool{ + MaxActive: maxconn, + MaxIdle: 3, + IdleTimeout: 240 * time.Second, + // Dial or DialContext must be set. When both are set, DialContext takes precedence over Dial. + Dial: func() (redis.Conn, error) { return redis.Dial("tcp", addr) }, } }