add: [grok] Stream d4 redis pulling
parent
593c6425b5
commit
9a4d57ee0a
|
@ -1,8 +1,8 @@
|
||||||
package logcompiler
|
package logcompiler
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"io"
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/gomodule/redigo/redis"
|
"github.com/gomodule/redigo/redis"
|
||||||
)
|
)
|
||||||
|
@ -14,7 +14,7 @@ type (
|
||||||
// Parse to parse a line of log
|
// Parse to parse a line of log
|
||||||
// Flush recomputes statisitcs and recompile output
|
// Flush recomputes statisitcs and recompile output
|
||||||
Compiler interface {
|
Compiler interface {
|
||||||
Set(*sync.WaitGroup, *redis.Conn, *redis.Conn, *redis.Conn, int, string, int, int, *sync.WaitGroup)
|
Set(*sync.WaitGroup, *redis.Conn, *redis.Conn, io.Reader, int, *sync.WaitGroup)
|
||||||
Pull() error
|
Pull() error
|
||||||
Flush() error
|
Flush() error
|
||||||
Compile() error
|
Compile() error
|
||||||
|
@ -27,13 +27,8 @@ type (
|
||||||
r0 *redis.Conn
|
r0 *redis.Conn
|
||||||
// Compiler redis Write
|
// Compiler redis Write
|
||||||
r1 *redis.Conn
|
r1 *redis.Conn
|
||||||
// Input Read
|
// Input Reader
|
||||||
r2 *redis.Conn
|
reader io.Reader
|
||||||
db int
|
|
||||||
// Dedicated queue
|
|
||||||
q string
|
|
||||||
// Time in minute before retrying
|
|
||||||
retryPeriod time.Duration
|
|
||||||
// Number of line to process before triggering output
|
// Number of line to process before triggering output
|
||||||
compilationTrigger int
|
compilationTrigger int
|
||||||
// Current line processed
|
// Current line processed
|
||||||
|
@ -51,14 +46,11 @@ type (
|
||||||
)
|
)
|
||||||
|
|
||||||
// Set set the redis connections to this compiler
|
// Set set the redis connections to this compiler
|
||||||
func (s *CompilerStruct) Set(wg *sync.WaitGroup, rconn0 *redis.Conn, rconn1 *redis.Conn, rconn2 *redis.Conn, db int, queue string, ct int, rt int, compilegr *sync.WaitGroup) {
|
func (s *CompilerStruct) Set(wg *sync.WaitGroup, rconn0 *redis.Conn, rconn1 *redis.Conn, reader io.Reader, ct int, compilegr *sync.WaitGroup) {
|
||||||
s.r0 = rconn0
|
s.r0 = rconn0
|
||||||
s.r1 = rconn1
|
s.r1 = rconn1
|
||||||
s.r2 = rconn2
|
s.reader = reader
|
||||||
s.q = queue
|
|
||||||
s.db = db
|
|
||||||
s.compilationTrigger = ct
|
s.compilationTrigger = ct
|
||||||
s.retryPeriod = time.Duration(rt) * time.Minute
|
|
||||||
s.compiling = false
|
s.compiling = false
|
||||||
s.compilegr = compilegr
|
s.compilegr = compilegr
|
||||||
}
|
}
|
||||||
|
|
|
@ -37,8 +37,6 @@ type GrokedSSHD struct {
|
||||||
SshdInvalidUser string `json:"sshd_invalid_user"`
|
SshdInvalidUser string `json:"sshd_invalid_user"`
|
||||||
}
|
}
|
||||||
|
|
||||||
var m GrokedSSHD
|
|
||||||
|
|
||||||
// Flush recomputes statistics and recompile HTML output
|
// Flush recomputes statistics and recompile HTML output
|
||||||
// TODO : review after refacto
|
// TODO : review after refacto
|
||||||
func (s *SSHDCompiler) Flush() error {
|
func (s *SSHDCompiler) Flush() error {
|
||||||
|
@ -103,33 +101,12 @@ func (s *SSHDCompiler) Flush() error {
|
||||||
// Pull pulls a line of groked sshd logline from redis
|
// Pull pulls a line of groked sshd logline from redis
|
||||||
func (s *SSHDCompiler) Pull() error {
|
func (s *SSHDCompiler) Pull() error {
|
||||||
r1 := *s.r1
|
r1 := *s.r1
|
||||||
r2 := *s.r2
|
|
||||||
|
|
||||||
for {
|
jsoner := json.NewDecoder(s.reader)
|
||||||
|
|
||||||
// Reading from specified database on r2 - input
|
for jsoner.More() {
|
||||||
if _, err := r2.Do("SELECT", s.db); err != nil {
|
var m GrokedSSHD
|
||||||
r2.Close()
|
err := jsoner.Decode(&m)
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
grokedline, err := redis.Bytes(r2.Do("LPOP", s.q))
|
|
||||||
fmt.Printf("%s\n", grokedline)
|
|
||||||
|
|
||||||
if err == redis.ErrNil {
|
|
||||||
// redis queue empty, let's sleep for a while
|
|
||||||
time.Sleep(s.retryPeriod)
|
|
||||||
} else if err != nil {
|
|
||||||
log.Fatal(err)
|
|
||||||
} else {
|
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
r1.Close()
|
|
||||||
r2.Close()
|
|
||||||
log.Fatal(err)
|
|
||||||
}
|
|
||||||
// Compile statistics
|
|
||||||
err = json.Unmarshal([]byte(grokedline), &m)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Println(err)
|
log.Println(err)
|
||||||
}
|
}
|
||||||
|
@ -171,7 +148,7 @@ func (s *SSHDCompiler) Pull() error {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func compileStats(s *SSHDCompiler, parsedTime time.Time, src string, username string, host string) error {
|
func compileStats(s *SSHDCompiler, parsedTime time.Time, src string, username string, host string) error {
|
||||||
|
|
21
main.go
21
main.go
|
@ -1,7 +1,6 @@
|
||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bufio"
|
|
||||||
"flag"
|
"flag"
|
||||||
"fmt"
|
"fmt"
|
||||||
"log"
|
"log"
|
||||||
|
@ -12,6 +11,7 @@ import (
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"github.com/D4-project/analyzer-d4-log/inputreader"
|
||||||
"github.com/D4-project/analyzer-d4-log/logcompiler"
|
"github.com/D4-project/analyzer-d4-log/logcompiler"
|
||||||
config "github.com/D4-project/d4-golang-utils/config"
|
config "github.com/D4-project/d4-golang-utils/config"
|
||||||
"github.com/gomodule/redigo/redis"
|
"github.com/gomodule/redigo/redis"
|
||||||
|
@ -175,8 +175,9 @@ func main() {
|
||||||
log.Fatal("Could not connect to output line on Input Redis")
|
log.Fatal("Could not connect to output line on Input Redis")
|
||||||
}
|
}
|
||||||
defer sshdrcon2.Close()
|
defer sshdrcon2.Close()
|
||||||
|
redisReader := inputreader.NewLPOPReader(&sshdrcon2, ri.redisDB, "sshd", *retry)
|
||||||
sshd := logcompiler.SSHDCompiler{}
|
sshd := logcompiler.SSHDCompiler{}
|
||||||
sshd.Set(&pullgr, &sshdrcon0, &sshdrcon1, &sshdrcon2, ri.redisDB, "sshd", compilationTrigger, *retry, &compilegr)
|
sshd.Set(&pullgr, &sshdrcon0, &sshdrcon1, redisReader, compilationTrigger, &compilegr)
|
||||||
torun = append(torun, &sshd)
|
torun = append(torun, &sshd)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -201,24 +202,16 @@ func main() {
|
||||||
log.Fatalf("Error opening seed file: %v", err)
|
log.Fatalf("Error opening seed file: %v", err)
|
||||||
}
|
}
|
||||||
defer f.Close()
|
defer f.Close()
|
||||||
scanner := bufio.NewScanner(f)
|
// scanner := bufio.NewScanner(f)
|
||||||
for scanner.Scan() {
|
// for scanner.Scan() {
|
||||||
// logline := scanner.Text()
|
// logline := scanner.Bytes()
|
||||||
// for _, v := range torun {
|
// for _, v := range torun {
|
||||||
// err := v.Pull(logline)
|
// go v.Pull()
|
||||||
// if err != nil {
|
// if err != nil {
|
||||||
// log.Fatal(err)
|
// log.Fatal(err)
|
||||||
// }
|
// }
|
||||||
// }
|
// }
|
||||||
// nblines++
|
|
||||||
// if nblines > compilationTrigger {
|
|
||||||
// nblines = 0
|
|
||||||
// Non-blocking
|
|
||||||
// if !compiling.compiling {
|
|
||||||
// go compile()
|
|
||||||
// }
|
// }
|
||||||
// }
|
|
||||||
}
|
|
||||||
} else {
|
} else {
|
||||||
// Launching Pull routines
|
// Launching Pull routines
|
||||||
for _, v := range torun {
|
for _, v := range torun {
|
||||||
|
|
Loading…
Reference in New Issue