add: [grok] ingest from file
parent
9a4d57ee0a
commit
aef4b518c0
|
@ -15,6 +15,7 @@ type (
|
||||||
// Flush recomputes statisitcs and recompile output
|
// Flush recomputes statisitcs and recompile output
|
||||||
Compiler interface {
|
Compiler interface {
|
||||||
Set(*sync.WaitGroup, *redis.Conn, *redis.Conn, io.Reader, int, *sync.WaitGroup)
|
Set(*sync.WaitGroup, *redis.Conn, *redis.Conn, io.Reader, int, *sync.WaitGroup)
|
||||||
|
SetReader(io.Reader)
|
||||||
Pull() error
|
Pull() error
|
||||||
Flush() error
|
Flush() error
|
||||||
Compile() error
|
Compile() error
|
||||||
|
@ -54,3 +55,8 @@ func (s *CompilerStruct) Set(wg *sync.WaitGroup, rconn0 *redis.Conn, rconn1 *red
|
||||||
s.compiling = false
|
s.compiling = false
|
||||||
s.compilegr = compilegr
|
s.compilegr = compilegr
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// SetReader Changes compiler's input
|
||||||
|
func (s *CompilerStruct) SetReader(reader io.Reader) {
|
||||||
|
s.reader = reader
|
||||||
|
}
|
||||||
|
|
30
main.go
30
main.go
|
@ -192,35 +192,29 @@ func main() {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Fatal(err)
|
log.Fatal(err)
|
||||||
}
|
}
|
||||||
// TODO
|
log.Println("Exit")
|
||||||
// compile()
|
os.Exit(0)
|
||||||
}
|
}
|
||||||
// TODO update that -- deprecated
|
}
|
||||||
} else if *fromfile != "" {
|
|
||||||
|
// Launching Pull routines
|
||||||
|
for _, v := range torun {
|
||||||
|
|
||||||
|
// If we read from a file, we set the reader to os.open
|
||||||
|
if *fromfile != "" {
|
||||||
f, err = os.Open(*fromfile)
|
f, err = os.Open(*fromfile)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Fatalf("Error opening seed file: %v", err)
|
log.Fatalf("Error opening seed file: %v", err)
|
||||||
}
|
}
|
||||||
defer f.Close()
|
defer f.Close()
|
||||||
// scanner := bufio.NewScanner(f)
|
v.SetReader(f)
|
||||||
// for scanner.Scan() {
|
}
|
||||||
// logline := scanner.Bytes()
|
|
||||||
// for _, v := range torun {
|
|
||||||
// go v.Pull()
|
|
||||||
// if err != nil {
|
|
||||||
// log.Fatal(err)
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
} else {
|
|
||||||
// Launching Pull routines
|
|
||||||
for _, v := range torun {
|
|
||||||
// we add pulling routines to a waitgroup,
|
// we add pulling routines to a waitgroup,
|
||||||
// they can immediately die when exiting.
|
// they can immediately die when exiting.
|
||||||
pullgr.Add(1)
|
pullgr.Add(1)
|
||||||
go v.Pull()
|
go v.Pull()
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
pullgr.Wait()
|
pullgr.Wait()
|
||||||
log.Println("Exit")
|
log.Println("Exit")
|
||||||
|
|
Loading…
Reference in New Issue