add: [grok] ingest from file

logcompiler/compiler.go

@@ -15,6 +15,7 @@ type (
 	//  Flush recomputes statisitcs and recompile output
 	Compiler interface {
 		Set(*sync.WaitGroup, *redis.Conn, *redis.Conn, io.Reader, int, *sync.WaitGroup)
+		SetReader(io.Reader)
 		Pull() error
 		Flush() error
 		Compile() error
@@ -54,3 +55,8 @@ func (s *CompilerStruct) Set(wg *sync.WaitGroup, rconn0 *redis.Conn, rconn1 *red
 	s.compiling = false
 	s.compilegr = compilegr
 }
+
+// SetReader Changes compiler's input
+func (s *CompilerStruct) SetReader(reader io.Reader) {
+	s.reader = reader
+}

main.go

@@ -192,34 +192,28 @@ func main() {
 			if err != nil {
 				log.Fatal(err)
 			}
-			// TODO
-			// compile()
+			log.Println("Exit")
+			os.Exit(0)
 		}
-		// TODO update that -- deprecated
-	} else if *fromfile != "" {
-		f, err = os.Open(*fromfile)
-		if err != nil {
-			log.Fatalf("Error opening seed file: %v", err)
-		}
-		defer f.Close()
-		// scanner := bufio.NewScanner(f)
-		// for scanner.Scan() {
-		// logline := scanner.Bytes()
-		// for _, v := range torun {
-		// 	go v.Pull()
-		// 	if err != nil {
-		// 		log.Fatal(err)
-		// 	}
-		// }
-		// }
-	} else {
-		// Launching Pull routines
-		for _, v := range torun {
-			// we add pulling routines to a waitgroup,
-			// they can immediately die when exiting.
-			pullgr.Add(1)
-			go v.Pull()
+	}
+
+	// Launching Pull routines
+	for _, v := range torun {
+
+		// If we read from a file, we set the reader to os.open
+		if *fromfile != "" {
+			f, err = os.Open(*fromfile)
+			if err != nil {
+				log.Fatalf("Error opening seed file: %v", err)
+			}
+			defer f.Close()
+			v.SetReader(f)
 		}
+
+		// we add pulling routines to a waitgroup,
+		// they can immediately die when exiting.
+		pullgr.Add(1)
+		go v.Pull()
 	}
 
 	pullgr.Wait()