Merge pull request #21 from D4-project/dependabot/go_modules/golang.org/x/net-0.17.0

build(deps): bump golang.org/x/net from 0.12.0 to 0.17.0

.gitignore

@@ -15,6 +15,7 @@ d4-arm5l
 d4-amd64l
 # Output binaries from gox
 d4-goclient_*
+d4-goclient
 
 # Configuration files
 /conf.*

Makefile

@@ -1,4 +1,4 @@
 arm5l: d4-goclient.go
-	env GOOS=linux GOARCH=arm GOARM=5 go build -o d4-arm5l d4-goclient.go
+	env GOOS=linux GOARCH=arm GOARM=5 go build -o d4-goclient-arm5l d4-goclient.go
 amd64l: d4-goclient.go
-	env GOOS=linux GOARCH=amd64  go build -o d4-amd64l d4-goclient.go
+	env GOOS=linux GOARCH=amd64  go build -o d4-goclient-amd64l d4-goclient.go

README.md

@@ -19,21 +19,12 @@ For more information about the [D4 project](https://www.d4-project.org/).
 Fetch d4-goclient code and dependencies
 
 ```bash
-go get github.com/satori/go.uuid
 go get github.com/D4-project/d4-goclient
 ```
 
-Use make to build binaries:
-
-```bash
-make arm5l  # for raspberry pi / linux
-make amd64l # for amd64 / linux
-```
-
 ## Dependencies
 
- - golang 1.10 (tested)
- - go.uuid
+ - golang 1.13 (tested)
 
 # Use
 
@@ -47,13 +38,17 @@ Part of the client configuration can be stored in folder containing the followin
 
  - key: your Pre-Shared-Key
  - snaplen: default is 4096
- - source: stdin
+ - source: stdin or d4server
  - destination: stdout, [fe80::ffff:ffff:ffff:a6fb]:4443, 127.0.0.1:4443
- - type: D4 packat type, see [types](https://github.com/D4-project/architecture/tree/master/format)
- - uuid: generated automiatically if empty
+ - type: D4 packet type, see [types](https://github.com/D4-project/architecture/tree/master/format)
+ - uuid: generated automatically if empty
  - version: protocol version
  - rootCA.crt: optional : CA certificate to check the server certificate
  - metaheader.json: optional : a json file describing feed's meta-type [types](https://github.com/D4-project/architecture/tree/master/format)
+ 
+If source is set to d4server, then one also 2 additional files:
+ - redis_queue: redis queue in the form analyzer:typeofqueue:queueuuid, for instance `analyzer:3:d42967c1-f7ad-464e-bbc7-4464c653d7a6`
+ - redis_d4: redis server location:port/database, for instance localhost:6385/2
 
 ## Flags
 
@@ -68,6 +63,8 @@ Part of the client configuration can be stored in folder containing the followin
     	Keep Alive time human format, 0 to disable (default 30s)
   -ct duration
     	Set timeout in human format
+  -rl duration
+        Rate limiter: time in human format before retry after EOF (default 200ms)
   -rt duration
     	Time in human format before retry after connection failure, set to 0 to exit on failure (default 30s)
   -v	Set to True, true, TRUE, 1, or t to enable verbose output on stdout
@@ -86,3 +83,8 @@ $IP being the monitoring computer ip
 ```bash
 tcpdump not dst $IP and not src $IP -w - | ./d4-goclient -c conf.sample/ |  socat - OPENSSL-CONNECT:$IP_SRV:$PORT,verify=0
 ```
+
+## Forwarding data from a D4 server to another D4 server
+Add two files to you configuration folder: `redis_d4` and `redis_queue`:
+ - `redis_d4` contains the location of the source d4's redis server database, for instance `127.0.0.1:6380/2`
+ - `redis_queue` contains the queue to forward to the other D4 server, for instance `analyzer:3:d42967c1-f7ad-464e-bbc7-4464c653d7a6`

conf.sample/destination

@@ -1 +1 @@
-0.0.0.0:4443
+stdout

conf.sample/redis_d4

@@ -0,0 +1 @@
+localhost:6385/2

conf.sample/redis_queue

@@ -0,0 +1 @@
+analyzer:3:d42967c1-f7ad-464e-bbc7-4464c653d7a6

conf.sample/source

@@ -1 +1 @@
-stdin
+d4server

conf.sample/uuid

@@ -0,0 +1 @@
+1d940331-9fc9-4381-8fc9-3b624db66025

d4-goclient.go

@@ -10,6 +10,7 @@ import (
 	"encoding/json"
 	"flag"
 	"fmt"
+	"golang.org/x/net/proxy"
 	"io"
 	"io/ioutil"
 	"log"
@@ -18,10 +19,13 @@ import (
 	"os/signal"
 	"strconv"
 	"strings"
+	"syscall"
 	"time"
 
 	config "github.com/D4-project/d4-golang-utils/config"
 	uuid "github.com/D4-project/d4-golang-utils/crypto/hash"
+	"github.com/D4-project/d4-golang-utils/inputreader"
+	"github.com/gomodule/redigo/redis"
 )
 
 const (
@@ -56,21 +60,27 @@ type (
 	}
 
 	d4S struct {
-		src       io.Reader
-		dst       d4Writer
-		confdir   string
-		cka       time.Duration
-		ct        time.Duration
-		ce        bool
-		retry     time.Duration
-		cc        bool
-		ca        x509.CertPool
-		d4error   uint8
-		errnoCopy uint8
-		debug     bool
-		conf      d4params
-		mhb       *bytes.Buffer
-		mh        []byte
+		src            io.Reader
+		dst            d4Writer
+		confdir        string
+		cka            time.Duration
+		ct             time.Duration
+		ce             bool
+		retry          time.Duration
+		rate           time.Duration
+		cc             bool
+		tor            bool
+		daily          bool
+		json           bool
+		ca             x509.CertPool
+		d4error        uint8
+		errnoCopy      uint8
+		debug          bool
+		conf           d4params
+		mhb            *bytes.Buffer
+		mh             []byte
+		redisInputPool *redis.Pool
+		redisCon       redis.Conn
 	}
 
 	d4params struct {
@@ -81,28 +91,39 @@ type (
 		source      string
 		destination string
 		ttype       uint8
+		redisHost   string
+		redisPort   string
+		redisQueue  string
+		redisDB     int
+		folderstr   string
 	}
 )
 
 var (
-	// verbose
-	buf    bytes.Buffer
-	logger = log.New(&buf, "INFO: ", log.Lshortfile)
-	infof  = func(info string) {
-		logger.Output(2, info)
+	// Verbose mode and logging
+	buf      bytes.Buffer
+	logger   = log.New(&buf, "INFO: ", log.Lshortfile)
+	debugger = log.New(&buf, "DEBUG: ", log.Lmicroseconds)
+	debugf   = func(debug string) {
+		debugger.Println("", debug)
 	}
 
 	tmpct, _    = time.ParseDuration("5mn")
 	tmpcka, _   = time.ParseDuration("30s")
 	tmpretry, _ = time.ParseDuration("30s")
+	tmprate, _  = time.ParseDuration("200ms")
 
-	confdir = flag.String("c", "", "configuration directory")
-	debug   = flag.Bool("v", false, "Set to True, true, TRUE, 1, or t to enable verbose output on stdout")
-	ce      = flag.Bool("ce", true, "Set to True, true, TRUE, 1, or t to enable TLS on network destination")
-	ct      = flag.Duration("ct", tmpct, "Set timeout in human format")
-	cka     = flag.Duration("cka", tmpcka, "Keep Alive time human format, 0 to disable")
-	retry   = flag.Duration("rt", tmpretry, "Time in human format before retry after connection failure, set to 0 to exit on failure")
-	cc      = flag.Bool("cc", false, "Check TLS certificate against rootCA.crt")
+	confdir   = flag.String("c", "", "configuration directory")
+	debug     = flag.Bool("v", false, "Set to True, true, TRUE, 1, or t to enable verbose output on stdout - Don't use in production")
+	ce        = flag.Bool("ce", true, "Set to True, true, TRUE, 1, or t to enable TLS on network destination")
+	ct        = flag.Duration("ct", tmpct, "Set timeout in human format")
+	cka       = flag.Duration("cka", tmpcka, "Keep Alive time human format, 0 to disable")
+	retry     = flag.Duration("rt", tmpretry, "Time in human format before retry after connection failure, set to 0 to exit on failure")
+	rate      = flag.Duration("rl", tmprate, "Rate limiter: time in human format before retry after EOF")
+	cc        = flag.Bool("cc", false, "Check TLS certificate against rootCA.crt")
+	torflag   = flag.Bool("tor", false, "Use a SOCKS5 tor proxy on 9050")
+	dailyflag = flag.Bool("daily", false, "Sets up filewatcher to watch a new %Y%M%D folder at midnight")
+	jsonflag  = flag.Bool("json", false, "The files watched are json files")
 )
 
 func main() {
@@ -110,6 +131,16 @@ func main() {
 	var d4 d4S
 	d4p := &d4
 
+	// Setting up log file
+	f, err := os.OpenFile("d4-goclient.log", os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
+	if err != nil {
+		log.Fatalf("error opening file: %v", err)
+	}
+	defer f.Close()
+	logger.SetOutput(f)
+	logger.SetFlags(log.LstdFlags | log.Lshortfile)
+	logger.Println("Init")
+
 	flag.Usage = func() {
 		fmt.Printf("d4 - d4 client\n")
 		fmt.Printf("Read data from the configured <source> and send it to <destination>\n")
@@ -129,6 +160,8 @@ func main() {
 		fmt.Printf("type        - the type of data that is send. pcap, netflow, ...\n")
 		fmt.Printf("source      - the source where the data is read from\n")
 		fmt.Printf("destination - the destination where the data is written to\n")
+		fmt.Printf("redis_d4    - location of redis d4 server\n")
+		fmt.Printf("redis_queue - analyzer:type:queueuuid to pop\n")
 		fmt.Printf("\n")
 		flag.PrintDefaults()
 	}
@@ -141,65 +174,144 @@ func main() {
 		*confdir = strings.TrimSuffix(*confdir, "/")
 		*confdir = strings.TrimSuffix(*confdir, "\\")
 	}
+
 	d4.confdir = *confdir
 	d4.ce = *ce
 	d4.ct = *ct
 	d4.cc = *cc
+	d4.json = *jsonflag
 	d4.cka = *cka
 	d4.retry = *retry
+	d4.rate = *rate
+	d4.tor = *torflag
+	d4.daily = *dailyflag
 
 	s := make(chan os.Signal, 1)
 	signal.Notify(s, os.Interrupt, os.Kill)
-	c := make(chan string)
-	k := make(chan string)
+	errchan := make(chan error)
+	eofchan := make(chan string)
+	metachan := make(chan string)
 
-	d4.mhb = bytes.NewBuffer(d4.mh)
+	// Launching the Rate limiters
+	rateLimiter := time.Tick(d4.rate)
+	retryLimiter := time.Tick(d4.retry)
 
-	for {
-		// init or reinit after retry
-		if set(d4p) {
-			// type 254 requires to send a meta-header first
-			if d4.conf.ttype == 254 || d4.conf.ttype == 2 {
-				// create a jsonreader
-				d4p.dst.hijackHeader()
-				// Ugly hack to skip bytes.Buffer WriteTo check that bypasses my fixed lenght buffer
-				nread, err := io.CopyBuffer(&d4.dst, struct{ io.Reader }{d4.mhb}, d4.dst.pb)
-				if err != nil {
-					panic(fmt.Sprintf("Cannot initiate session %s", err))
+	//Setup
+	if !d4loadConfig(d4p) {
+		panic("Could not load Config.")
+	}
+	if !setReaderWriters(d4p, false) {
+		panic("Could not Init Inputs Outputs.")
+	}
+	if !d4.dst.initHeader(d4p) {
+		panic("Could not Init Headers.")
+	}
+
+	// force is a flag that forces the creation of a new connection
+	force := false
+
+	// On the first run, we send d4 meta header for type 2/254
+	if d4.conf.ttype == 254 || d4.conf.ttype == 2 {
+		go sendMeta(d4p, errchan, metachan)
+	H:
+		for {
+			select {
+			case <-errchan:
+				select {
+				case <-retryLimiter:
+					go sendMeta(d4p, errchan, metachan)
+				case <-s:
+					logger.Println("Exiting")
+					exit(d4p, 0)
 				}
-				infof(fmt.Sprintf("Meta-Header sent: %d bytes", nread))
-				d4p.dst.restoreHeader()
+			case <-metachan:
+				break H
 			}
-			// copy routine
-			go d4Copy(d4p, c, k)
-		} else if d4.retry > 0 {
-			go func() {
-				infof(fmt.Sprintf("Sleeping for %.f seconds before retry...\n", d4.retry.Seconds()))
-				fmt.Printf("Sleeping for %.f seconds before retry...\n", d4.retry.Seconds())
-				time.Sleep(d4.retry)
-				c <- "done waiting"
-			}()
-		} else {
-			panic("Unrecoverable error without retry.")
 		}
+	}
 
-		// Block until we catch an event
+	// Launch copy routine
+	go d4Copy(d4p, errchan, eofchan)
+
+	// Handle signals
+	for {
 		select {
-		case str := <-c:
-			infof(str)
-			continue
-		case str := <-k:
-			fmt.Println(str)
-			exit(d4p, 1)
+		// Case where the input ran out of data to consume1
+		case <-eofchan:
+			// We wait for ratelimiter before polling again
+		EOF:
+			for {
+				select {
+				case <-rateLimiter:
+					// copy routine
+					go d4Copy(d4p, errchan, eofchan)
+					break EOF
+					// Exit signal
+				case <-s:
+					logger.Println("Exiting")
+					exit(d4p, 0)
+				}
+			}
+		// ERROR, we check first whether it is network related
+		case err := <-errchan:
+			// On connection errors, we force setReaderWriter to reset the connection
+			force = false
+			switch t := err.(type) {
+			case *net.OpError:
+				force = true
+				if t.Op == "dial" {
+					logger.Println("Unknown Host")
+				} else if t.Op == "read" {
+					logger.Println("Connection Refused")
+				} else if t.Op == "write" {
+					logger.Println("Write error")
+				}
+			case syscall.Errno:
+				if t == syscall.ECONNREFUSED {
+					force = true
+					logger.Println("Connection Refused")
+				}
+			}
+			// We wait for retryLimiter before writing again
+		RETRY:
+			for {
+				select {
+				case <-retryLimiter:
+					if !setReaderWriters(d4p, force) {
+						// Can't connect, we break to retry
+						// force is still true
+						break
+					}
+					if !d4.dst.initHeader(d4p) {
+						panic("Could not Init Headers.")
+					}
+					if (d4.conf.ttype == 254 || d4.conf.ttype == 2) && force {
+						// setReaderWriter is happy, we should have a working
+						// connection from now on.
+						force = false
+						// Sending meta header for the first time on this new connection
+						go sendMeta(d4p, errchan, metachan)
+					}
+					break RETRY
+					// Exit signal
+				case <-s:
+					logger.Println("Exiting")
+					exit(d4p, 0)
+				}
+			}
+		// metaheader sent, launch the copy routine
+		case <-metachan:
+			go d4Copy(d4p, errchan, eofchan)
+		// Exit signal
 		case <-s:
-			fmt.Println(" Exiting")
+			logger.Println("Exiting")
 			exit(d4p, 0)
 		}
 	}
 }
 
 func exit(d4 *d4S, exitcode int) {
-	// Output logging before closing if debug is enabled
+	// Output debug info in the log before closing if debug is enabled
 	if *debug == true {
 		(*d4).debug = true
 		fmt.Print(&buf)
@@ -207,28 +319,31 @@ func exit(d4 *d4S, exitcode int) {
 	os.Exit(exitcode)
 }
 
-func set(d4 *d4S) bool {
-	if d4loadConfig(d4) {
-		if setReaderWriters(d4) {
-			if d4.dst.initHeader(d4) {
-				return true
-			}
-		}
-	}
-	return false
-}
-
-func d4Copy(d4 *d4S, c chan string, k chan string) {
+func d4Copy(d4 *d4S, errchan chan error, eofchan chan string) {
 	nread, err := io.CopyBuffer(&d4.dst, d4.src, d4.dst.pb)
+	// Always retry
 	if err != nil {
-		if (d4.retry.Seconds()) > 0 {
-			c <- fmt.Sprintf("%s", err)
-			return
-		}
-		k <- fmt.Sprintf("%s", err)
+		logger.Printf("D4copy: %s", err)
+		errchan <- err
 		return
 	}
-	k <- fmt.Sprintf("EOF: Nread: %d", nread)
+	eofchan <- fmt.Sprintf("EOF: Nread: %d", nread)
+}
+
+func sendMeta(d4 *d4S, errchan chan error, metachan chan string) {
+	// Fill metaheader buffer with metaheader data
+	d4.mhb = bytes.NewBuffer(d4.mh)
+	d4.dst.hijackHeader()
+	// Ugly hack to skip bytes.Buffer WriteTo check that bypasses my fixed lenght buffer
+	nread, err := io.CopyBuffer(&d4.dst, struct{ io.Reader }{d4.mhb}, d4.dst.pb)
+	if err != nil {
+		logger.Printf("Cannot sent meta-header: %s", err)
+		errchan <- err
+		return
+	}
+	logger.Println(fmt.Sprintf("Meta-Header sent: %d bytes", nread))
+	d4.dst.restoreHeader()
+	metachan <- "Header Sent"
 	return
 }
 
@@ -243,6 +358,36 @@ func d4loadConfig(d4 *d4S) bool {
 	if len((*d4).conf.source) < 1 {
 		log.Fatal("Unsupported source")
 	}
+	if (*d4).conf.source == "folder" {
+		fstr := string(readConfFile(d4, "folder"))
+		if ffd, err := os.Stat(fstr); os.IsNotExist(err) {
+			log.Fatal("Folder does not exist")
+		} else {
+			if !ffd.IsDir() {
+				log.Fatal("Folder is not a directory")
+			}
+		}
+		(*d4).conf.folderstr = fstr
+	}
+	if (*d4).conf.source == "d4server" {
+		// Parse Input Redis Config
+		tmp := string(readConfFile(d4, "redis_d4"))
+		ss := strings.Split(string(tmp), "/")
+		if len(ss) <= 1 {
+			log.Fatal("Missing Database in Redis input config: should be host:port/database_name")
+		}
+		(*d4).conf.redisDB, _ = strconv.Atoi(ss[1])
+		var ret bool
+		ret, ss[0] = config.IsNet(ss[0])
+		if ret {
+			sss := strings.Split(string(ss[0]), ":")
+			(*d4).conf.redisHost = sss[0]
+			(*d4).conf.redisPort = sss[1]
+		} else {
+			log.Fatal("Redis config error.")
+		}
+		(*d4).conf.redisQueue = string(config.ReadConfigFile(*confdir, "redis_queue"))
+	}
 	(*d4).conf.destination = string(readConfFile(d4, "destination"))
 	if len((*d4).conf.destination) < 1 {
 		log.Fatal("Unsupported Destination")
@@ -300,9 +445,13 @@ func d4loadConfig(d4 *d4S) bool {
 						if off, err := file.Seek(0, 0); err != nil || off != 0 {
 							panic(fmt.Sprintf("Cannot read Meta-Header file: %s", err))
 						} else {
+							// create metaheader buffer
+							d4.mhb = bytes.NewBuffer(d4.mh)
 							if err := json.Compact((*d4).mhb, data[:count]); err != nil {
-								infof("Failed to compact meta header file")
+								logger.Println("Failed to compact meta header file")
 							}
+							// Store the metaheader in d4 struct for subsequent retries
+							(*d4).mh = data[:count]
 						}
 					} else {
 						panic("A Meta-Header File should at least contain a 'type' field.")
@@ -349,7 +498,7 @@ func newD4Writer(writer io.Writer, key []byte) d4Writer {
 }
 
 // TODO QUICK IMPLEM, REVISE
-func setReaderWriters(d4 *d4S) bool {
+func setReaderWriters(d4 *d4S, force bool) bool {
 	//TODO implement other destination file, fifo unix_socket ...
 	switch (*d4).conf.source {
 	case "stdin":
@@ -357,37 +506,94 @@ func setReaderWriters(d4 *d4S) bool {
 	case "pcap":
 		f, _ := os.Open("capture.pcap")
 		(*d4).src = f
+	case "d4server":
+		// Create a new redis connection pool
+		(*d4).redisInputPool = newPool((*d4).conf.redisHost+":"+(*d4).conf.redisPort, 16)
+		var err error
+		(*d4).redisCon, err = (*d4).redisInputPool.Dial()
+		if err != nil {
+			logger.Println("Could not connect to d4 Redis")
+			return false
+		}
+		(*d4).src, err = inputreader.NewLPOPReader(&(*d4).redisCon, (*d4).conf.redisDB, (*d4).conf.redisQueue)
+		if err != nil {
+			log.Printf("Could not create d4 Redis Descriptor %q \n", err)
+			return false
+		}
+	case "folder":
+		var err error
+		(*d4).src, err = inputreader.NewFileWatcherReader((*d4).conf.folderstr, (*d4).json, (*d4).daily, logger)
+		if err != nil {
+			log.Printf("Could not create File Watcher %q \n", err)
+			return false
+		}
 	}
 	isn, dstnet := config.IsNet((*d4).conf.destination)
 	if isn {
-		dial := net.Dialer{
-			Timeout:       (*d4).ct,
-			KeepAlive:     (*d4).cka,
-			FallbackDelay: 0,
-		}
-		tlsc := tls.Config{
-			InsecureSkipVerify: true,
-		}
-		if (*d4).cc {
-			tlsc = tls.Config{
-				InsecureSkipVerify: false,
-				RootCAs:            &(*d4).ca,
+		// We test whether a connection already exist
+		// (case where the reader run out of data)
+		// force forces to reset the connections after
+		// failure to reuse it
+		if _, ok := (*d4).dst.w.(net.Conn); !ok || force {
+			if (*d4).tor {
+				dialer := net.Dialer{
+					Timeout:       (*d4).ct,
+					KeepAlive:     (*d4).cka,
+					FallbackDelay: 0,
+				}
+				dial, err := proxy.SOCKS5("tcp", "127.0.0.1:9050", nil, &dialer)
+				if err != nil {
+					log.Fatal(err)
+				}
+				tlsc := tls.Config{
+					InsecureSkipVerify: true,
+				}
+				if (*d4).cc {
+					tlsc = tls.Config{
+						InsecureSkipVerify: false,
+						RootCAs:            &(*d4).ca,
+					}
+				}
+				conn, errc := dial.Dial("tcp", dstnet)
+				if errc != nil {
+					logger.Println(errc)
+					return false
+				}
+				if (*d4).ce == true {
+					conn = tls.Client(conn, &tlsc) // use tls
+				}
+				(*d4).dst = newD4Writer(conn, (*d4).conf.key)
+			} else {
+				dial := net.Dialer{
+					Timeout:       (*d4).ct,
+					KeepAlive:     (*d4).cka,
+					FallbackDelay: 0,
+				}
+				tlsc := tls.Config{
+					InsecureSkipVerify: true,
+				}
+				if (*d4).cc {
+					tlsc = tls.Config{
+						InsecureSkipVerify: false,
+						RootCAs:            &(*d4).ca,
+					}
+				}
+				if (*d4).ce == true {
+					conn, errc := tls.DialWithDialer(&dial, "tcp", dstnet, &tlsc)
+					if errc != nil {
+						logger.Println(errc)
+						return false
+					}
+					(*d4).dst = newD4Writer(conn, (*d4).conf.key)
+				} else {
+					conn, errc := dial.Dial("tcp", dstnet)
+					if errc != nil {
+						return false
+					}
+					(*d4).dst = newD4Writer(conn, (*d4).conf.key)
+				}
 			}
 		}
-		if (*d4).ce == true {
-			conn, errc := tls.DialWithDialer(&dial, "tcp", dstnet, &tlsc)
-			if errc != nil {
-				fmt.Println(errc)
-				return false
-			}
-			(*d4).dst = newD4Writer(conn, (*d4).conf.key)
-		} else {
-			conn, errc := dial.Dial("tcp", dstnet)
-			if errc != nil {
-				return false
-			}
-			(*d4).dst = newD4Writer(conn, (*d4).conf.key)
-		}
 	} else {
 		switch (*d4).conf.destination {
 		case "stdout":
@@ -412,7 +618,7 @@ func generateUUIDv4() []byte {
 	if err != nil {
 		log.Fatal(err)
 	}
-	infof(fmt.Sprintf("UUIDv4: %s\n", uuid))
+	logger.Println(fmt.Sprintf("UUIDv4: %s", uuid))
 	return uuid.Bytes()
 }
 
@@ -467,8 +673,8 @@ func (d4w *d4Writer) initHeader(d4 *d4S) bool {
 	// hmac is set to zero during hmac operations, so leave it alone
 	// init size of payload at 0
 	binary.LittleEndian.PutUint32(d4w.fb[58:62], uint32(0))
-	infof(fmt.Sprintf("Initialized a %d bytes header:\n", HDR_SIZE))
-	infof(fmt.Sprintf("%b\n", d4w.fb[:HDR_SIZE]))
+	debugf(fmt.Sprintf("Initialized a %d bytes header:\n", HDR_SIZE))
+	debugf(fmt.Sprintf("%b\n", d4w.fb[:HDR_SIZE]))
 	return true
 }
 
@@ -483,3 +689,13 @@ func (d4w *d4Writer) restoreHeader() bool {
 	d4w.fb[1] = 254
 	return true
 }
+
+func newPool(addr string, maxconn int) *redis.Pool {
+	return &redis.Pool{
+		MaxActive:   maxconn,
+		MaxIdle:     3,
+		IdleTimeout: 240 * time.Second,
+		// Dial or DialContext must be set. When both are set, DialContext takes precedence over Dial.
+		Dial: func() (redis.Conn, error) { return redis.Dial("tcp", addr) },
+	}
+}

go.mod

@@ -2,4 +2,10 @@ module github.com/D4-project/d4-goclient
 
 go 1.13
 
-require github.com/D4-project/d4-golang-utils v0.0.0-20190603131519-c10ee092655c
+require (
+	github.com/D4-project/d4-golang-utils v0.1.14
+	github.com/gofrs/uuid v4.4.0+incompatible // indirect
+	github.com/gomodule/redigo v2.0.0+incompatible
+	github.com/rjeczalik/notify v0.9.3 // indirect
+	golang.org/x/net v0.17.0
+)

go.sum

@@ -1,4 +1,63 @@
-github.com/D4-project/d4-golang-utils v0.0.0-20190603131519-c10ee092655c h1:NfASgeIzH3ULEOYgDZwZCmq+C+LgrcSBOzNLsWT+RAc=
-github.com/D4-project/d4-golang-utils v0.0.0-20190603131519-c10ee092655c/go.mod h1:2rq8KBQnNNDocwc/49cnpaqoQA/komoSHKom7ynvqJc=
-github.com/gofrs/uuid v3.2.0+incompatible h1:y12jRkkFxsd7GpqdSZ+/KCs/fJbqpEXSGd4+jfEaewE=
+github.com/D4-project/d4-golang-utils v0.1.14 h1:APwN+i4qyDrxT8gvlbeV/VXfNas2GvPWOnkTGX1K2Lo=
+github.com/D4-project/d4-golang-utils v0.1.14/go.mod h1:qXVZ3kCL72i3uYe29t7aEy9dU0bNqtFvcoNE1dJu0zo=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
+github.com/gofrs/uuid v4.4.0+incompatible h1:3qXRTX8/NbyulANqlc0lchS1gqAVxRgsuW1YrTJupqA=
+github.com/gofrs/uuid v4.4.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
+github.com/gomodule/redigo v2.0.0+incompatible h1:K/R+8tc58AaqLkqG2Ol3Qk+DR/TlNuhuh457pBFPtt0=
+github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4=
+github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
+github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/redislabs/redisgraph-go v2.0.2+incompatible/go.mod h1:GYn4oUFkbkHx49xm2H4G8jZziqKDKdRtDUuTBZTmqBE=
+github.com/rjeczalik/notify v0.9.2/go.mod h1:aErll2f0sUX9PXZnVNyeiObbmTlk5jnMoCa4QEjJeqM=
+github.com/rjeczalik/notify v0.9.3 h1:6rJAzHTGKXGj76sbRgDiDcYj/HniypXmSJo1SWakZeY=
+github.com/rjeczalik/notify v0.9.3/go.mod h1:gF3zSOrafR9DQEWSE8TjfI9NkooDxbyT4UgRGKZA0lc=
+github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
+github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
+golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
+golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180926160741-c2ed4eda69e7/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
+golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
+golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
+golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
+golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=