Merge pull request #21 from D4-project/dependabot/go_modules/golang.org/x/net-0.17.0

build(deps): bump golang.org/x/net from 0.12.0 to 0.17.0

.gitignore

@@ -5,9 +5,7 @@
 *.so
 *.dylib
 *.vscode
-
-# Test binary, build with `go test -c`
-*.test
+*.idea
 
 # Output of the go coverage tool, specifically when used with LiteIDE
 *.out
@@ -17,6 +15,7 @@ d4-arm5l
 d4-amd64l
 # Output binaries from gox
 d4-goclient_*
+d4-goclient
 
 # Configuration files
-/conf.sample/*
+/conf.*

Makefile

@@ -1,4 +1,4 @@
 arm5l: d4-goclient.go
-	env GOOS=linux GOARCH=arm GOARM=5 go build -o d4-arm5l d4-goclient.go
+	env GOOS=linux GOARCH=arm GOARM=5 go build -o d4-goclient-arm5l d4-goclient.go
 amd64l: d4-goclient.go
-	env GOOS=linux GOARCH=amd64  go build -o d4-amd64l d4-goclient.go
+	env GOOS=linux GOARCH=amd64  go build -o d4-goclient-amd64l d4-goclient.go

README.md

@@ -1,6 +1,13 @@
-# d4-goclient
+<p align="center">
+  <img alt="d4-goclient" src="https://raw.githubusercontent.com/D4-project/d4-goclient/master/media/gopherd4.png" height="140" />
+  <p align="center">
+    <a href="https://github.com/D4-project/d4-goclient/releases/latest"><img alt="Release" src="https://img.shields.io/github/release/D4-project/d4-goclient/all.svg"></a>
+    <a href="https://github.com/D4-project/d4-goclient/blob/master/LICENSE"><img alt="Software License" src="https://img.shields.io/badge/License-MIT-yellow.svg"></a>
+    <a href="https://goreportcard.com/report/github.com/D4-Project/d4-goclient"><img alt="Go Report Card" src="https://goreportcard.com/badge/github.com/D4-Project/d4-goclient"></a>
+  </p>
+</p>
 
-A D4 project client (sensor) implementing the [D4 encapsulation protocol](https://github.com/D4-project/architecture/tree/master/format).
+**d4-goclient** is a D4 project client (sensor) implementing the [D4 encapsulation protocol](https://github.com/D4-project/architecture/tree/master/format).
 
 The client can be used on different targets and architectures to collect network capture, logs, specific network monitoring and send it
 back to a [D4 server](https://github.com/D4-project/d4-core).
@@ -12,21 +19,12 @@ For more information about the [D4 project](https://www.d4-project.org/).
 Fetch d4-goclient code and dependencies
 
 ```bash
-go get github.com/satori/go.uuid
 go get github.com/D4-project/d4-goclient
 ```
 
-Use make to build binaries:
-
-```bash
-make arm5l  # for raspberry pi / linux
-make amd64l # for amd64 / linux
-```
-
 ## Dependencies
 
- - golang 1.10 (tested)
- - go.uuid
+ - golang 1.13 (tested)
 
 # Use
 
@@ -35,7 +33,45 @@ make amd64l # for amd64 / linux
 See https://github.com/D4-project/d4-core/tree/master/server
 $IP_SRV being the d4-server's address, $PORT its listening port
 
+## Configuration files
+Part of the client configuration can be stored in folder containing the following files:
+
+ - key: your Pre-Shared-Key
+ - snaplen: default is 4096
+ - source: stdin or d4server
+ - destination: stdout, [fe80::ffff:ffff:ffff:a6fb]:4443, 127.0.0.1:4443
+ - type: D4 packet type, see [types](https://github.com/D4-project/architecture/tree/master/format)
+ - uuid: generated automatically if empty
+ - version: protocol version
+ - rootCA.crt: optional : CA certificate to check the server certificate
+ - metaheader.json: optional : a json file describing feed's meta-type [types](https://github.com/D4-project/architecture/tree/master/format)
+ 
+If source is set to d4server, then one also 2 additional files:
+ - redis_queue: redis queue in the form analyzer:typeofqueue:queueuuid, for instance `analyzer:3:d42967c1-f7ad-464e-bbc7-4464c653d7a6`
+ - redis_d4: redis server location:port/database, for instance localhost:6385/2
+
+## Flags
+
+```bash
+  -c string
+    	configuration directory
+  -cc
+    	Check TLS certificate against rootCA.crt
+  -ce
+    	Set to True, true, TRUE, 1, or t to enable TLS on network destination (default true)
+  -cka duration
+    	Keep Alive time human format, 0 to disable (default 30s)
+  -ct duration
+    	Set timeout in human format
+  -rl duration
+        Rate limiter: time in human format before retry after EOF (default 200ms)
+  -rt duration
+    	Time in human format before retry after connection failure, set to 0 to exit on failure (default 30s)
+  -v	Set to True, true, TRUE, 1, or t to enable verbose output on stdout
+```
+
 ## Pipe data into the client
+In the followin examples, destination is set to stdout.
 
 ### Some file
 ```bash
@@ -47,3 +83,8 @@ $IP being the monitoring computer ip
 ```bash
 tcpdump not dst $IP and not src $IP -w - | ./d4-goclient -c conf.sample/ |  socat - OPENSSL-CONNECT:$IP_SRV:$PORT,verify=0
 ```
+
+## Forwarding data from a D4 server to another D4 server
+Add two files to you configuration folder: `redis_d4` and `redis_queue`:
+ - `redis_d4` contains the location of the source d4's redis server database, for instance `127.0.0.1:6380/2`
+ - `redis_queue` contains the queue to forward to the other D4 server, for instance `analyzer:3:d42967c1-f7ad-464e-bbc7-4464c653d7a6`

conf.sample/destination

@@ -1 +1 @@
-127.0.0.1:4443
+stdout

conf.sample/redis_d4

@@ -0,0 +1 @@
+localhost:6385/2

conf.sample/redis_queue

@@ -0,0 +1 @@
+analyzer:3:d42967c1-f7ad-464e-bbc7-4464c653d7a6

conf.sample/source

@@ -1 +1 @@
-stdin
+d4server

conf.sample/uuid

@@ -0,0 +1 @@
+1d940331-9fc9-4381-8fc9-3b624db66025

d4-goclient.go

@@ -7,8 +7,10 @@ import (
 	"crypto/tls"
 	"crypto/x509"
 	"encoding/binary"
+	"encoding/json"
 	"flag"
 	"fmt"
+	"golang.org/x/net/proxy"
 	"io"
 	"io/ioutil"
 	"log"
@@ -17,31 +19,37 @@ import (
 	"os/signal"
 	"strconv"
 	"strings"
+	"syscall"
 	"time"
 
-	//BSD 3
-	uuid "github.com/satori/go.uuid"
+	config "github.com/D4-project/d4-golang-utils/config"
+	uuid "github.com/D4-project/d4-golang-utils/crypto/hash"
+	"github.com/D4-project/d4-golang-utils/inputreader"
+	"github.com/gomodule/redigo/redis"
 )
 
 const (
-	// Version Size
+	// VERSION_SIZE
 	VERSION_SIZE = 1
-	// Type Size
+	// TYPE_SIZE
 	TYPE_SIZE = 1
-	// UUID v4 Size
+	// UUID_SIZE
 	UUID_SIZE = 16
-	// Timestamp Size
+	// TIMESTAMP_SIZE
 	TIMESTAMP_SIZE = 8
-	// HMAC-SHA256 MAC Size
+	// HMAC_SIZE
 	HMAC_SIZE = 32
-	// Payload size Size
+	// SSIZE payload size size
 	SSIZE = 4
 
+	// HDR_SIZE total header size
 	HDR_SIZE = VERSION_SIZE + TYPE_SIZE + UUID_SIZE + HMAC_SIZE + TIMESTAMP_SIZE + SSIZE
+
+	// MH_FILE_LIMIT defines in bytes the max size of the json meta header file
+	MH_FILE_LIMIT = 100000
 )
 
 type (
-
 	// A d4 writer implements the io.Writer Interface by implementing Write() and Close()
 	// it accepts an io.Writer as sink
 	d4Writer struct {
@@ -52,19 +60,27 @@ type (
 	}
 
 	d4S struct {
-		src       io.Reader
-		dst       d4Writer
-		confdir   string
-		cka       time.Duration
-		ct        time.Duration
-		ce        bool
-		retry     time.Duration
-		cc        bool
-		ca        x509.CertPool
-		d4error   uint8
-		errnoCopy uint8
-		debug     bool
-		conf      d4params
+		src            io.Reader
+		dst            d4Writer
+		confdir        string
+		cka            time.Duration
+		ct             time.Duration
+		ce             bool
+		retry          time.Duration
+		rate           time.Duration
+		cc             bool
+		tor            bool
+		daily          bool
+		json           bool
+		ca             x509.CertPool
+		d4error        uint8
+		errnoCopy      uint8
+		debug          bool
+		conf           d4params
+		mhb            *bytes.Buffer
+		mh             []byte
+		redisInputPool *redis.Pool
+		redisCon       redis.Conn
 	}
 
 	d4params struct {
@@ -75,28 +91,39 @@ type (
 		source      string
 		destination string
 		ttype       uint8
+		redisHost   string
+		redisPort   string
+		redisQueue  string
+		redisDB     int
+		folderstr   string
 	}
 )
 
 var (
-	// verbose
-	buf    bytes.Buffer
-	logger = log.New(&buf, "INFO: ", log.Lshortfile)
-	infof  = func(info string) {
-		logger.Output(2, info)
+	// Verbose mode and logging
+	buf      bytes.Buffer
+	logger   = log.New(&buf, "INFO: ", log.Lshortfile)
+	debugger = log.New(&buf, "DEBUG: ", log.Lmicroseconds)
+	debugf   = func(debug string) {
+		debugger.Println("", debug)
 	}
 
 	tmpct, _    = time.ParseDuration("5mn")
 	tmpcka, _   = time.ParseDuration("30s")
 	tmpretry, _ = time.ParseDuration("30s")
+	tmprate, _  = time.ParseDuration("200ms")
 
-	confdir = flag.String("c", "", "configuration directory")
-	debug   = flag.Bool("v", false, "Set to True, true, TRUE, 1, or t to enable verbose output on stdout")
-	ce      = flag.Bool("ce", true, "Set to True, true, TRUE, 1, or t to enable TLS on network destination")
-	ct      = flag.Duration("ct", tmpct, "Set timeout in human format")
-	cka     = flag.Duration("cka", tmpcka, "Keep Alive time human format, 0 to disable")
-	retry   = flag.Duration("rt", tmpretry, "Time in human format before retry after connection failure, set to 0 to exit on failure")
-	cc      = flag.Bool("cc", false, "Check TLS certificate againt rootCA.crt")
+	confdir   = flag.String("c", "", "configuration directory")
+	debug     = flag.Bool("v", false, "Set to True, true, TRUE, 1, or t to enable verbose output on stdout - Don't use in production")
+	ce        = flag.Bool("ce", true, "Set to True, true, TRUE, 1, or t to enable TLS on network destination")
+	ct        = flag.Duration("ct", tmpct, "Set timeout in human format")
+	cka       = flag.Duration("cka", tmpcka, "Keep Alive time human format, 0 to disable")
+	retry     = flag.Duration("rt", tmpretry, "Time in human format before retry after connection failure, set to 0 to exit on failure")
+	rate      = flag.Duration("rl", tmprate, "Rate limiter: time in human format before retry after EOF")
+	cc        = flag.Bool("cc", false, "Check TLS certificate against rootCA.crt")
+	torflag   = flag.Bool("tor", false, "Use a SOCKS5 tor proxy on 9050")
+	dailyflag = flag.Bool("daily", false, "Sets up filewatcher to watch a new %Y%M%D folder at midnight")
+	jsonflag  = flag.Bool("json", false, "The files watched are json files")
 )
 
 func main() {
@@ -104,6 +131,16 @@ func main() {
 	var d4 d4S
 	d4p := &d4
 
+	// Setting up log file
+	f, err := os.OpenFile("d4-goclient.log", os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
+	if err != nil {
+		log.Fatalf("error opening file: %v", err)
+	}
+	defer f.Close()
+	logger.SetOutput(f)
+	logger.SetFlags(log.LstdFlags | log.Lshortfile)
+	logger.Println("Init")
+
 	flag.Usage = func() {
 		fmt.Printf("d4 - d4 client\n")
 		fmt.Printf("Read data from the configured <source> and send it to <destination>\n")
@@ -123,13 +160,9 @@ func main() {
 		fmt.Printf("type        - the type of data that is send. pcap, netflow, ...\n")
 		fmt.Printf("source      - the source where the data is read from\n")
 		fmt.Printf("destination - the destination where the data is written to\n")
+		fmt.Printf("redis_d4    - location of redis d4 server\n")
+		fmt.Printf("redis_queue - analyzer:type:queueuuid to pop\n")
 		fmt.Printf("\n")
-		fmt.Printf("-v [TRUE] for verbose output on stdout")
-		fmt.Printf("-ce [TRUE] if destination is set to ip:port, use of tls")
-		fmt.Printf("-cc [FALSE] if destination is set to ip:port, verification of server's tls certificate againt rootCA.crt")
-		fmt.Printf("-ct [300] if destination is set to ip:port, timeout")
-		fmt.Printf("-cka [3600] if destination is set to ip:port, keepalive")
-		fmt.Printf("-retry [5] if destination is set to ip:port, retry period ")
 		flag.PrintDefaults()
 	}
 
@@ -141,45 +174,144 @@ func main() {
 		*confdir = strings.TrimSuffix(*confdir, "/")
 		*confdir = strings.TrimSuffix(*confdir, "\\")
 	}
+
 	d4.confdir = *confdir
 	d4.ce = *ce
 	d4.ct = *ct
 	d4.cc = *cc
+	d4.json = *jsonflag
 	d4.cka = *cka
 	d4.retry = *retry
+	d4.rate = *rate
+	d4.tor = *torflag
+	d4.daily = *dailyflag
 
 	s := make(chan os.Signal, 1)
 	signal.Notify(s, os.Interrupt, os.Kill)
-	c := make(chan string)
-	k := make(chan string)
-	for {
-		if set(d4p) {
-			go d4Copy(d4p, c, k)
-		} else if d4.retry > 0 {
-			go func() {
-				time.Sleep(d4.retry)
-				infof(fmt.Sprintf("Sleeping for %f seconds before retry.\n", d4.retry.Seconds()))
-				c <- "done waiting"
-			}()
-		} else {
-			panic("Unrecoverable error without retry.")
+	errchan := make(chan error)
+	eofchan := make(chan string)
+	metachan := make(chan string)
+
+	// Launching the Rate limiters
+	rateLimiter := time.Tick(d4.rate)
+	retryLimiter := time.Tick(d4.retry)
+
+	//Setup
+	if !d4loadConfig(d4p) {
+		panic("Could not load Config.")
+	}
+	if !setReaderWriters(d4p, false) {
+		panic("Could not Init Inputs Outputs.")
+	}
+	if !d4.dst.initHeader(d4p) {
+		panic("Could not Init Headers.")
+	}
+
+	// force is a flag that forces the creation of a new connection
+	force := false
+
+	// On the first run, we send d4 meta header for type 2/254
+	if d4.conf.ttype == 254 || d4.conf.ttype == 2 {
+		go sendMeta(d4p, errchan, metachan)
+	H:
+		for {
+			select {
+			case <-errchan:
+				select {
+				case <-retryLimiter:
+					go sendMeta(d4p, errchan, metachan)
+				case <-s:
+					logger.Println("Exiting")
+					exit(d4p, 0)
+				}
+			case <-metachan:
+				break H
+			}
 		}
+	}
+
+	// Launch copy routine
+	go d4Copy(d4p, errchan, eofchan)
+
+	// Handle signals
+	for {
 		select {
-		case str := <-c:
-			fmt.Println(str)
-			continue
-		case str := <-k:
-			fmt.Println(str)
-			exit(d4p, 1)
+		// Case where the input ran out of data to consume1
+		case <-eofchan:
+			// We wait for ratelimiter before polling again
+		EOF:
+			for {
+				select {
+				case <-rateLimiter:
+					// copy routine
+					go d4Copy(d4p, errchan, eofchan)
+					break EOF
+					// Exit signal
+				case <-s:
+					logger.Println("Exiting")
+					exit(d4p, 0)
+				}
+			}
+		// ERROR, we check first whether it is network related
+		case err := <-errchan:
+			// On connection errors, we force setReaderWriter to reset the connection
+			force = false
+			switch t := err.(type) {
+			case *net.OpError:
+				force = true
+				if t.Op == "dial" {
+					logger.Println("Unknown Host")
+				} else if t.Op == "read" {
+					logger.Println("Connection Refused")
+				} else if t.Op == "write" {
+					logger.Println("Write error")
+				}
+			case syscall.Errno:
+				if t == syscall.ECONNREFUSED {
+					force = true
+					logger.Println("Connection Refused")
+				}
+			}
+			// We wait for retryLimiter before writing again
+		RETRY:
+			for {
+				select {
+				case <-retryLimiter:
+					if !setReaderWriters(d4p, force) {
+						// Can't connect, we break to retry
+						// force is still true
+						break
+					}
+					if !d4.dst.initHeader(d4p) {
+						panic("Could not Init Headers.")
+					}
+					if (d4.conf.ttype == 254 || d4.conf.ttype == 2) && force {
+						// setReaderWriter is happy, we should have a working
+						// connection from now on.
+						force = false
+						// Sending meta header for the first time on this new connection
+						go sendMeta(d4p, errchan, metachan)
+					}
+					break RETRY
+					// Exit signal
+				case <-s:
+					logger.Println("Exiting")
+					exit(d4p, 0)
+				}
+			}
+		// metaheader sent, launch the copy routine
+		case <-metachan:
+			go d4Copy(d4p, errchan, eofchan)
+		// Exit signal
 		case <-s:
-			fmt.Println(" Exiting")
+			logger.Println("Exiting")
 			exit(d4p, 0)
 		}
 	}
 }
 
 func exit(d4 *d4S, exitcode int) {
-	// Output logging before closing if debug is enabled
+	// Output debug info in the log before closing if debug is enabled
 	if *debug == true {
 		(*d4).debug = true
 		fmt.Print(&buf)
@@ -187,58 +319,79 @@ func exit(d4 *d4S, exitcode int) {
 	os.Exit(exitcode)
 }
 
-func set(d4 *d4S) bool {
-	if d4loadConfig(d4) {
-		if setReaderWriters(d4) {
-			if d4.dst.initHeader(d4) {
-				return true
-			}
-		}
+func d4Copy(d4 *d4S, errchan chan error, eofchan chan string) {
+	nread, err := io.CopyBuffer(&d4.dst, d4.src, d4.dst.pb)
+	// Always retry
+	if err != nil {
+		logger.Printf("D4copy: %s", err)
+		errchan <- err
+		return
 	}
-	return false
+	eofchan <- fmt.Sprintf("EOF: Nread: %d", nread)
 }
 
-func d4Copy(d4 *d4S, c chan string, k chan string) {
-	nread, err := io.CopyBuffer(&d4.dst, d4.src, d4.dst.pb)
+func sendMeta(d4 *d4S, errchan chan error, metachan chan string) {
+	// Fill metaheader buffer with metaheader data
+	d4.mhb = bytes.NewBuffer(d4.mh)
+	d4.dst.hijackHeader()
+	// Ugly hack to skip bytes.Buffer WriteTo check that bypasses my fixed lenght buffer
+	nread, err := io.CopyBuffer(&d4.dst, struct{ io.Reader }{d4.mhb}, d4.dst.pb)
 	if err != nil {
-		if (d4.retry.Seconds()) > 0 {
-			c <- fmt.Sprintf("%s", err)
-			return
-		} else {
-			k <- fmt.Sprintf("%s", err)
-			return
-		}
+		logger.Printf("Cannot sent meta-header: %s", err)
+		errchan <- err
+		return
 	}
-	k <- fmt.Sprintf("EOF: Nread: %d", nread)
+	logger.Println(fmt.Sprintf("Meta-Header sent: %d bytes", nread))
+	d4.dst.restoreHeader()
+	metachan <- "Header Sent"
 	return
 }
 
 func readConfFile(d4 *d4S, fileName string) []byte {
-	f, err := os.OpenFile((*d4).confdir+"/"+fileName, os.O_RDWR|os.O_CREATE, 0666)
-	defer f.Close()
-	if err != nil {
-		log.Fatal(err)
-	}
-	data := make([]byte, 100)
-	count, err := f.Read(data)
-	if err != nil {
-		if err != io.EOF {
-			log.Fatal(err)
-		}
-	}
-	infof(fmt.Sprintf("read %d bytes: %q\n", count, data[:count]))
-	if err := f.Close(); err != nil {
-		log.Fatal(err)
-	}
-	// trim \n if present
-	return bytes.TrimSuffix(data[:count], []byte("\n"))
+	return config.ReadConfigFile((*d4).confdir, fileName)
 }
 
 func d4loadConfig(d4 *d4S) bool {
 	// populate the map
 	(*d4).conf = d4params{}
 	(*d4).conf.source = string(readConfFile(d4, "source"))
+	if len((*d4).conf.source) < 1 {
+		log.Fatal("Unsupported source")
+	}
+	if (*d4).conf.source == "folder" {
+		fstr := string(readConfFile(d4, "folder"))
+		if ffd, err := os.Stat(fstr); os.IsNotExist(err) {
+			log.Fatal("Folder does not exist")
+		} else {
+			if !ffd.IsDir() {
+				log.Fatal("Folder is not a directory")
+			}
+		}
+		(*d4).conf.folderstr = fstr
+	}
+	if (*d4).conf.source == "d4server" {
+		// Parse Input Redis Config
+		tmp := string(readConfFile(d4, "redis_d4"))
+		ss := strings.Split(string(tmp), "/")
+		if len(ss) <= 1 {
+			log.Fatal("Missing Database in Redis input config: should be host:port/database_name")
+		}
+		(*d4).conf.redisDB, _ = strconv.Atoi(ss[1])
+		var ret bool
+		ret, ss[0] = config.IsNet(ss[0])
+		if ret {
+			sss := strings.Split(string(ss[0]), ":")
+			(*d4).conf.redisHost = sss[0]
+			(*d4).conf.redisPort = sss[1]
+		} else {
+			log.Fatal("Redis config error.")
+		}
+		(*d4).conf.redisQueue = string(config.ReadConfigFile(*confdir, "redis_queue"))
+	}
 	(*d4).conf.destination = string(readConfFile(d4, "destination"))
+	if len((*d4).conf.destination) < 1 {
+		log.Fatal("Unsupported Destination")
+	}
 	tmpu, err := uuid.FromString(string(readConfFile(d4, "uuid")))
 	if err != nil {
 		// generate new uuid
@@ -255,15 +408,60 @@ func d4loadConfig(d4 *d4S) bool {
 		(*d4).conf.uuid = tmpu.Bytes()
 	}
 	// parse snaplen to uint32
-	tmp, _ := strconv.ParseUint(string(readConfFile(d4, "snaplen")), 10, 32)
-	(*d4).conf.snaplen = uint32(tmp)
+	tmp, err := strconv.ParseUint(string(readConfFile(d4, "snaplen")), 10, 32)
+	if err != nil || tmp < 1 {
+		(*d4).conf.snaplen = uint32(4096)
+	} else {
+		(*d4).conf.snaplen = uint32(tmp)
+	}
 	(*d4).conf.key = readConfFile(d4, "key")
 	// parse version to uint8
 	tmp, _ = strconv.ParseUint(string(readConfFile(d4, "version")), 10, 8)
-	(*d4).conf.version = uint8(tmp)
+	if err != nil || tmp < 1 {
+		(*d4).conf.version = uint8(1)
+	} else {
+		(*d4).conf.version = uint8(tmp)
+	}
 	// parse type to uint8
 	tmp, _ = strconv.ParseUint(string(readConfFile(d4, "type")), 10, 8)
-	(*d4).conf.ttype = uint8(tmp)
+	if err != nil || tmp < 1 {
+		log.Fatal("Unsupported type")
+	} else {
+		(*d4).conf.ttype = uint8(tmp)
+	}
+	// parse meta header file
+	data := make([]byte, MH_FILE_LIMIT)
+	if tmp == 254 || tmp == 2 {
+		file, err := os.Open((*d4).confdir + "/metaheader.json")
+		defer file.Close()
+		if err != nil {
+			panic("Failed to open Meta-Header File.")
+		} else {
+			if count, err := file.Read(data); err != nil {
+				panic("Failed to open Meta-Header File.")
+			} else {
+				if json.Valid(data[:count]) {
+					if checkType(data[:count]) {
+						if off, err := file.Seek(0, 0); err != nil || off != 0 {
+							panic(fmt.Sprintf("Cannot read Meta-Header file: %s", err))
+						} else {
+							// create metaheader buffer
+							d4.mhb = bytes.NewBuffer(d4.mh)
+							if err := json.Compact((*d4).mhb, data[:count]); err != nil {
+								logger.Println("Failed to compact meta header file")
+							}
+							// Store the metaheader in d4 struct for subsequent retries
+							(*d4).mh = data[:count]
+						}
+					} else {
+						panic("A Meta-Header File should at least contain a 'type' field.")
+					}
+				} else {
+					panic("Failed to validate open Meta-Header File.")
+				}
+			}
+		}
+	}
 	// Add the custom CA cert in D4 certpool
 	if (*d4).cc {
 		certb, _ := ioutil.ReadFile((*d4).confdir + "rootCA.crt")
@@ -276,13 +474,31 @@ func d4loadConfig(d4 *d4S) bool {
 	return true
 }
 
+func checkType(b []byte) bool {
+	var f interface{}
+	if err := json.Unmarshal(b, &f); err != nil {
+		return false
+	}
+	m := f.(map[string]interface{})
+	for k, v := range m {
+		if k == "type" {
+			switch v.(type) {
+			case string:
+				if v != nil {
+					return true
+				}
+			}
+		}
+	}
+	return false
+}
+
 func newD4Writer(writer io.Writer, key []byte) d4Writer {
 	return d4Writer{w: writer, key: key}
 }
 
 // TODO QUICK IMPLEM, REVISE
-func setReaderWriters(d4 *d4S) bool {
-
+func setReaderWriters(d4 *d4S, force bool) bool {
 	//TODO implement other destination file, fifo unix_socket ...
 	switch (*d4).conf.source {
 	case "stdin":
@@ -290,38 +506,94 @@ func setReaderWriters(d4 *d4S) bool {
 	case "pcap":
 		f, _ := os.Open("capture.pcap")
 		(*d4).src = f
+	case "d4server":
+		// Create a new redis connection pool
+		(*d4).redisInputPool = newPool((*d4).conf.redisHost+":"+(*d4).conf.redisPort, 16)
+		var err error
+		(*d4).redisCon, err = (*d4).redisInputPool.Dial()
+		if err != nil {
+			logger.Println("Could not connect to d4 Redis")
+			return false
+		}
+		(*d4).src, err = inputreader.NewLPOPReader(&(*d4).redisCon, (*d4).conf.redisDB, (*d4).conf.redisQueue)
+		if err != nil {
+			log.Printf("Could not create d4 Redis Descriptor %q \n", err)
+			return false
+		}
+	case "folder":
+		var err error
+		(*d4).src, err = inputreader.NewFileWatcherReader((*d4).conf.folderstr, (*d4).json, (*d4).daily, logger)
+		if err != nil {
+			log.Printf("Could not create File Watcher %q \n", err)
+			return false
+		}
 	}
-	isn, dstnet := isNet((*d4).conf.destination)
+	isn, dstnet := config.IsNet((*d4).conf.destination)
 	if isn {
-		dial := net.Dialer{
-			DualStack:     true,
-			Timeout:       (*d4).ct,
-			KeepAlive:     (*d4).cka,
-			FallbackDelay: 0,
-		}
-		tlsc := tls.Config{
-			InsecureSkipVerify: true,
-		}
-		if (*d4).cc {
-			tlsc = tls.Config{
-				InsecureSkipVerify: false,
-				RootCAs:            &(*d4).ca,
+		// We test whether a connection already exist
+		// (case where the reader run out of data)
+		// force forces to reset the connections after
+		// failure to reuse it
+		if _, ok := (*d4).dst.w.(net.Conn); !ok || force {
+			if (*d4).tor {
+				dialer := net.Dialer{
+					Timeout:       (*d4).ct,
+					KeepAlive:     (*d4).cka,
+					FallbackDelay: 0,
+				}
+				dial, err := proxy.SOCKS5("tcp", "127.0.0.1:9050", nil, &dialer)
+				if err != nil {
+					log.Fatal(err)
+				}
+				tlsc := tls.Config{
+					InsecureSkipVerify: true,
+				}
+				if (*d4).cc {
+					tlsc = tls.Config{
+						InsecureSkipVerify: false,
+						RootCAs:            &(*d4).ca,
+					}
+				}
+				conn, errc := dial.Dial("tcp", dstnet)
+				if errc != nil {
+					logger.Println(errc)
+					return false
+				}
+				if (*d4).ce == true {
+					conn = tls.Client(conn, &tlsc) // use tls
+				}
+				(*d4).dst = newD4Writer(conn, (*d4).conf.key)
+			} else {
+				dial := net.Dialer{
+					Timeout:       (*d4).ct,
+					KeepAlive:     (*d4).cka,
+					FallbackDelay: 0,
+				}
+				tlsc := tls.Config{
+					InsecureSkipVerify: true,
+				}
+				if (*d4).cc {
+					tlsc = tls.Config{
+						InsecureSkipVerify: false,
+						RootCAs:            &(*d4).ca,
+					}
+				}
+				if (*d4).ce == true {
+					conn, errc := tls.DialWithDialer(&dial, "tcp", dstnet, &tlsc)
+					if errc != nil {
+						logger.Println(errc)
+						return false
+					}
+					(*d4).dst = newD4Writer(conn, (*d4).conf.key)
+				} else {
+					conn, errc := dial.Dial("tcp", dstnet)
+					if errc != nil {
+						return false
+					}
+					(*d4).dst = newD4Writer(conn, (*d4).conf.key)
+				}
 			}
 		}
-		if (*d4).ce == true {
-			conn, errc := tls.DialWithDialer(&dial, "tcp", dstnet, &tlsc)
-			if errc != nil {
-				fmt.Println(errc)
-				return false
-			}
-			(*d4).dst = newD4Writer(conn, (*d4).conf.key)
-		} else {
-			conn, errc := dial.Dial("tcp", dstnet)
-			if errc != nil {
-				return false
-			}
-			(*d4).dst = newD4Writer(conn, (*d4).conf.key)
-		}
 	} else {
 		switch (*d4).conf.destination {
 		case "stdout":
@@ -341,62 +613,12 @@ func setReaderWriters(d4 *d4S) bool {
 	return true
 }
 
-func isNet(host string) (bool, string) {
-	// Check ipv6
-	if strings.HasPrefix(host, "[") {
-		// Parse an IP-Literal in RFC 3986 and RFC 6874.
-		// E.g., "[fe80::1]", "[fe80::1%25en0]", "[fe80::1]:80".
-		i := strings.LastIndex(host, "]")
-		if i < 0 {
-			panic("Unmatched [ in destination config")
-		}
-		if !validPort(host[i+1:]) {
-			panic("No valid port specified")
-		}
-		// trim brackets
-
-		if net.ParseIP(strings.Trim(host[:i+1], "[]")) != nil {
-			infof(fmt.Sprintf("Server IP: %s, Server Port: %s\n", host[:i+1], host[i+1:]))
-			return true, host
-		}
-	} else {
-		// Ipv4
-		ss := strings.Split(string(host), ":")
-		if !validPort(":" + ss[1]) {
-			panic("No valid port specified")
-		}
-		if net.ParseIP(ss[0]) != nil {
-			infof(fmt.Sprintf("Server IP: %s, Server Port: %s\n", ss[0], ss[1]))
-			return true, host
-		}
-	}
-	return false, host
-}
-
-// Reusing code from net.url
-// validOptionalPort reports whether port is either an empty string
-// or matches /^:\d*$/
-func validPort(port string) bool {
-	if port == "" {
-		return false
-	}
-	if port[0] != ':' {
-		return false
-	}
-	for _, b := range port[1:] {
-		if b < '0' || b > '9' {
-			return false
-		}
-	}
-	return true
-}
-
 func generateUUIDv4() []byte {
 	uuid, err := uuid.NewV4()
 	if err != nil {
 		log.Fatal(err)
 	}
-	infof(fmt.Sprintf("UUIDv4: %s\n", uuid))
+	logger.Println(fmt.Sprintf("UUIDv4: %s", uuid))
 	return uuid.Bytes()
 }
 
@@ -440,7 +662,7 @@ func (d4w *d4Writer) updateHMAC(ps int) bool {
 func (d4w *d4Writer) initHeader(d4 *d4S) bool {
 	// zero out the header
 	copy(d4w.fb[:HDR_SIZE], make([]byte, HDR_SIZE))
-	// put version a type into the header
+	// put version and type into the header
 	d4w.fb[0] = (*d4).conf.version
 	d4w.fb[1] = (*d4).conf.ttype
 	// put uuid into the header
@@ -451,7 +673,29 @@ func (d4w *d4Writer) initHeader(d4 *d4S) bool {
 	// hmac is set to zero during hmac operations, so leave it alone
 	// init size of payload at 0
 	binary.LittleEndian.PutUint32(d4w.fb[58:62], uint32(0))
-	infof(fmt.Sprintf("Initialized a %d bytes header:\n", HDR_SIZE))
-	infof(fmt.Sprintf("%b\n", d4w.fb[:HDR_SIZE]))
+	debugf(fmt.Sprintf("Initialized a %d bytes header:\n", HDR_SIZE))
+	debugf(fmt.Sprintf("%b\n", d4w.fb[:HDR_SIZE]))
 	return true
 }
+
+// We use type 2 to send the meta header
+func (d4w *d4Writer) hijackHeader() bool {
+	d4w.fb[1] = 2
+	return true
+}
+
+// Switch back the header to 254
+func (d4w *d4Writer) restoreHeader() bool {
+	d4w.fb[1] = 254
+	return true
+}
+
+func newPool(addr string, maxconn int) *redis.Pool {
+	return &redis.Pool{
+		MaxActive:   maxconn,
+		MaxIdle:     3,
+		IdleTimeout: 240 * time.Second,
+		// Dial or DialContext must be set. When both are set, DialContext takes precedence over Dial.
+		Dial: func() (redis.Conn, error) { return redis.Dial("tcp", addr) },
+	}
+}

d4-goclient_test.go

@@ -0,0 +1,35 @@
+package main
+
+import (
+	"testing"
+)
+
+var testCases = []struct {
+	name     string
+	str      string
+	expected bool
+}{
+	{"Well-formed IPv4 with port", "127.0.0.1:4443", true},
+	{"Well-formed IPv4 without port", "127.0.0.1", false},
+	{"Malformed IPv4 with port", "127..0.1:4443", false},
+	{"Malformed IPv4 without port", "127..0.1", false},
+	{"Well-formed IPv6 with port - 2", "[::1]:4443", true},
+	{"Well-formed IPv6 without port", "[fe80::1%25en0]", false},
+	{"Malformed IPv6 with port", "[::::1]:4443", false},
+	{"Malformed IPv6 without port", "[::::::::1]", false},
+	{"Malformed IPv6 : missing square brackets", "::::::::1:4443", false},
+	{"Well-formed DNS name with port", "toto.circl.lu:4443", true},
+	{"Well-formed DNS name without port", "toto.circl.lu", false},
+	{"Malformed DNS name with port", ".:4443", false},
+}
+
+func TestIsNet(t *testing.T) {
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			b, _ := isNet(tc.str)
+			if b != tc.expected {
+				t.Fail()
+			}
+		})
+	}
+}

go.mod

@@ -0,0 +1,11 @@
+module github.com/D4-project/d4-goclient
+
+go 1.13
+
+require (
+	github.com/D4-project/d4-golang-utils v0.1.14
+	github.com/gofrs/uuid v4.4.0+incompatible // indirect
+	github.com/gomodule/redigo v2.0.0+incompatible
+	github.com/rjeczalik/notify v0.9.3 // indirect
+	golang.org/x/net v0.17.0
+)

go.sum

@@ -0,0 +1,63 @@
+github.com/D4-project/d4-golang-utils v0.1.14 h1:APwN+i4qyDrxT8gvlbeV/VXfNas2GvPWOnkTGX1K2Lo=
+github.com/D4-project/d4-golang-utils v0.1.14/go.mod h1:qXVZ3kCL72i3uYe29t7aEy9dU0bNqtFvcoNE1dJu0zo=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
+github.com/gofrs/uuid v4.4.0+incompatible h1:3qXRTX8/NbyulANqlc0lchS1gqAVxRgsuW1YrTJupqA=
+github.com/gofrs/uuid v4.4.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
+github.com/gomodule/redigo v2.0.0+incompatible h1:K/R+8tc58AaqLkqG2Ol3Qk+DR/TlNuhuh457pBFPtt0=
+github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4=
+github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
+github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/redislabs/redisgraph-go v2.0.2+incompatible/go.mod h1:GYn4oUFkbkHx49xm2H4G8jZziqKDKdRtDUuTBZTmqBE=
+github.com/rjeczalik/notify v0.9.2/go.mod h1:aErll2f0sUX9PXZnVNyeiObbmTlk5jnMoCa4QEjJeqM=
+github.com/rjeczalik/notify v0.9.3 h1:6rJAzHTGKXGj76sbRgDiDcYj/HniypXmSJo1SWakZeY=
+github.com/rjeczalik/notify v0.9.3/go.mod h1:gF3zSOrafR9DQEWSE8TjfI9NkooDxbyT4UgRGKZA0lc=
+github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
+github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
+golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
+golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180926160741-c2ed4eda69e7/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
+golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
+golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
+golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
+golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=