Merge pull request #5 from D4-project/DecodingLayerParser

Decoding layer parser

d4-tlsf.go

@@ -33,7 +33,6 @@ var nodefrag = flag.Bool("nodefrag", false, "If true, do not do IPv4 defrag")
 var checksum = flag.Bool("checksum", false, "Check TCP checksum")
 var nooptcheck = flag.Bool("nooptcheck", false, "Do not check TCP options (useful to ignore MSS on captures with TSO)")
 var ignorefsmerr = flag.Bool("ignorefsmerr", false, "Ignore TCP FSM errors")
-var allowmissinginit = flag.Bool("allowmissinginit", false, "Support streams without SYN/SYN+ACK/ACK sequence")
 var verbose = flag.Bool("verbose", false, "Be verbose")
 var debug = flag.Bool("debug", false, "Display debug information")
 var quiet = flag.Bool("quiet", false, "Be quiet regarding errors")
@@ -42,9 +41,6 @@ var quiet = flag.Bool("quiet", false, "Be quiet regarding errors")
 var iface = flag.String("i", "eth0", "Interface to read packets from")
 var fname = flag.String("r", "", "Filename to read from, overrides -i")
 
-// decoding
-//var LayerTypeETLS gopacket.LayerType
-
 // writing
 var outCerts = flag.String("w", "", "Folder to write certificates into")
 var outJSON = flag.String("j", "", "Folder to write certificates into, stdin if not set")
@@ -88,9 +84,8 @@ type tcpStreamFactory struct {
 }
 
 func (factory *tcpStreamFactory) New(net, transport gopacket.Flow, tcp *layers.TCP, ac reassembly.AssemblerContext) reassembly.Stream {
-	Debug("* NEW: %s %s\n", net, transport)
 	fsmOptions := reassembly.TCPSimpleFSMOptions{
-		SupportMissingEstablishment: *allowmissinginit,
+		SupportMissingEstablishment: true,
 	}
 	stream := &tcpStream{
 		net:        net,
@@ -134,37 +129,36 @@ type tcpStream struct {
 	urls           []string
 	ident          string
 	tlsSession     d4tls.TLSSession
+	ignorefsmerr   bool
+	nooptcheck     bool
+	checksum       bool
 	sync.Mutex
 }
 
 func (t *tcpStream) Accept(tcp *layers.TCP, ci gopacket.CaptureInfo, dir reassembly.TCPFlowDirection, nextSeq reassembly.Sequence, start *bool, ac reassembly.AssemblerContext) bool {
 	// FSM
 	if !t.tcpstate.CheckState(tcp, dir) {
-		Error("FSM", "%s: Packet rejected by FSM (state:%s)\n", t.ident, t.tcpstate.String())
 		if !t.fsmerr {
 			t.fsmerr = true
 		}
-		if !*ignorefsmerr {
+		if !t.ignorefsmerr {
 			return false
 		}
 	}
 	// Options
 	err := t.optchecker.Accept(tcp, ci, dir, nextSeq, start)
 	if err != nil {
-		Error("OptionChecker", "%s: Packet rejected by OptionChecker: %s\n", t.ident, err)
+		if !t.nooptcheck {
-		if !*nooptcheck {
 			return false
 		}
 	}
 	// Checksum
 	accept := true
-	if *checksum {
+	if t.checksum {
 		c, err := tcp.ComputeChecksum()
 		if err != nil {
-			Error("ChecksumCompute", "%s: Got error computing checksum: %s\n", t.ident, err)
 			accept = false
 		} else if c != 0x0 {
-			Error("Checksum", "%s: Invalid checksum: 0x%x\n", t.ident, c)
 			accept = false
 		}
 	}
@@ -174,7 +168,7 @@ func (t *tcpStream) Accept(tcp *layers.TCP, ci gopacket.CaptureInfo, dir reassem
 func (t *tcpStream) ReassembledSG(sg reassembly.ScatterGather, ac reassembly.AssemblerContext) {
 	_, _, _, skip := sg.Info()
 	length, _ := sg.Lengths()
-	if skip == -1 && *allowmissinginit {
+	if skip == -1 {
 		// this is allowed
 	} else if skip != 0 {
 		// Missing bytes in stream: do not even try to parse it
@@ -236,7 +230,6 @@ func getIPPorts(t *tcpStream) (string, string, string, string) {
 }
 
 func (t *tcpStream) ReassemblyComplete(ac reassembly.AssemblerContext) bool {
-	Debug("%s: Connection closed\n", t.ident)
 	// do not remove the connection to allow last ACK
 	return false
 }
@@ -272,12 +265,7 @@ func main() {
 		}
 	}
 
-	var dec gopacket.Decoder
+	source := gopacket.NewPacketSource(handle, handle.LinkType())
-	var ok bool
-	if dec, ok = gopacket.DecodersByLayerName["Ethernet"]; !ok {
-		log.Fatal("No eth decoder")
-	}
-	source := gopacket.NewPacketSource(handle, dec)
 	source.NoCopy = true
 	Info("Starting to read packets\n")
 	count := 0
@@ -301,12 +289,32 @@ func main() {
 	w.Add(1)
 	go processCompletedSession(jobQ, &w)
 
+	var eth layers.Ethernet
+	var ip4 layers.IPv4
+	var ip6 layers.IPv6
+	parser := gopacket.NewDecodingLayerParser(layers.LayerTypeEthernet, &eth, &ip4, &ip6)
+	decoded := []gopacket.LayerType{}
+
 	for packet := range source.Packets() {
 		count++
 		Debug("PACKET #%d\n", count)
-		data := packet.Data()
-		bytes += int64(len(data))
 
+		data := packet.Data()
+
+		if err := parser.DecodeLayers(data, &decoded); err != nil {
+			// Well it sures complaing about not knowing how to decode TCP
+		}
+
+		fmt.Printf("%s\n", ip4.SrcIP)
+		//		fmt.Printf("%s", hex.Dump(decoded))
+
+		for _, layerType := range decoded {
+			switch layerType {
+			case layers.LayerTypeIPv6:
+				fmt.Println("    IP6 ", ip6.SrcIP, ip6.DstIP)
+			case layers.LayerTypeIPv4:
+
+				fmt.Println("    IP4 ", ip4.SrcIP, ip4.DstIP)
 				// defrag the IPv4 packet if required
 				if !*nodefrag {
 					ip4Layer := packet.Layer(layers.LayerTypeIPv4)
@@ -348,6 +356,11 @@ func main() {
 					assembler.AssembleWithContext(packet.NetworkLayer().NetworkFlow(), tcp, &c)
 				}
 
+			}
+		}
+
+		bytes += int64(len(data))
+
 		var done bool
 		select {
 		case <-signalChan:
@@ -371,6 +384,16 @@ func main() {
 	w.Wait()
 }
 
+// Tries to enqueue or false
+func queueSession(t d4tls.TLSSession) bool {
+	select {
+	case jobQ <- t:
+		return true
+	default:
+		return false
+	}
+}
+
 func processCompletedSession(jobQ <-chan d4tls.TLSSession, w *sync.WaitGroup) {
 	for {
 		tlss, more := <-jobQ
@@ -383,16 +406,6 @@ func processCompletedSession(jobQ <-chan d4tls.TLSSession, w *sync.WaitGroup) {
 	}
 }
 
-// Tries to enqueue or false
-func queueSession(t d4tls.TLSSession) bool {
-	select {
-	case jobQ <- t:
-		return true
-	default:
-		return false
-	}
-}
-
 func output(t d4tls.TLSSession) {
 
 	jsonRecord, _ := json.MarshalIndent(t.Record, "", "    ")

d4tls/fingerprinter.go

@@ -1,17 +1,12 @@
 package d4tls
 
 import (
-	"bytes"
 	"crypto/md5"
-	"crypto/sha256"
-	"crypto/x509"
 	"encoding/hex"
 	"fmt"
 	"strconv"
 	"strings"
-	"time"
 
-	"github.com/D4-project/sensor-d4-tls-fingerprinting/etls"
 	"github.com/glaslos/tlsh"
 )
 
@@ -22,93 +17,6 @@ var grease = map[uint16]bool{
 	0xcaca: true, 0xdada: true, 0xeaea: true, 0xfafa: true,
 }
 
-type certMapElm struct {
-	CertHash string
-	*x509.Certificate
-}
-
-type sessionRecord struct {
-	ServerIP     string
-	ServerPort   string
-	ClientIP     string
-	ClientPort   string
-	TLSH         string
-	Timestamp    time.Time
-	JA3          string
-	JA3Digest    string
-	JA3S         string
-	JA3SDigest   string
-	Certificates []certMapElm
-}
-
-// TLSSession contains a handshakeRecord that had to be filled during the handshake,
-// and a Record that will be at last exported to Json
-type TLSSession struct {
-	Record          sessionRecord
-	handShakeRecord etls.ETLSHandshakeRecord
-	stage           int
-}
-
-// String returns a string that describes a TLSSession
-func (t *TLSSession) String() string {
-	var buf bytes.Buffer
-	buf.WriteString(fmt.Sprintf("---------------SESSION START-------------------\n"))
-	buf.WriteString(fmt.Sprintf("Time: %v\n", t.Record.Timestamp))
-	buf.WriteString(fmt.Sprintf("Client: %v:%v\n", t.Record.ClientIP, t.Record.ClientPort))
-	buf.WriteString(fmt.Sprintf("Server: %v:%v\n", t.Record.ServerIP, t.Record.ServerPort))
-	buf.WriteString(fmt.Sprintf("TLSH: %q\n", t.Record.TLSH))
-	buf.WriteString(fmt.Sprintf("ja3: %q\n", t.Record.JA3))
-	buf.WriteString(fmt.Sprintf("ja3 Digest: %q\n", t.Record.JA3Digest))
-	buf.WriteString(fmt.Sprintf("ja3s: %q\n", t.Record.JA3S))
-	buf.WriteString(fmt.Sprintf("ja3s Digest: %q\n", t.Record.JA3SDigest))
-	for _, certMe := range t.Record.Certificates {
-		buf.WriteString(fmt.Sprintf("Certificate Issuer: %q\n", certMe.Certificate.Issuer))
-		buf.WriteString(fmt.Sprintf("Certificate Subject: %q\n", certMe.Certificate.Subject))
-		buf.WriteString(fmt.Sprintf("Certificate is CA: %t\n", certMe.Certificate.IsCA))
-		buf.WriteString(fmt.Sprintf("Certificate SHA256: %q\n", certMe.CertHash))
-	}
-	buf.WriteString(fmt.Sprintf("---------------SESSION  END--------------------\n"))
-	return buf.String()
-}
-
-// PopulateClientHello takes a pointer to an etls ClientHelloMsg and writes it to the the TLSSession struct
-func (t *TLSSession) PopulateClientHello(h *etls.ClientHelloMsg, cip string, sip string, cp string, sp string, ti time.Time) {
-	if t.stage < 1 {
-		t.Record.ClientIP = cip
-		t.Record.ServerIP = sip
-		t.Record.ClientPort = cp
-		t.Record.ServerPort = sp
-		t.Record.Timestamp = ti
-		t.handShakeRecord.ETLSHandshakeClientHello = h
-		t.stage = 1
-	}
-}
-
-// PopulateServerHello takes a pointer to an etls ServerHelloMsg and writes it to the TLSSession struct
-func (t *TLSSession) PopulateServerHello(h *etls.ServerHelloMsg) {
-	if t.stage < 2 {
-		t.handShakeRecord.ETLSHandshakeServerHello = h
-		t.stage = 2
-	}
-}
-
-// PopulateCertificate takes a pointer to an etls ServerHelloMsg and writes it to the TLSSession struct
-func (t *TLSSession) PopulateCertificate(c *etls.CertificateMsg) {
-	if t.stage < 3 {
-		t.handShakeRecord.ETLSHandshakeCertificate = c
-		for _, asn1Data := range t.handShakeRecord.ETLSHandshakeCertificate.Certificates {
-			cert, err := x509.ParseCertificate(asn1Data)
-			if err != nil {
-				//return err
-			} else {
-				h := sha256.New()
-				h.Write(cert.Raw)
-				t.Record.Certificates = append(t.Record.Certificates, certMapElm{Certificate: cert, CertHash: fmt.Sprintf("%x", h.Sum(nil))})
-			}
-		}
-	}
-}
-
 // D4Fingerprinting computes fingerprints doh
 func (t *TLSSession) D4Fingerprinting(fd string) bool {
 	switch fd {

d4tls/fingerprinter_test.go

@@ -205,7 +205,6 @@ func TestJA3(t *testing.T) {
 					// Populate Client Hello related fields
 					tlss.PopulateClientHello(etlsrecord.ETLSHandshakeClientHello, "", "", "", "", time.Now())
 					tlss.D4Fingerprinting("ja3")
-					// TODO check that againt the reference python implementation
 					t.Logf("%v", tlss.Record.JA3)
 					t.Logf("%v", tlss.Record.JA3Digest)
 				}
@@ -229,7 +228,6 @@ func TestJA3s(t *testing.T) {
 					// Populate Server Hello related fields
 					tlss.PopulateServerHello(etlsrecord.ETLSHandshakeServerHello)
 					tlss.D4Fingerprinting("ja3s")
-					// TODO check that againt the reference python implementation
 					t.Logf("%v", tlss.Record.JA3S)
 					t.Logf("%v", tlss.Record.JA3SDigest)
 				}
@@ -253,7 +251,7 @@ func TestTLSH(t *testing.T) {
 					// Populate Cert
 					tlss.PopulateCertificate(etlsrecord.ETLSHandshakeCertificate)
 					tlss.D4Fingerprinting("tlsh")
-					// TODO check that againt the reference implementation
+					// TODO check that against the reference implementation
 					t.Logf("%v", tlss.Record.TLSH)
 				}
 			}

d4tls/tlsdecoder.go

@@ -0,0 +1,98 @@
+package d4tls
+
+import (
+	"bytes"
+	"crypto/sha256"
+	"crypto/x509"
+	"fmt"
+	"time"
+
+	"github.com/D4-project/sensor-d4-tls-fingerprinting/etls"
+)
+
+type certMapElm struct {
+	CertHash string
+	*x509.Certificate
+}
+
+type sessionRecord struct {
+	ServerIP     string
+	ServerPort   string
+	ClientIP     string
+	ClientPort   string
+	TLSH         string
+	Timestamp    time.Time
+	JA3          string
+	JA3Digest    string
+	JA3S         string
+	JA3SDigest   string
+	Certificates []certMapElm
+}
+
+// TLSSession contains a handshakeRecord that had to be filled during the handshake,
+// and a Record that will be at last exported to Json
+type TLSSession struct {
+	Record          sessionRecord
+	handShakeRecord etls.ETLSHandshakeRecord
+	stage           int
+}
+
+// String returns a string that describes a TLSSession
+func (t *TLSSession) String() string {
+	var buf bytes.Buffer
+	buf.WriteString(fmt.Sprintf("---------------SESSION START-------------------\n"))
+	buf.WriteString(fmt.Sprintf("Time: %v\n", t.Record.Timestamp))
+	buf.WriteString(fmt.Sprintf("Client: %v:%v\n", t.Record.ClientIP, t.Record.ClientPort))
+	buf.WriteString(fmt.Sprintf("Server: %v:%v\n", t.Record.ServerIP, t.Record.ServerPort))
+	buf.WriteString(fmt.Sprintf("TLSH: %q\n", t.Record.TLSH))
+	buf.WriteString(fmt.Sprintf("ja3: %q\n", t.Record.JA3))
+	buf.WriteString(fmt.Sprintf("ja3 Digest: %q\n", t.Record.JA3Digest))
+	buf.WriteString(fmt.Sprintf("ja3s: %q\n", t.Record.JA3S))
+	buf.WriteString(fmt.Sprintf("ja3s Digest: %q\n", t.Record.JA3SDigest))
+	for _, certMe := range t.Record.Certificates {
+		buf.WriteString(fmt.Sprintf("Certificate Issuer: %q\n", certMe.Certificate.Issuer))
+		buf.WriteString(fmt.Sprintf("Certificate Subject: %q\n", certMe.Certificate.Subject))
+		buf.WriteString(fmt.Sprintf("Certificate is CA: %t\n", certMe.Certificate.IsCA))
+		buf.WriteString(fmt.Sprintf("Certificate SHA256: %q\n", certMe.CertHash))
+	}
+	buf.WriteString(fmt.Sprintf("---------------SESSION  END--------------------\n"))
+	return buf.String()
+}
+
+// PopulateClientHello takes a pointer to an etls ClientHelloMsg and writes it to the the TLSSession struct
+func (t *TLSSession) PopulateClientHello(h *etls.ClientHelloMsg, cip string, sip string, cp string, sp string, ti time.Time) {
+	if t.stage < 1 {
+		t.Record.ClientIP = cip
+		t.Record.ServerIP = sip
+		t.Record.ClientPort = cp
+		t.Record.ServerPort = sp
+		t.Record.Timestamp = ti
+		t.handShakeRecord.ETLSHandshakeClientHello = h
+		t.stage = 1
+	}
+}
+
+// PopulateServerHello takes a pointer to an etls ServerHelloMsg and writes it to the TLSSession struct
+func (t *TLSSession) PopulateServerHello(h *etls.ServerHelloMsg) {
+	if t.stage < 2 {
+		t.handShakeRecord.ETLSHandshakeServerHello = h
+		t.stage = 2
+	}
+}
+
+// PopulateCertificate takes a pointer to an etls ServerHelloMsg and writes it to the TLSSession struct
+func (t *TLSSession) PopulateCertificate(c *etls.CertificateMsg) {
+	if t.stage < 3 {
+		t.handShakeRecord.ETLSHandshakeCertificate = c
+		for _, asn1Data := range t.handShakeRecord.ETLSHandshakeCertificate.Certificates {
+			cert, err := x509.ParseCertificate(asn1Data)
+			if err != nil {
+				//return err
+			} else {
+				h := sha256.New()
+				h.Write(cert.Raw)
+				t.Record.Certificates = append(t.Record.Certificates, certMapElm{Certificate: cert, CertHash: fmt.Sprintf("%x", h.Sum(nil))})
+			}
+		}
+	}
+}