From 00bd0836d9d4a47087dc7086a1d9ea329a9d75c8 Mon Sep 17 00:00:00 2001 From: Jean-Louis Huynen Date: Fri, 15 Feb 2019 10:28:25 +0100 Subject: [PATCH] splits fingerprinter and tlsdecoder --- d4-tlsf.go | 43 ++++---- d4tls/{d4tls.go => fingerprinter.go} | 92 ----------------- .../{d4tls_test.go => fingerprinter_test.go} | 0 d4tls/tlsdecoder.go | 98 +++++++++++++++++++ 4 files changed, 116 insertions(+), 117 deletions(-) rename d4tls/{d4tls.go => fingerprinter.go} (54%) rename d4tls/{d4tls_test.go => fingerprinter_test.go} (100%) create mode 100644 d4tls/tlsdecoder.go diff --git a/d4-tlsf.go b/d4-tlsf.go index e77639a..9e16ef7 100644 --- a/d4-tlsf.go +++ b/d4-tlsf.go @@ -33,7 +33,6 @@ var nodefrag = flag.Bool("nodefrag", false, "If true, do not do IPv4 defrag") var checksum = flag.Bool("checksum", false, "Check TCP checksum") var nooptcheck = flag.Bool("nooptcheck", false, "Do not check TCP options (useful to ignore MSS on captures with TSO)") var ignorefsmerr = flag.Bool("ignorefsmerr", false, "Ignore TCP FSM errors") -var allowmissinginit = flag.Bool("allowmissinginit", false, "Support streams without SYN/SYN+ACK/ACK sequence") var verbose = flag.Bool("verbose", false, "Be verbose") var debug = flag.Bool("debug", false, "Display debug information") var quiet = flag.Bool("quiet", false, "Be quiet regarding errors") @@ -42,9 +41,6 @@ var quiet = flag.Bool("quiet", false, "Be quiet regarding errors") var iface = flag.String("i", "eth0", "Interface to read packets from") var fname = flag.String("r", "", "Filename to read from, overrides -i") -// decoding -//var LayerTypeETLS gopacket.LayerType - // writing var outCerts = flag.String("w", "", "Folder to write certificates into") var outJSON = flag.String("j", "", "Folder to write certificates into, stdin if not set") @@ -88,9 +84,8 @@ type tcpStreamFactory struct { } func (factory *tcpStreamFactory) New(net, transport gopacket.Flow, tcp *layers.TCP, ac reassembly.AssemblerContext) reassembly.Stream { - Debug("* NEW: %s %s\n", net, transport) fsmOptions := reassembly.TCPSimpleFSMOptions{ - SupportMissingEstablishment: *allowmissinginit, + SupportMissingEstablishment: true, } stream := &tcpStream{ net: net, @@ -134,37 +129,36 @@ type tcpStream struct { urls []string ident string tlsSession d4tls.TLSSession + ignorefsmerr bool + nooptcheck bool + checksum bool sync.Mutex } func (t *tcpStream) Accept(tcp *layers.TCP, ci gopacket.CaptureInfo, dir reassembly.TCPFlowDirection, nextSeq reassembly.Sequence, start *bool, ac reassembly.AssemblerContext) bool { // FSM if !t.tcpstate.CheckState(tcp, dir) { - Error("FSM", "%s: Packet rejected by FSM (state:%s)\n", t.ident, t.tcpstate.String()) if !t.fsmerr { t.fsmerr = true } - if !*ignorefsmerr { + if !t.ignorefsmerr { return false } } // Options err := t.optchecker.Accept(tcp, ci, dir, nextSeq, start) if err != nil { - Error("OptionChecker", "%s: Packet rejected by OptionChecker: %s\n", t.ident, err) - if !*nooptcheck { + if !t.nooptcheck { return false } } // Checksum accept := true - if *checksum { + if t.checksum { c, err := tcp.ComputeChecksum() if err != nil { - Error("ChecksumCompute", "%s: Got error computing checksum: %s\n", t.ident, err) accept = false } else if c != 0x0 { - Error("Checksum", "%s: Invalid checksum: 0x%x\n", t.ident, c) accept = false } } @@ -174,7 +168,7 @@ func (t *tcpStream) Accept(tcp *layers.TCP, ci gopacket.CaptureInfo, dir reassem func (t *tcpStream) ReassembledSG(sg reassembly.ScatterGather, ac reassembly.AssemblerContext) { _, _, _, skip := sg.Info() length, _ := sg.Lengths() - if skip == -1 && *allowmissinginit { + if skip == -1 { // this is allowed } else if skip != 0 { // Missing bytes in stream: do not even try to parse it @@ -236,7 +230,6 @@ func getIPPorts(t *tcpStream) (string, string, string, string) { } func (t *tcpStream) ReassemblyComplete(ac reassembly.AssemblerContext) bool { - Debug("%s: Connection closed\n", t.ident) // do not remove the connection to allow last ACK return false } @@ -371,6 +364,16 @@ func main() { w.Wait() } +// Tries to enqueue or false +func queueSession(t d4tls.TLSSession) bool { + select { + case jobQ <- t: + return true + default: + return false + } +} + func processCompletedSession(jobQ <-chan d4tls.TLSSession, w *sync.WaitGroup) { for { tlss, more := <-jobQ @@ -383,16 +386,6 @@ func processCompletedSession(jobQ <-chan d4tls.TLSSession, w *sync.WaitGroup) { } } -// Tries to enqueue or false -func queueSession(t d4tls.TLSSession) bool { - select { - case jobQ <- t: - return true - default: - return false - } -} - func output(t d4tls.TLSSession) { jsonRecord, _ := json.MarshalIndent(t.Record, "", " ") diff --git a/d4tls/d4tls.go b/d4tls/fingerprinter.go similarity index 54% rename from d4tls/d4tls.go rename to d4tls/fingerprinter.go index dd006e2..6afacb8 100644 --- a/d4tls/d4tls.go +++ b/d4tls/fingerprinter.go @@ -1,17 +1,12 @@ package d4tls import ( - "bytes" "crypto/md5" - "crypto/sha256" - "crypto/x509" "encoding/hex" "fmt" "strconv" "strings" - "time" - "github.com/D4-project/sensor-d4-tls-fingerprinting/etls" "github.com/glaslos/tlsh" ) @@ -22,93 +17,6 @@ var grease = map[uint16]bool{ 0xcaca: true, 0xdada: true, 0xeaea: true, 0xfafa: true, } -type certMapElm struct { - CertHash string - *x509.Certificate -} - -type sessionRecord struct { - ServerIP string - ServerPort string - ClientIP string - ClientPort string - TLSH string - Timestamp time.Time - JA3 string - JA3Digest string - JA3S string - JA3SDigest string - Certificates []certMapElm -} - -// TLSSession contains a handshakeRecord that had to be filled during the handshake, -// and a Record that will be at last exported to Json -type TLSSession struct { - Record sessionRecord - handShakeRecord etls.ETLSHandshakeRecord - stage int -} - -// String returns a string that describes a TLSSession -func (t *TLSSession) String() string { - var buf bytes.Buffer - buf.WriteString(fmt.Sprintf("---------------SESSION START-------------------\n")) - buf.WriteString(fmt.Sprintf("Time: %v\n", t.Record.Timestamp)) - buf.WriteString(fmt.Sprintf("Client: %v:%v\n", t.Record.ClientIP, t.Record.ClientPort)) - buf.WriteString(fmt.Sprintf("Server: %v:%v\n", t.Record.ServerIP, t.Record.ServerPort)) - buf.WriteString(fmt.Sprintf("TLSH: %q\n", t.Record.TLSH)) - buf.WriteString(fmt.Sprintf("ja3: %q\n", t.Record.JA3)) - buf.WriteString(fmt.Sprintf("ja3 Digest: %q\n", t.Record.JA3Digest)) - buf.WriteString(fmt.Sprintf("ja3s: %q\n", t.Record.JA3S)) - buf.WriteString(fmt.Sprintf("ja3s Digest: %q\n", t.Record.JA3SDigest)) - for _, certMe := range t.Record.Certificates { - buf.WriteString(fmt.Sprintf("Certificate Issuer: %q\n", certMe.Certificate.Issuer)) - buf.WriteString(fmt.Sprintf("Certificate Subject: %q\n", certMe.Certificate.Subject)) - buf.WriteString(fmt.Sprintf("Certificate is CA: %t\n", certMe.Certificate.IsCA)) - buf.WriteString(fmt.Sprintf("Certificate SHA256: %q\n", certMe.CertHash)) - } - buf.WriteString(fmt.Sprintf("---------------SESSION END--------------------\n")) - return buf.String() -} - -// PopulateClientHello takes a pointer to an etls ClientHelloMsg and writes it to the the TLSSession struct -func (t *TLSSession) PopulateClientHello(h *etls.ClientHelloMsg, cip string, sip string, cp string, sp string, ti time.Time) { - if t.stage < 1 { - t.Record.ClientIP = cip - t.Record.ServerIP = sip - t.Record.ClientPort = cp - t.Record.ServerPort = sp - t.Record.Timestamp = ti - t.handShakeRecord.ETLSHandshakeClientHello = h - t.stage = 1 - } -} - -// PopulateServerHello takes a pointer to an etls ServerHelloMsg and writes it to the TLSSession struct -func (t *TLSSession) PopulateServerHello(h *etls.ServerHelloMsg) { - if t.stage < 2 { - t.handShakeRecord.ETLSHandshakeServerHello = h - t.stage = 2 - } -} - -// PopulateCertificate takes a pointer to an etls ServerHelloMsg and writes it to the TLSSession struct -func (t *TLSSession) PopulateCertificate(c *etls.CertificateMsg) { - if t.stage < 3 { - t.handShakeRecord.ETLSHandshakeCertificate = c - for _, asn1Data := range t.handShakeRecord.ETLSHandshakeCertificate.Certificates { - cert, err := x509.ParseCertificate(asn1Data) - if err != nil { - //return err - } else { - h := sha256.New() - h.Write(cert.Raw) - t.Record.Certificates = append(t.Record.Certificates, certMapElm{Certificate: cert, CertHash: fmt.Sprintf("%x", h.Sum(nil))}) - } - } - } -} - // D4Fingerprinting computes fingerprints doh func (t *TLSSession) D4Fingerprinting(fd string) bool { switch fd { diff --git a/d4tls/d4tls_test.go b/d4tls/fingerprinter_test.go similarity index 100% rename from d4tls/d4tls_test.go rename to d4tls/fingerprinter_test.go diff --git a/d4tls/tlsdecoder.go b/d4tls/tlsdecoder.go new file mode 100644 index 0000000..a61e38a --- /dev/null +++ b/d4tls/tlsdecoder.go @@ -0,0 +1,98 @@ +package d4tls + +import ( + "bytes" + "crypto/sha256" + "crypto/x509" + "fmt" + "time" + + "github.com/D4-project/sensor-d4-tls-fingerprinting/etls" +) + +type certMapElm struct { + CertHash string + *x509.Certificate +} + +type sessionRecord struct { + ServerIP string + ServerPort string + ClientIP string + ClientPort string + TLSH string + Timestamp time.Time + JA3 string + JA3Digest string + JA3S string + JA3SDigest string + Certificates []certMapElm +} + +// TLSSession contains a handshakeRecord that had to be filled during the handshake, +// and a Record that will be at last exported to Json +type TLSSession struct { + Record sessionRecord + handShakeRecord etls.ETLSHandshakeRecord + stage int +} + +// String returns a string that describes a TLSSession +func (t *TLSSession) String() string { + var buf bytes.Buffer + buf.WriteString(fmt.Sprintf("---------------SESSION START-------------------\n")) + buf.WriteString(fmt.Sprintf("Time: %v\n", t.Record.Timestamp)) + buf.WriteString(fmt.Sprintf("Client: %v:%v\n", t.Record.ClientIP, t.Record.ClientPort)) + buf.WriteString(fmt.Sprintf("Server: %v:%v\n", t.Record.ServerIP, t.Record.ServerPort)) + buf.WriteString(fmt.Sprintf("TLSH: %q\n", t.Record.TLSH)) + buf.WriteString(fmt.Sprintf("ja3: %q\n", t.Record.JA3)) + buf.WriteString(fmt.Sprintf("ja3 Digest: %q\n", t.Record.JA3Digest)) + buf.WriteString(fmt.Sprintf("ja3s: %q\n", t.Record.JA3S)) + buf.WriteString(fmt.Sprintf("ja3s Digest: %q\n", t.Record.JA3SDigest)) + for _, certMe := range t.Record.Certificates { + buf.WriteString(fmt.Sprintf("Certificate Issuer: %q\n", certMe.Certificate.Issuer)) + buf.WriteString(fmt.Sprintf("Certificate Subject: %q\n", certMe.Certificate.Subject)) + buf.WriteString(fmt.Sprintf("Certificate is CA: %t\n", certMe.Certificate.IsCA)) + buf.WriteString(fmt.Sprintf("Certificate SHA256: %q\n", certMe.CertHash)) + } + buf.WriteString(fmt.Sprintf("---------------SESSION END--------------------\n")) + return buf.String() +} + +// PopulateClientHello takes a pointer to an etls ClientHelloMsg and writes it to the the TLSSession struct +func (t *TLSSession) PopulateClientHello(h *etls.ClientHelloMsg, cip string, sip string, cp string, sp string, ti time.Time) { + if t.stage < 1 { + t.Record.ClientIP = cip + t.Record.ServerIP = sip + t.Record.ClientPort = cp + t.Record.ServerPort = sp + t.Record.Timestamp = ti + t.handShakeRecord.ETLSHandshakeClientHello = h + t.stage = 1 + } +} + +// PopulateServerHello takes a pointer to an etls ServerHelloMsg and writes it to the TLSSession struct +func (t *TLSSession) PopulateServerHello(h *etls.ServerHelloMsg) { + if t.stage < 2 { + t.handShakeRecord.ETLSHandshakeServerHello = h + t.stage = 2 + } +} + +// PopulateCertificate takes a pointer to an etls ServerHelloMsg and writes it to the TLSSession struct +func (t *TLSSession) PopulateCertificate(c *etls.CertificateMsg) { + if t.stage < 3 { + t.handShakeRecord.ETLSHandshakeCertificate = c + for _, asn1Data := range t.handShakeRecord.ETLSHandshakeCertificate.Certificates { + cert, err := x509.ParseCertificate(asn1Data) + if err != nil { + //return err + } else { + h := sha256.New() + h.Write(cert.Raw) + t.Record.Certificates = append(t.Record.Certificates, certMapElm{Certificate: cert, CertHash: fmt.Sprintf("%x", h.Sum(nil))}) + } + } + } +}