From f788f64fd0b9d84a083139c245702b116a1f7dde Mon Sep 17 00:00:00 2001 From: Kjell Tore Fossbakk Date: Thu, 28 Feb 2019 15:51:42 +0100 Subject: [PATCH] Allow for out-of-order TLS Handshake processing and partial outputs --- d4-tlsf.go | 48 +++++++++++++++++------- d4tls/fingerprinter.go | 2 + d4tls/flags.go | 19 ++++++++++ d4tls/tlsdecoder.go | 84 +++++++++++++++++++++++++++++------------- 4 files changed, 114 insertions(+), 39 deletions(-) create mode 100644 d4tls/flags.go diff --git a/d4-tlsf.go b/d4-tlsf.go index ebaf54f..f5610c2 100644 --- a/d4-tlsf.go +++ b/d4-tlsf.go @@ -36,6 +36,7 @@ var ignorefsmerr = flag.Bool("ignorefsmerr", false, "Ignore TCP FSM errors") var verbose = flag.Bool("verbose", false, "Be verbose") var debug = flag.Bool("debug", false, "Display debug information") var quiet = flag.Bool("quiet", false, "Be quiet regarding errors") +var partial = flag.Bool("partial", true, "Output partial TLS Sessions, e.g. where we don't see all three of client hello, server hello and certificate") // capture var iface = flag.String("i", "eth0", "Interface to read packets from") @@ -101,6 +102,11 @@ func (factory *tcpStreamFactory) New(net, transport gopacket.Flow, tcp *layers.T optchecker: reassembly.NewTCPOptionCheck(), tlsSession: d4tls.TLSSession{}, } + + // Set network information in the TLSSession + cip, sip, cp, sp := getIPPorts(stream) + stream.tlsSession.SetNetwork(cip, sip, cp, sp) + return stream } @@ -134,6 +140,7 @@ type tcpStream struct { urls []string ident string tlsSession d4tls.TLSSession + queued bool ignorefsmerr bool nooptcheck bool checksum bool @@ -181,6 +188,7 @@ func (t *tcpStream) ReassembledSG(sg reassembly.ScatterGather, ac reassembly.Ass } data := sg.Fetch(length) if t.isTLS { + if length > 0 { // We attempt to decode TLS tls := &etls.ETLS{} @@ -195,13 +203,18 @@ func (t *tcpStream) ReassembledSG(sg reassembly.ScatterGather, ac reassembly.Ass //Debug("TLS: %s\n", gopacket.LayerDump(tls)) // Debug("TLS: %s\n", gopacket.LayerGoString(tls)) if tls.Handshake != nil { + + // If the timestamp has not been set, we set it for the first time. + if t.tlsSession.Record.Timestamp.IsZero() { + info := sg.CaptureInfo(0) + t.tlsSession.SetTimestamp(info.Timestamp) + } + for _, tlsrecord := range tls.Handshake { switch tlsrecord.ETLSHandshakeMsgType { // Client Hello case 1: - info := sg.CaptureInfo(0) - cip, sip, cp, sp := getIPPorts(t) - t.tlsSession.PopulateClientHello(tlsrecord.ETLSHandshakeClientHello, cip, sip, cp, sp, info.Timestamp) + t.tlsSession.PopulateClientHello(tlsrecord.ETLSHandshakeClientHello) t.tlsSession.D4Fingerprinting("ja3") // Server Hello case 2: @@ -210,14 +223,15 @@ func (t *tcpStream) ReassembledSG(sg reassembly.ScatterGather, ac reassembly.Ass // Server Certificate case 11: t.tlsSession.PopulateCertificate(tlsrecord.ETLSHandshakeCertificate) - t.tlsSession.D4Fingerprinting("tlsh") - // If we get a cert, we consider the handshake as finished and ready to ship to D4 - queueSession(t.tlsSession) - default: - break } } + + // If the handshake is considered finished and we have not yet outputted it we ship it to output. + if t.tlsSession.HandshakeComplete() && !t.queued { + t.queueSession() + } + } } } @@ -235,6 +249,12 @@ func getIPPorts(t *tcpStream) (string, string, string, string) { } func (t *tcpStream) ReassemblyComplete(ac reassembly.AssemblerContext) bool { + // If the handshakre has not yet been outputted, but there are some information such as + // either the client hello or the server hello, we also output a partial. + if *partial && !t.queued && t.tlsSession.HandshakeAny() { + t.queueSession() + } + // remove connection from the pool return true } @@ -288,7 +308,7 @@ func main() { signal.Notify(signalChan, os.Interrupt) // Job chan to hold Completed sessions to write - jobQ = make(chan d4tls.TLSSession, 100) + jobQ = make(chan d4tls.TLSSession, 4096) cancelC := make(chan string) // We start a worker to send the processed TLS connection the outside world @@ -387,10 +407,13 @@ func main() { w.Wait() } -// Tries to enqueue or false -func queueSession(t d4tls.TLSSession) bool { +// queueSession tries to enqueue the tlsSession for output +// returns true if it succeeded or false if it failed to publish the +// tlsSession for output +func (t *tcpStream) queueSession() bool { + t.queued = true select { - case jobQ <- t: + case jobQ <- t.tlsSession: return true default: return false @@ -414,7 +437,6 @@ func processCompletedSession(cancelC <-chan string, jobQ <-chan d4tls.TLSSession } func output(t d4tls.TLSSession) { - jsonRecord, _ := json.MarshalIndent(t.Record, "", " ") // If an output folder was specified for certificates diff --git a/d4tls/fingerprinter.go b/d4tls/fingerprinter.go index 6afacb8..3b99206 100644 --- a/d4tls/fingerprinter.go +++ b/d4tls/fingerprinter.go @@ -49,6 +49,7 @@ func (t *TLSSession) d4fg() string { func (t *TLSSession) ja3s() bool { var buf []byte + buf = strconv.AppendInt(buf, int64(t.handShakeRecord.ETLSHandshakeServerHello.Vers), 10) // byte (44) is "," buf = append(buf, byte(44)) @@ -81,6 +82,7 @@ func (t *TLSSession) ja3s() bool { func (t *TLSSession) ja3() bool { var buf []byte + buf = strconv.AppendInt(buf, int64(t.handShakeRecord.ETLSHandshakeClientHello.Vers), 10) // byte (44) is "," buf = append(buf, byte(44)) diff --git a/d4tls/flags.go b/d4tls/flags.go new file mode 100644 index 0000000..2567adf --- /dev/null +++ b/d4tls/flags.go @@ -0,0 +1,19 @@ +package d4tls + +// HandshakeState is a flag which keeps record of which handeshake message types +// have been parsed. +type HandshakeState uint8 + +const ( + StateClientHello = 1 << iota + StateServerHello + StateCertificate +) + +func (s *HandshakeState) Set(flag HandshakeState) { + *s |= flag +} + +func (s HandshakeState) Has(flag HandshakeState) bool { + return s&flag != 0 +} diff --git a/d4tls/tlsdecoder.go b/d4tls/tlsdecoder.go index a61e38a..df8afcb 100644 --- a/d4tls/tlsdecoder.go +++ b/d4tls/tlsdecoder.go @@ -34,7 +34,35 @@ type sessionRecord struct { type TLSSession struct { Record sessionRecord handShakeRecord etls.ETLSHandshakeRecord - stage int + state HandshakeState +} + +// HandshakeComplete returns true if the TLS session has seen all three client helo, server helo and the certificate. +func (t *TLSSession) HandshakeComplete() bool { + return t.state.Has(StateClientHello) && + t.state.Has(StateServerHello) && + t.state.Has(StateCertificate) +} + +// HandshakePartially returns true if the client hello and server hello is set, but not the certificate. +func (t *TLSSession) HandshakePartially() bool { + return t.state.Has(StateClientHello) && + t.state.Has(StateServerHello) && + !t.state.Has(StateCertificate) +} + +// HandshakeAny returns true if any of the client or server has been seen +func (t *TLSSession) HandshakeAny() bool { + return t.state.Has(StateClientHello) || + t.state.Has(StateServerHello) +} + +func (t *TLSSession) HandshakeState() string { + return fmt.Sprintf("ClientHello:%t ServerHello:%t Certificate:%t", + t.state.Has(StateClientHello), + t.state.Has(StateServerHello), + t.state.Has(StateCertificate), + ) } // String returns a string that describes a TLSSession @@ -59,40 +87,44 @@ func (t *TLSSession) String() string { return buf.String() } +// SetNetwork sets the network part in the TLSSession.Record struct. +func (t *TLSSession) SetNetwork(cip string, sip string, cp string, sp string) { + t.Record.ClientIP = cip + t.Record.ServerIP = sip + t.Record.ClientPort = cp + t.Record.ServerPort = sp +} + +// SetTimestamp sets the timestamp of this TLSSession in its TLSSession.Record struct +func (t *TLSSession) SetTimestamp(ti time.Time) { + t.Record.Timestamp = ti +} + // PopulateClientHello takes a pointer to an etls ClientHelloMsg and writes it to the the TLSSession struct -func (t *TLSSession) PopulateClientHello(h *etls.ClientHelloMsg, cip string, sip string, cp string, sp string, ti time.Time) { - if t.stage < 1 { - t.Record.ClientIP = cip - t.Record.ServerIP = sip - t.Record.ClientPort = cp - t.Record.ServerPort = sp - t.Record.Timestamp = ti - t.handShakeRecord.ETLSHandshakeClientHello = h - t.stage = 1 - } +func (t *TLSSession) PopulateClientHello(h *etls.ClientHelloMsg) { + t.state.Set(StateClientHello) + t.handShakeRecord.ETLSHandshakeClientHello = h } // PopulateServerHello takes a pointer to an etls ServerHelloMsg and writes it to the TLSSession struct func (t *TLSSession) PopulateServerHello(h *etls.ServerHelloMsg) { - if t.stage < 2 { - t.handShakeRecord.ETLSHandshakeServerHello = h - t.stage = 2 - } + t.state.Set(StateServerHello) + t.handShakeRecord.ETLSHandshakeServerHello = h } // PopulateCertificate takes a pointer to an etls ServerHelloMsg and writes it to the TLSSession struct func (t *TLSSession) PopulateCertificate(c *etls.CertificateMsg) { - if t.stage < 3 { - t.handShakeRecord.ETLSHandshakeCertificate = c - for _, asn1Data := range t.handShakeRecord.ETLSHandshakeCertificate.Certificates { - cert, err := x509.ParseCertificate(asn1Data) - if err != nil { - //return err - } else { - h := sha256.New() - h.Write(cert.Raw) - t.Record.Certificates = append(t.Record.Certificates, certMapElm{Certificate: cert, CertHash: fmt.Sprintf("%x", h.Sum(nil))}) - } + t.state.Set(StateCertificate) + t.handShakeRecord.ETLSHandshakeCertificate = c + + for _, asn1Data := range t.handShakeRecord.ETLSHandshakeCertificate.Certificates { + cert, err := x509.ParseCertificate(asn1Data) + if err != nil { + //return err + } else { + h := sha256.New() + h.Write(cert.Raw) + t.Record.Certificates = append(t.Record.Certificates, certMapElm{Certificate: cert, CertHash: fmt.Sprintf("%x", h.Sum(nil))}) } } }