D4
/
sensor-d4-tls-fingerprinting
mirror of https://github.com/D4-project/sensor-d4-tls-fingerprinting
2
0
Fork 0
Code Issues Releases Wiki Activity

Merge pull request #8 from kjelle/master 

Allow for out-of-order TLS Handshake processing and partial outputs
master
Jean-Louis Huynen 2019-03-01 10:33:13 +01:00 committed by GitHub
parent d997cb8911 45159fdca6
commit c779f589fe
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 120 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
d4-tlsf.go
View File

@ -36,6 +36,7 @@ var ignorefsmerr = flag.Bool("ignorefsmerr", false, "Ignore TCP FSM errors")
var verbose = flag.Bool("verbose", false, "Be verbose")
var debug = flag.Bool("debug", false, "Display debug information")
var quiet = flag.Bool("quiet", false, "Be quiet regarding errors")
var partial = flag.Bool("partial", true, "Output partial TLS Sessions, e.g. where we don't see all three of client hello, server hello and certificate")
// capture
var iface = flag.String("i", "eth0", "Interface to read packets from")
@ -101,6 +102,11 @@ func (factory *tcpStreamFactory) New(net, transport gopacket.Flow, tcp *layers.T
optchecker: reassembly.NewTCPOptionCheck(),
tlsSession: d4tls.TLSSession{},
}
// Set network information in the TLSSession
cip, sip, cp, sp := getIPPorts(stream)
stream.tlsSession.SetNetwork(cip, sip, cp, sp)
return stream
}
@ -134,6 +140,7 @@ type tcpStream struct {
urls []string
ident string
tlsSession d4tls.TLSSession
queued bool
ignorefsmerr bool
nooptcheck bool
checksum bool
@ -181,6 +188,7 @@ func (t *tcpStream) ReassembledSG(sg reassembly.ScatterGather, ac reassembly.Ass
}
data := sg.Fetch(length)
if t.isTLS {
if length > 0 {
// We attempt to decode TLS
tls := &etls.ETLS{}
@ -195,13 +203,18 @@ func (t *tcpStream) ReassembledSG(sg reassembly.ScatterGather, ac reassembly.Ass
//Debug("TLS: %s\n", gopacket.LayerDump(tls))
// Debug("TLS: %s\n", gopacket.LayerGoString(tls))
if tls.Handshake != nil {
// If the timestamp has not been set, we set it for the first time.
if t.tlsSession.Record.Timestamp.IsZero() {
info := sg.CaptureInfo(0)
t.tlsSession.SetTimestamp(info.Timestamp)
}
for _, tlsrecord := range tls.Handshake {
switch tlsrecord.ETLSHandshakeMsgType {
// Client Hello
case 1:
info := sg.CaptureInfo(0)
cip, sip, cp, sp := getIPPorts(t)
t.tlsSession.PopulateClientHello(tlsrecord.ETLSHandshakeClientHello, cip, sip, cp, sp, info.Timestamp)
t.tlsSession.PopulateClientHello(tlsrecord.ETLSHandshakeClientHello)
t.tlsSession.D4Fingerprinting("ja3")
// Server Hello
case 2:
@ -210,14 +223,15 @@ func (t *tcpStream) ReassembledSG(sg reassembly.ScatterGather, ac reassembly.Ass
// Server Certificate
case 11:
t.tlsSession.PopulateCertificate(tlsrecord.ETLSHandshakeCertificate)
t.tlsSession.D4Fingerprinting("tlsh")
// If we get a cert, we consider the handshake as finished and ready to ship to D4
queueSession(t.tlsSession)
default:
break
}
}
// If the handshake is considered finished and we have not yet outputted it we ship it to output.
if t.tlsSession.HandshakeComplete() && !t.queued {
t.queueSession()
}
}
}
}
@ -235,6 +249,12 @@ func getIPPorts(t *tcpStream) (string, string, string, string) {
}
func (t *tcpStream) ReassemblyComplete(ac reassembly.AssemblerContext) bool {
// If the handshakre has not yet been outputted, but there are some information such as
// either the client hello or the server hello, we also output a partial.
if *partial && !t.queued && t.tlsSession.HandshakeAny() {
t.queueSession()
}
// remove connection from the pool
return true
}
@ -288,7 +308,7 @@ func main() {
signal.Notify(signalChan, os.Interrupt)
// Job chan to hold Completed sessions to write
jobQ = make(chan d4tls.TLSSession, 100)
jobQ = make(chan d4tls.TLSSession, 4096)
cancelC := make(chan string)
// We start a worker to send the processed TLS connection the outside world
@ -387,10 +407,13 @@ func main() {
w.Wait()
}
// Tries to enqueue or false
func queueSession(t d4tls.TLSSession) bool {
// queueSession tries to enqueue the tlsSession for output
// returns true if it succeeded or false if it failed to publish the
// tlsSession for output
func (t *tcpStream) queueSession() bool {
t.queued = true
select {
case jobQ <- t:
case jobQ <- t.tlsSession:
return true
default:
return false
@ -414,7 +437,6 @@ func processCompletedSession(cancelC <-chan string, jobQ <-chan d4tls.TLSSession
}
func output(t d4tls.TLSSession) {
jsonRecord, _ := json.MarshalIndent(t.Record, "", " ")
// If an output folder was specified for certificates

2
d4tls/fingerprinter.go
View File

@ -49,6 +49,7 @@ func (t *TLSSession) d4fg() string {
func (t *TLSSession) ja3s() bool {
var buf []byte
buf = strconv.AppendInt(buf, int64(t.handShakeRecord.ETLSHandshakeServerHello.Vers), 10)
// byte (44) is ","
buf = append(buf, byte(44))
@ -81,6 +82,7 @@ func (t *TLSSession) ja3s() bool {
func (t *TLSSession) ja3() bool {
var buf []byte
buf = strconv.AppendInt(buf, int64(t.handShakeRecord.ETLSHandshakeClientHello.Vers), 10)
// byte (44) is ","
buf = append(buf, byte(44))

9
d4tls/fingerprinter_test.go
View File

@ -203,7 +203,8 @@ func TestJA3(t *testing.T) {
for _, etlsrecord := range tls.Handshake {
if etlsrecord.ETLSHandshakeMsgType == 1 {
// Populate Client Hello related fields
tlss.PopulateClientHello(etlsrecord.ETLSHandshakeClientHello, "", "", "", "", time.Now())
tlss.PopulateClientHello(etlsrecord.ETLSHandshakeClientHello)
tlss.SetTimestamp(time.Now())
tlss.D4Fingerprinting("ja3")
t.Logf("%v", tlss.Record.JA3)
t.Logf("%v", tlss.Record.JA3Digest)
@ -279,7 +280,8 @@ func TestGreaseClientHelloExtensionExlusion(t *testing.T) {
for _, etlsrecord := range tls.Handshake {
if etlsrecord.ETLSHandshakeMsgType == 1 {
// Populate Client Hello related fields
tlss.PopulateClientHello(etlsrecord.ETLSHandshakeClientHello, "", "", "", "", time.Now())
tlss.PopulateClientHello(etlsrecord.ETLSHandshakeClientHello)
tlss.SetTimestamp(time.Now())
tlss.D4Fingerprinting("ja3")
t.Logf("%v", tlss.Record.JA3Digest)
if strings.Index(tlss.Record.JA3, "2570") != -1 {
@ -312,7 +314,8 @@ func TestGreaseClientHelloCipherExlusion(t *testing.T) {
for _, etlsrecord := range tls.Handshake {
if etlsrecord.ETLSHandshakeMsgType == 1 {
// Populate Client Hello related fields
tlss.PopulateClientHello(etlsrecord.ETLSHandshakeClientHello, "", "", "", "", time.Now())
tlss.PopulateClientHello(etlsrecord.ETLSHandshakeClientHello)
tlss.SetTimestamp(time.Now())
tlss.D4Fingerprinting("ja3")
if strings.Index(tlss.Record.JA3, "2570") != -1 {
t.Logf("GREASE values should not end up in JA3\n")

19
d4tls/flags.go Normal file
View File

@ -0,0 +1,19 @@
package d4tls
// HandshakeState is a flag which keeps record of which handeshake message types
// have been parsed.
type HandshakeState uint8
const (
StateClientHello = 1 << iota
StateServerHello
StateCertificate
)
func (s *HandshakeState) Set(flag HandshakeState) {
*s |= flag
}
func (s HandshakeState) Has(flag HandshakeState) bool {
return s&flag != 0
}

54
d4tls/tlsdecoder.go
View File

@ -34,7 +34,35 @@ type sessionRecord struct {
type TLSSession struct {
Record sessionRecord
handShakeRecord etls.ETLSHandshakeRecord
stage int
state HandshakeState
}
// HandshakeComplete returns true if the TLS session has seen all three client helo, server helo and the certificate.
func (t *TLSSession) HandshakeComplete() bool {
return t.state.Has(StateClientHello) &&
t.state.Has(StateServerHello) &&
t.state.Has(StateCertificate)
}
// HandshakePartially returns true if the client hello and server hello is set, but not the certificate.
func (t *TLSSession) HandshakePartially() bool {
return t.state.Has(StateClientHello) &&
t.state.Has(StateServerHello) &&
!t.state.Has(StateCertificate)
}
// HandshakeAny returns true if any of the client or server has been seen
func (t *TLSSession) HandshakeAny() bool {
return t.state.Has(StateClientHello) ||
t.state.Has(StateServerHello)
}
func (t *TLSSession) HandshakeState() string {
return fmt.Sprintf("ClientHello:%t ServerHello:%t Certificate:%t",
t.state.Has(StateClientHello),
t.state.Has(StateServerHello),
t.state.Has(StateCertificate),
)
}
// String returns a string that describes a TLSSession
@ -59,31 +87,36 @@ func (t *TLSSession) String() string {
return buf.String()
}
// PopulateClientHello takes a pointer to an etls ClientHelloMsg and writes it to the the TLSSession struct
func (t *TLSSession) PopulateClientHello(h *etls.ClientHelloMsg, cip string, sip string, cp string, sp string, ti time.Time) {
if t.stage < 1 {
// SetNetwork sets the network part in the TLSSession.Record struct.
func (t *TLSSession) SetNetwork(cip string, sip string, cp string, sp string) {
t.Record.ClientIP = cip
t.Record.ServerIP = sip
t.Record.ClientPort = cp
t.Record.ServerPort = sp
}
// SetTimestamp sets the timestamp of this TLSSession in its TLSSession.Record struct
func (t *TLSSession) SetTimestamp(ti time.Time) {
t.Record.Timestamp = ti
}
// PopulateClientHello takes a pointer to an etls ClientHelloMsg and writes it to the the TLSSession struct
func (t *TLSSession) PopulateClientHello(h *etls.ClientHelloMsg) {
t.state.Set(StateClientHello)
t.handShakeRecord.ETLSHandshakeClientHello = h
t.stage = 1
}
}
// PopulateServerHello takes a pointer to an etls ServerHelloMsg and writes it to the TLSSession struct
func (t *TLSSession) PopulateServerHello(h *etls.ServerHelloMsg) {
if t.stage < 2 {
t.state.Set(StateServerHello)
t.handShakeRecord.ETLSHandshakeServerHello = h
t.stage = 2
}
}
// PopulateCertificate takes a pointer to an etls ServerHelloMsg and writes it to the TLSSession struct
func (t *TLSSession) PopulateCertificate(c *etls.CertificateMsg) {
if t.stage < 3 {
t.state.Set(StateCertificate)
t.handShakeRecord.ETLSHandshakeCertificate = c
for _, asn1Data := range t.handShakeRecord.ETLSHandshakeCertificate.Certificates {
cert, err := x509.ParseCertificate(asn1Data)
if err != nil {
@ -94,5 +127,4 @@ func (t *TLSSession) PopulateCertificate(c *etls.CertificateMsg) {
t.Record.Certificates = append(t.Record.Certificates, certMapElm{Certificate: cert, CertHash: fmt.Sprintf("%x", h.Sum(nil))})
}
}
}
}