D4
/
sensor-d4-tls-fingerprinting
mirror of https://github.com/D4-project/sensor-d4-tls-fingerprinting
2
0
Fork 0
Code Issues Releases Wiki Activity

Merge pull request #5 from D4-project/DecodingLayerParser 

Decoding layer parser
pull/8/head
Jean-Louis Huynen 2019-02-15 12:53:57 +01:00 committed by GitHub
parent 32a75bfa95 003bb39786
commit 7f13807d98
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 184 additions and 167 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

157
d4-tlsf.go
View File

@ -33,7 +33,6 @@ var nodefrag = flag.Bool("nodefrag", false, "If true, do not do IPv4 defrag")
var checksum = flag.Bool("checksum", false, "Check TCP checksum")
var nooptcheck = flag.Bool("nooptcheck", false, "Do not check TCP options (useful to ignore MSS on captures with TSO)")
var ignorefsmerr = flag.Bool("ignorefsmerr", false, "Ignore TCP FSM errors")
var allowmissinginit = flag.Bool("allowmissinginit", false, "Support streams without SYN/SYN+ACK/ACK sequence")
var verbose = flag.Bool("verbose", false, "Be verbose")
var debug = flag.Bool("debug", false, "Display debug information")
var quiet = flag.Bool("quiet", false, "Be quiet regarding errors")
@ -42,9 +41,6 @@ var quiet = flag.Bool("quiet", false, "Be quiet regarding errors")
var iface = flag.String("i", "eth0", "Interface to read packets from")
var fname = flag.String("r", "", "Filename to read from, overrides -i")
// decoding
//var LayerTypeETLS gopacket.LayerType
// writing
var outCerts = flag.String("w", "", "Folder to write certificates into")
var outJSON = flag.String("j", "", "Folder to write certificates into, stdin if not set")
@ -88,9 +84,8 @@ type tcpStreamFactory struct {
}
func (factory *tcpStreamFactory) New(net, transport gopacket.Flow, tcp *layers.TCP, ac reassembly.AssemblerContext) reassembly.Stream {
Debug("* NEW: %s %s\n", net, transport)
fsmOptions := reassembly.TCPSimpleFSMOptions{
SupportMissingEstablishment: *allowmissinginit,
SupportMissingEstablishment: true,
}
stream := &tcpStream{
net: net,
@ -134,37 +129,36 @@ type tcpStream struct {
urls []string
ident string
tlsSession d4tls.TLSSession
ignorefsmerr bool
nooptcheck bool
checksum bool
sync.Mutex
}
func (t *tcpStream) Accept(tcp *layers.TCP, ci gopacket.CaptureInfo, dir reassembly.TCPFlowDirection, nextSeq reassembly.Sequence, start *bool, ac reassembly.AssemblerContext) bool {
// FSM
if !t.tcpstate.CheckState(tcp, dir) {
Error("FSM", "%s: Packet rejected by FSM (state:%s)\n", t.ident, t.tcpstate.String())
if !t.fsmerr {
t.fsmerr = true
}
if !*ignorefsmerr {
if !t.ignorefsmerr {
return false
}
}
// Options
err := t.optchecker.Accept(tcp, ci, dir, nextSeq, start)
if err != nil {
Error("OptionChecker", "%s: Packet rejected by OptionChecker: %s\n", t.ident, err)
if !*nooptcheck {
if !t.nooptcheck {
return false
}
}
// Checksum
accept := true
if *checksum {
if t.checksum {
c, err := tcp.ComputeChecksum()
if err != nil {
Error("ChecksumCompute", "%s: Got error computing checksum: %s\n", t.ident, err)
accept = false
} else if c != 0x0 {
Error("Checksum", "%s: Invalid checksum: 0x%x\n", t.ident, c)
accept = false
}
}
@ -174,7 +168,7 @@ func (t *tcpStream) Accept(tcp *layers.TCP, ci gopacket.CaptureInfo, dir reassem
func (t *tcpStream) ReassembledSG(sg reassembly.ScatterGather, ac reassembly.AssemblerContext) {
_, _, _, skip := sg.Info()
length, _ := sg.Lengths()
if skip == -1 && *allowmissinginit {
if skip == -1 {
// this is allowed
} else if skip != 0 {
// Missing bytes in stream: do not even try to parse it
@ -236,7 +230,6 @@ func getIPPorts(t *tcpStream) (string, string, string, string) {
}
func (t *tcpStream) ReassemblyComplete(ac reassembly.AssemblerContext) bool {
Debug("%s: Connection closed\n", t.ident)
// do not remove the connection to allow last ACK
return false
}
@ -272,12 +265,7 @@ func main() {
}
}
var dec gopacket.Decoder
var ok bool
if dec, ok = gopacket.DecodersByLayerName["Ethernet"]; !ok {
log.Fatal("No eth decoder")
}
source := gopacket.NewPacketSource(handle, dec)
source := gopacket.NewPacketSource(handle, handle.LinkType())
source.NoCopy = true
Info("Starting to read packets\n")
count := 0
@ -301,53 +289,78 @@ func main() {
w.Add(1)
go processCompletedSession(jobQ, &w)
var eth layers.Ethernet
var ip4 layers.IPv4
var ip6 layers.IPv6
parser := gopacket.NewDecodingLayerParser(layers.LayerTypeEthernet, &eth, &ip4, &ip6)
decoded := []gopacket.LayerType{}
for packet := range source.Packets() {
count++
Debug("PACKET #%d\n", count)
data := packet.Data()
if err := parser.DecodeLayers(data, &decoded); err != nil {
// Well it sures complaing about not knowing how to decode TCP
}
fmt.Printf("%s\n", ip4.SrcIP)
// fmt.Printf("%s", hex.Dump(decoded))
for _, layerType := range decoded {
switch layerType {
case layers.LayerTypeIPv6:
fmt.Println(" IP6 ", ip6.SrcIP, ip6.DstIP)
case layers.LayerTypeIPv4:
fmt.Println(" IP4 ", ip4.SrcIP, ip4.DstIP)
// defrag the IPv4 packet if required
if !*nodefrag {
ip4Layer := packet.Layer(layers.LayerTypeIPv4)
if ip4Layer == nil {
continue
}
ip4 := ip4Layer.(*layers.IPv4)
l := ip4.Length
newip4, err := defragger.DefragIPv4(ip4)
if err != nil {
log.Fatalln("Error while de-fragmenting", err)
} else if newip4 == nil {
Debug("Fragment...\n")
continue // ip packet fragment, we don't have whole packet yet.
}
if newip4.Length != l {
Debug("Decoding re-assembled packet: %s\n", newip4.NextLayerType())
pb, ok := packet.(gopacket.PacketBuilder)
if !ok {
panic("Not a PacketBuilder")
}
nextDecoder := newip4.NextLayerType()
nextDecoder.Decode(newip4.Payload, pb)
}
}
tcp := packet.Layer(layers.LayerTypeTCP)
if tcp != nil {
tcp := tcp.(*layers.TCP)
if *checksum {
err := tcp.SetNetworkLayerForChecksum(packet.NetworkLayer())
if err != nil {
log.Fatalf("Failed to set network layer for checksum: %s\n", err)
}
}
c := Context{
CaptureInfo: packet.Metadata().CaptureInfo,
}
assembler.AssembleWithContext(packet.NetworkLayer().NetworkFlow(), tcp, &c)
}
}
}
bytes += int64(len(data))
// defrag the IPv4 packet if required
if !*nodefrag {
ip4Layer := packet.Layer(layers.LayerTypeIPv4)
if ip4Layer == nil {
continue
}
ip4 := ip4Layer.(*layers.IPv4)
l := ip4.Length
newip4, err := defragger.DefragIPv4(ip4)
if err != nil {
log.Fatalln("Error while de-fragmenting", err)
} else if newip4 == nil {
Debug("Fragment...\n")
continue // ip packet fragment, we don't have whole packet yet.
}
if newip4.Length != l {
Debug("Decoding re-assembled packet: %s\n", newip4.NextLayerType())
pb, ok := packet.(gopacket.PacketBuilder)
if !ok {
panic("Not a PacketBuilder")
}
nextDecoder := newip4.NextLayerType()
nextDecoder.Decode(newip4.Payload, pb)
}
}
tcp := packet.Layer(layers.LayerTypeTCP)
if tcp != nil {
tcp := tcp.(*layers.TCP)
if *checksum {
err := tcp.SetNetworkLayerForChecksum(packet.NetworkLayer())
if err != nil {
log.Fatalf("Failed to set network layer for checksum: %s\n", err)
}
}
c := Context{
CaptureInfo: packet.Metadata().CaptureInfo,
}
assembler.AssembleWithContext(packet.NetworkLayer().NetworkFlow(), tcp, &c)
}
var done bool
select {
case <-signalChan:
@ -371,6 +384,16 @@ func main() {
w.Wait()
}
// Tries to enqueue or false
func queueSession(t d4tls.TLSSession) bool {
select {
case jobQ <- t:
return true
default:
return false
}
}
func processCompletedSession(jobQ <-chan d4tls.TLSSession, w *sync.WaitGroup) {
for {
tlss, more := <-jobQ
@ -383,16 +406,6 @@ func processCompletedSession(jobQ <-chan d4tls.TLSSession, w *sync.WaitGroup) {
}
}
// Tries to enqueue or false
func queueSession(t d4tls.TLSSession) bool {
select {
case jobQ <- t:
return true
default:
return false
}
}
func output(t d4tls.TLSSession) {
jsonRecord, _ := json.MarshalIndent(t.Record, "", " ")

92
d4tls/d4tls.go → d4tls/fingerprinter.go
View File

@ -1,17 +1,12 @@
package d4tls
import (
"bytes"
"crypto/md5"
"crypto/sha256"
"crypto/x509"
"encoding/hex"
"fmt"
"strconv"
"strings"
"time"
"github.com/D4-project/sensor-d4-tls-fingerprinting/etls"
"github.com/glaslos/tlsh"
)
@ -22,93 +17,6 @@ var grease = map[uint16]bool{
0xcaca: true, 0xdada: true, 0xeaea: true, 0xfafa: true,
}
type certMapElm struct {
CertHash string
*x509.Certificate
}
type sessionRecord struct {
ServerIP string
ServerPort string
ClientIP string
ClientPort string
TLSH string
Timestamp time.Time
JA3 string
JA3Digest string
JA3S string
JA3SDigest string
Certificates []certMapElm
}
// TLSSession contains a handshakeRecord that had to be filled during the handshake,
// and a Record that will be at last exported to Json
type TLSSession struct {
Record sessionRecord
handShakeRecord etls.ETLSHandshakeRecord
stage int
}
// String returns a string that describes a TLSSession
func (t *TLSSession) String() string {
var buf bytes.Buffer
buf.WriteString(fmt.Sprintf("---------------SESSION START-------------------\n"))
buf.WriteString(fmt.Sprintf("Time: %v\n", t.Record.Timestamp))
buf.WriteString(fmt.Sprintf("Client: %v:%v\n", t.Record.ClientIP, t.Record.ClientPort))
buf.WriteString(fmt.Sprintf("Server: %v:%v\n", t.Record.ServerIP, t.Record.ServerPort))
buf.WriteString(fmt.Sprintf("TLSH: %q\n", t.Record.TLSH))
buf.WriteString(fmt.Sprintf("ja3: %q\n", t.Record.JA3))
buf.WriteString(fmt.Sprintf("ja3 Digest: %q\n", t.Record.JA3Digest))
buf.WriteString(fmt.Sprintf("ja3s: %q\n", t.Record.JA3S))
buf.WriteString(fmt.Sprintf("ja3s Digest: %q\n", t.Record.JA3SDigest))
for _, certMe := range t.Record.Certificates {
buf.WriteString(fmt.Sprintf("Certificate Issuer: %q\n", certMe.Certificate.Issuer))
buf.WriteString(fmt.Sprintf("Certificate Subject: %q\n", certMe.Certificate.Subject))
buf.WriteString(fmt.Sprintf("Certificate is CA: %t\n", certMe.Certificate.IsCA))
buf.WriteString(fmt.Sprintf("Certificate SHA256: %q\n", certMe.CertHash))
}
buf.WriteString(fmt.Sprintf("---------------SESSION END--------------------\n"))
return buf.String()
}
// PopulateClientHello takes a pointer to an etls ClientHelloMsg and writes it to the the TLSSession struct
func (t *TLSSession) PopulateClientHello(h *etls.ClientHelloMsg, cip string, sip string, cp string, sp string, ti time.Time) {
if t.stage < 1 {
t.Record.ClientIP = cip
t.Record.ServerIP = sip
t.Record.ClientPort = cp
t.Record.ServerPort = sp
t.Record.Timestamp = ti
t.handShakeRecord.ETLSHandshakeClientHello = h
t.stage = 1
}
}
// PopulateServerHello takes a pointer to an etls ServerHelloMsg and writes it to the TLSSession struct
func (t *TLSSession) PopulateServerHello(h *etls.ServerHelloMsg) {
if t.stage < 2 {
t.handShakeRecord.ETLSHandshakeServerHello = h
t.stage = 2
}
}
// PopulateCertificate takes a pointer to an etls ServerHelloMsg and writes it to the TLSSession struct
func (t *TLSSession) PopulateCertificate(c *etls.CertificateMsg) {
if t.stage < 3 {
t.handShakeRecord.ETLSHandshakeCertificate = c
for _, asn1Data := range t.handShakeRecord.ETLSHandshakeCertificate.Certificates {
cert, err := x509.ParseCertificate(asn1Data)
if err != nil {
//return err
} else {
h := sha256.New()
h.Write(cert.Raw)
t.Record.Certificates = append(t.Record.Certificates, certMapElm{Certificate: cert, CertHash: fmt.Sprintf("%x", h.Sum(nil))})
}
}
}
}
// D4Fingerprinting computes fingerprints doh
func (t *TLSSession) D4Fingerprinting(fd string) bool {
switch fd {

4
d4tls/d4tls_test.go → d4tls/fingerprinter_test.go
View File

@ -205,7 +205,6 @@ func TestJA3(t *testing.T) {
// Populate Client Hello related fields
tlss.PopulateClientHello(etlsrecord.ETLSHandshakeClientHello, "", "", "", "", time.Now())
tlss.D4Fingerprinting("ja3")
// TODO check that againt the reference python implementation
t.Logf("%v", tlss.Record.JA3)
t.Logf("%v", tlss.Record.JA3Digest)
}
@ -229,7 +228,6 @@ func TestJA3s(t *testing.T) {
// Populate Server Hello related fields
tlss.PopulateServerHello(etlsrecord.ETLSHandshakeServerHello)
tlss.D4Fingerprinting("ja3s")
// TODO check that againt the reference python implementation
t.Logf("%v", tlss.Record.JA3S)
t.Logf("%v", tlss.Record.JA3SDigest)
}
@ -253,7 +251,7 @@ func TestTLSH(t *testing.T) {
// Populate Cert
tlss.PopulateCertificate(etlsrecord.ETLSHandshakeCertificate)
tlss.D4Fingerprinting("tlsh")
// TODO check that againt the reference implementation
// TODO check that against the reference implementation
t.Logf("%v", tlss.Record.TLSH)
}
}

98
d4tls/tlsdecoder.go Normal file
View File

@ -0,0 +1,98 @@
package d4tls
import (
"bytes"
"crypto/sha256"
"crypto/x509"
"fmt"
"time"
"github.com/D4-project/sensor-d4-tls-fingerprinting/etls"
)
type certMapElm struct {
CertHash string
*x509.Certificate
}
type sessionRecord struct {
ServerIP string
ServerPort string
ClientIP string
ClientPort string
TLSH string
Timestamp time.Time
JA3 string
JA3Digest string
JA3S string
JA3SDigest string
Certificates []certMapElm
}
// TLSSession contains a handshakeRecord that had to be filled during the handshake,
// and a Record that will be at last exported to Json
type TLSSession struct {
Record sessionRecord
handShakeRecord etls.ETLSHandshakeRecord
stage int
}
// String returns a string that describes a TLSSession
func (t *TLSSession) String() string {
var buf bytes.Buffer
buf.WriteString(fmt.Sprintf("---------------SESSION START-------------------\n"))
buf.WriteString(fmt.Sprintf("Time: %v\n", t.Record.Timestamp))
buf.WriteString(fmt.Sprintf("Client: %v:%v\n", t.Record.ClientIP, t.Record.ClientPort))
buf.WriteString(fmt.Sprintf("Server: %v:%v\n", t.Record.ServerIP, t.Record.ServerPort))
buf.WriteString(fmt.Sprintf("TLSH: %q\n", t.Record.TLSH))
buf.WriteString(fmt.Sprintf("ja3: %q\n", t.Record.JA3))
buf.WriteString(fmt.Sprintf("ja3 Digest: %q\n", t.Record.JA3Digest))
buf.WriteString(fmt.Sprintf("ja3s: %q\n", t.Record.JA3S))
buf.WriteString(fmt.Sprintf("ja3s Digest: %q\n", t.Record.JA3SDigest))
for _, certMe := range t.Record.Certificates {
buf.WriteString(fmt.Sprintf("Certificate Issuer: %q\n", certMe.Certificate.Issuer))
buf.WriteString(fmt.Sprintf("Certificate Subject: %q\n", certMe.Certificate.Subject))
buf.WriteString(fmt.Sprintf("Certificate is CA: %t\n", certMe.Certificate.IsCA))
buf.WriteString(fmt.Sprintf("Certificate SHA256: %q\n", certMe.CertHash))
}
buf.WriteString(fmt.Sprintf("---------------SESSION END--------------------\n"))
return buf.String()
}
// PopulateClientHello takes a pointer to an etls ClientHelloMsg and writes it to the the TLSSession struct
func (t *TLSSession) PopulateClientHello(h *etls.ClientHelloMsg, cip string, sip string, cp string, sp string, ti time.Time) {
if t.stage < 1 {
t.Record.ClientIP = cip
t.Record.ServerIP = sip
t.Record.ClientPort = cp
t.Record.ServerPort = sp
t.Record.Timestamp = ti
t.handShakeRecord.ETLSHandshakeClientHello = h
t.stage = 1
}
}
// PopulateServerHello takes a pointer to an etls ServerHelloMsg and writes it to the TLSSession struct
func (t *TLSSession) PopulateServerHello(h *etls.ServerHelloMsg) {
if t.stage < 2 {
t.handShakeRecord.ETLSHandshakeServerHello = h
t.stage = 2
}
}
// PopulateCertificate takes a pointer to an etls ServerHelloMsg and writes it to the TLSSession struct
func (t *TLSSession) PopulateCertificate(c *etls.CertificateMsg) {
if t.stage < 3 {
t.handShakeRecord.ETLSHandshakeCertificate = c
for _, asn1Data := range t.handShakeRecord.ETLSHandshakeCertificate.Certificates {
cert, err := x509.ParseCertificate(asn1Data)
if err != nil {
//return err
} else {
h := sha256.New()
h.Write(cert.Raw)
t.Record.Certificates = append(t.Record.Certificates, certMapElm{Certificate: cert, CertHash: fmt.Sprintf("%x", h.Sum(nil))})
}
}
}
}