diff --git a/d4-tlsf.go b/d4-tlsf.go index e77639a..9995736 100644 --- a/d4-tlsf.go +++ b/d4-tlsf.go @@ -33,7 +33,6 @@ var nodefrag = flag.Bool("nodefrag", false, "If true, do not do IPv4 defrag") var checksum = flag.Bool("checksum", false, "Check TCP checksum") var nooptcheck = flag.Bool("nooptcheck", false, "Do not check TCP options (useful to ignore MSS on captures with TSO)") var ignorefsmerr = flag.Bool("ignorefsmerr", false, "Ignore TCP FSM errors") -var allowmissinginit = flag.Bool("allowmissinginit", false, "Support streams without SYN/SYN+ACK/ACK sequence") var verbose = flag.Bool("verbose", false, "Be verbose") var debug = flag.Bool("debug", false, "Display debug information") var quiet = flag.Bool("quiet", false, "Be quiet regarding errors") @@ -42,9 +41,6 @@ var quiet = flag.Bool("quiet", false, "Be quiet regarding errors") var iface = flag.String("i", "eth0", "Interface to read packets from") var fname = flag.String("r", "", "Filename to read from, overrides -i") -// decoding -//var LayerTypeETLS gopacket.LayerType - // writing var outCerts = flag.String("w", "", "Folder to write certificates into") var outJSON = flag.String("j", "", "Folder to write certificates into, stdin if not set") @@ -88,9 +84,8 @@ type tcpStreamFactory struct { } func (factory *tcpStreamFactory) New(net, transport gopacket.Flow, tcp *layers.TCP, ac reassembly.AssemblerContext) reassembly.Stream { - Debug("* NEW: %s %s\n", net, transport) fsmOptions := reassembly.TCPSimpleFSMOptions{ - SupportMissingEstablishment: *allowmissinginit, + SupportMissingEstablishment: true, } stream := &tcpStream{ net: net, @@ -134,37 +129,36 @@ type tcpStream struct { urls []string ident string tlsSession d4tls.TLSSession + ignorefsmerr bool + nooptcheck bool + checksum bool sync.Mutex } func (t *tcpStream) Accept(tcp *layers.TCP, ci gopacket.CaptureInfo, dir reassembly.TCPFlowDirection, nextSeq reassembly.Sequence, start *bool, ac reassembly.AssemblerContext) bool { // FSM if !t.tcpstate.CheckState(tcp, dir) { - Error("FSM", "%s: Packet rejected by FSM (state:%s)\n", t.ident, t.tcpstate.String()) if !t.fsmerr { t.fsmerr = true } - if !*ignorefsmerr { + if !t.ignorefsmerr { return false } } // Options err := t.optchecker.Accept(tcp, ci, dir, nextSeq, start) if err != nil { - Error("OptionChecker", "%s: Packet rejected by OptionChecker: %s\n", t.ident, err) - if !*nooptcheck { + if !t.nooptcheck { return false } } // Checksum accept := true - if *checksum { + if t.checksum { c, err := tcp.ComputeChecksum() if err != nil { - Error("ChecksumCompute", "%s: Got error computing checksum: %s\n", t.ident, err) accept = false } else if c != 0x0 { - Error("Checksum", "%s: Invalid checksum: 0x%x\n", t.ident, c) accept = false } } @@ -174,7 +168,7 @@ func (t *tcpStream) Accept(tcp *layers.TCP, ci gopacket.CaptureInfo, dir reassem func (t *tcpStream) ReassembledSG(sg reassembly.ScatterGather, ac reassembly.AssemblerContext) { _, _, _, skip := sg.Info() length, _ := sg.Lengths() - if skip == -1 && *allowmissinginit { + if skip == -1 { // this is allowed } else if skip != 0 { // Missing bytes in stream: do not even try to parse it @@ -236,7 +230,6 @@ func getIPPorts(t *tcpStream) (string, string, string, string) { } func (t *tcpStream) ReassemblyComplete(ac reassembly.AssemblerContext) bool { - Debug("%s: Connection closed\n", t.ident) // do not remove the connection to allow last ACK return false } @@ -272,12 +265,7 @@ func main() { } } - var dec gopacket.Decoder - var ok bool - if dec, ok = gopacket.DecodersByLayerName["Ethernet"]; !ok { - log.Fatal("No eth decoder") - } - source := gopacket.NewPacketSource(handle, dec) + source := gopacket.NewPacketSource(handle, handle.LinkType()) source.NoCopy = true Info("Starting to read packets\n") count := 0 @@ -301,53 +289,78 @@ func main() { w.Add(1) go processCompletedSession(jobQ, &w) + var eth layers.Ethernet + var ip4 layers.IPv4 + var ip6 layers.IPv6 + parser := gopacket.NewDecodingLayerParser(layers.LayerTypeEthernet, ð, &ip4, &ip6) + decoded := []gopacket.LayerType{} + for packet := range source.Packets() { count++ Debug("PACKET #%d\n", count) + data := packet.Data() + + if err := parser.DecodeLayers(data, &decoded); err != nil { + // Well it sures complaing about not knowing how to decode TCP + } + + fmt.Printf("%s\n", ip4.SrcIP) + // fmt.Printf("%s", hex.Dump(decoded)) + + for _, layerType := range decoded { + switch layerType { + case layers.LayerTypeIPv6: + fmt.Println(" IP6 ", ip6.SrcIP, ip6.DstIP) + case layers.LayerTypeIPv4: + + fmt.Println(" IP4 ", ip4.SrcIP, ip4.DstIP) + // defrag the IPv4 packet if required + if !*nodefrag { + ip4Layer := packet.Layer(layers.LayerTypeIPv4) + if ip4Layer == nil { + continue + } + ip4 := ip4Layer.(*layers.IPv4) + l := ip4.Length + newip4, err := defragger.DefragIPv4(ip4) + if err != nil { + log.Fatalln("Error while de-fragmenting", err) + } else if newip4 == nil { + Debug("Fragment...\n") + continue // ip packet fragment, we don't have whole packet yet. + } + if newip4.Length != l { + Debug("Decoding re-assembled packet: %s\n", newip4.NextLayerType()) + pb, ok := packet.(gopacket.PacketBuilder) + if !ok { + panic("Not a PacketBuilder") + } + nextDecoder := newip4.NextLayerType() + nextDecoder.Decode(newip4.Payload, pb) + } + } + + tcp := packet.Layer(layers.LayerTypeTCP) + if tcp != nil { + tcp := tcp.(*layers.TCP) + if *checksum { + err := tcp.SetNetworkLayerForChecksum(packet.NetworkLayer()) + if err != nil { + log.Fatalf("Failed to set network layer for checksum: %s\n", err) + } + } + c := Context{ + CaptureInfo: packet.Metadata().CaptureInfo, + } + assembler.AssembleWithContext(packet.NetworkLayer().NetworkFlow(), tcp, &c) + } + + } + } + bytes += int64(len(data)) - // defrag the IPv4 packet if required - if !*nodefrag { - ip4Layer := packet.Layer(layers.LayerTypeIPv4) - if ip4Layer == nil { - continue - } - ip4 := ip4Layer.(*layers.IPv4) - l := ip4.Length - newip4, err := defragger.DefragIPv4(ip4) - if err != nil { - log.Fatalln("Error while de-fragmenting", err) - } else if newip4 == nil { - Debug("Fragment...\n") - continue // ip packet fragment, we don't have whole packet yet. - } - if newip4.Length != l { - Debug("Decoding re-assembled packet: %s\n", newip4.NextLayerType()) - pb, ok := packet.(gopacket.PacketBuilder) - if !ok { - panic("Not a PacketBuilder") - } - nextDecoder := newip4.NextLayerType() - nextDecoder.Decode(newip4.Payload, pb) - } - } - - tcp := packet.Layer(layers.LayerTypeTCP) - if tcp != nil { - tcp := tcp.(*layers.TCP) - if *checksum { - err := tcp.SetNetworkLayerForChecksum(packet.NetworkLayer()) - if err != nil { - log.Fatalf("Failed to set network layer for checksum: %s\n", err) - } - } - c := Context{ - CaptureInfo: packet.Metadata().CaptureInfo, - } - assembler.AssembleWithContext(packet.NetworkLayer().NetworkFlow(), tcp, &c) - } - var done bool select { case <-signalChan: @@ -371,6 +384,16 @@ func main() { w.Wait() } +// Tries to enqueue or false +func queueSession(t d4tls.TLSSession) bool { + select { + case jobQ <- t: + return true + default: + return false + } +} + func processCompletedSession(jobQ <-chan d4tls.TLSSession, w *sync.WaitGroup) { for { tlss, more := <-jobQ @@ -383,16 +406,6 @@ func processCompletedSession(jobQ <-chan d4tls.TLSSession, w *sync.WaitGroup) { } } -// Tries to enqueue or false -func queueSession(t d4tls.TLSSession) bool { - select { - case jobQ <- t: - return true - default: - return false - } -} - func output(t d4tls.TLSSession) { jsonRecord, _ := json.MarshalIndent(t.Record, "", " ") diff --git a/d4tls/d4tls.go b/d4tls/fingerprinter.go similarity index 54% rename from d4tls/d4tls.go rename to d4tls/fingerprinter.go index dd006e2..6afacb8 100644 --- a/d4tls/d4tls.go +++ b/d4tls/fingerprinter.go @@ -1,17 +1,12 @@ package d4tls import ( - "bytes" "crypto/md5" - "crypto/sha256" - "crypto/x509" "encoding/hex" "fmt" "strconv" "strings" - "time" - "github.com/D4-project/sensor-d4-tls-fingerprinting/etls" "github.com/glaslos/tlsh" ) @@ -22,93 +17,6 @@ var grease = map[uint16]bool{ 0xcaca: true, 0xdada: true, 0xeaea: true, 0xfafa: true, } -type certMapElm struct { - CertHash string - *x509.Certificate -} - -type sessionRecord struct { - ServerIP string - ServerPort string - ClientIP string - ClientPort string - TLSH string - Timestamp time.Time - JA3 string - JA3Digest string - JA3S string - JA3SDigest string - Certificates []certMapElm -} - -// TLSSession contains a handshakeRecord that had to be filled during the handshake, -// and a Record that will be at last exported to Json -type TLSSession struct { - Record sessionRecord - handShakeRecord etls.ETLSHandshakeRecord - stage int -} - -// String returns a string that describes a TLSSession -func (t *TLSSession) String() string { - var buf bytes.Buffer - buf.WriteString(fmt.Sprintf("---------------SESSION START-------------------\n")) - buf.WriteString(fmt.Sprintf("Time: %v\n", t.Record.Timestamp)) - buf.WriteString(fmt.Sprintf("Client: %v:%v\n", t.Record.ClientIP, t.Record.ClientPort)) - buf.WriteString(fmt.Sprintf("Server: %v:%v\n", t.Record.ServerIP, t.Record.ServerPort)) - buf.WriteString(fmt.Sprintf("TLSH: %q\n", t.Record.TLSH)) - buf.WriteString(fmt.Sprintf("ja3: %q\n", t.Record.JA3)) - buf.WriteString(fmt.Sprintf("ja3 Digest: %q\n", t.Record.JA3Digest)) - buf.WriteString(fmt.Sprintf("ja3s: %q\n", t.Record.JA3S)) - buf.WriteString(fmt.Sprintf("ja3s Digest: %q\n", t.Record.JA3SDigest)) - for _, certMe := range t.Record.Certificates { - buf.WriteString(fmt.Sprintf("Certificate Issuer: %q\n", certMe.Certificate.Issuer)) - buf.WriteString(fmt.Sprintf("Certificate Subject: %q\n", certMe.Certificate.Subject)) - buf.WriteString(fmt.Sprintf("Certificate is CA: %t\n", certMe.Certificate.IsCA)) - buf.WriteString(fmt.Sprintf("Certificate SHA256: %q\n", certMe.CertHash)) - } - buf.WriteString(fmt.Sprintf("---------------SESSION END--------------------\n")) - return buf.String() -} - -// PopulateClientHello takes a pointer to an etls ClientHelloMsg and writes it to the the TLSSession struct -func (t *TLSSession) PopulateClientHello(h *etls.ClientHelloMsg, cip string, sip string, cp string, sp string, ti time.Time) { - if t.stage < 1 { - t.Record.ClientIP = cip - t.Record.ServerIP = sip - t.Record.ClientPort = cp - t.Record.ServerPort = sp - t.Record.Timestamp = ti - t.handShakeRecord.ETLSHandshakeClientHello = h - t.stage = 1 - } -} - -// PopulateServerHello takes a pointer to an etls ServerHelloMsg and writes it to the TLSSession struct -func (t *TLSSession) PopulateServerHello(h *etls.ServerHelloMsg) { - if t.stage < 2 { - t.handShakeRecord.ETLSHandshakeServerHello = h - t.stage = 2 - } -} - -// PopulateCertificate takes a pointer to an etls ServerHelloMsg and writes it to the TLSSession struct -func (t *TLSSession) PopulateCertificate(c *etls.CertificateMsg) { - if t.stage < 3 { - t.handShakeRecord.ETLSHandshakeCertificate = c - for _, asn1Data := range t.handShakeRecord.ETLSHandshakeCertificate.Certificates { - cert, err := x509.ParseCertificate(asn1Data) - if err != nil { - //return err - } else { - h := sha256.New() - h.Write(cert.Raw) - t.Record.Certificates = append(t.Record.Certificates, certMapElm{Certificate: cert, CertHash: fmt.Sprintf("%x", h.Sum(nil))}) - } - } - } -} - // D4Fingerprinting computes fingerprints doh func (t *TLSSession) D4Fingerprinting(fd string) bool { switch fd { diff --git a/d4tls/d4tls_test.go b/d4tls/fingerprinter_test.go similarity index 99% rename from d4tls/d4tls_test.go rename to d4tls/fingerprinter_test.go index c16bb52..7112285 100644 --- a/d4tls/d4tls_test.go +++ b/d4tls/fingerprinter_test.go @@ -205,7 +205,6 @@ func TestJA3(t *testing.T) { // Populate Client Hello related fields tlss.PopulateClientHello(etlsrecord.ETLSHandshakeClientHello, "", "", "", "", time.Now()) tlss.D4Fingerprinting("ja3") - // TODO check that againt the reference python implementation t.Logf("%v", tlss.Record.JA3) t.Logf("%v", tlss.Record.JA3Digest) } @@ -229,7 +228,6 @@ func TestJA3s(t *testing.T) { // Populate Server Hello related fields tlss.PopulateServerHello(etlsrecord.ETLSHandshakeServerHello) tlss.D4Fingerprinting("ja3s") - // TODO check that againt the reference python implementation t.Logf("%v", tlss.Record.JA3S) t.Logf("%v", tlss.Record.JA3SDigest) } @@ -253,7 +251,7 @@ func TestTLSH(t *testing.T) { // Populate Cert tlss.PopulateCertificate(etlsrecord.ETLSHandshakeCertificate) tlss.D4Fingerprinting("tlsh") - // TODO check that againt the reference implementation + // TODO check that against the reference implementation t.Logf("%v", tlss.Record.TLSH) } } diff --git a/d4tls/tlsdecoder.go b/d4tls/tlsdecoder.go new file mode 100644 index 0000000..a61e38a --- /dev/null +++ b/d4tls/tlsdecoder.go @@ -0,0 +1,98 @@ +package d4tls + +import ( + "bytes" + "crypto/sha256" + "crypto/x509" + "fmt" + "time" + + "github.com/D4-project/sensor-d4-tls-fingerprinting/etls" +) + +type certMapElm struct { + CertHash string + *x509.Certificate +} + +type sessionRecord struct { + ServerIP string + ServerPort string + ClientIP string + ClientPort string + TLSH string + Timestamp time.Time + JA3 string + JA3Digest string + JA3S string + JA3SDigest string + Certificates []certMapElm +} + +// TLSSession contains a handshakeRecord that had to be filled during the handshake, +// and a Record that will be at last exported to Json +type TLSSession struct { + Record sessionRecord + handShakeRecord etls.ETLSHandshakeRecord + stage int +} + +// String returns a string that describes a TLSSession +func (t *TLSSession) String() string { + var buf bytes.Buffer + buf.WriteString(fmt.Sprintf("---------------SESSION START-------------------\n")) + buf.WriteString(fmt.Sprintf("Time: %v\n", t.Record.Timestamp)) + buf.WriteString(fmt.Sprintf("Client: %v:%v\n", t.Record.ClientIP, t.Record.ClientPort)) + buf.WriteString(fmt.Sprintf("Server: %v:%v\n", t.Record.ServerIP, t.Record.ServerPort)) + buf.WriteString(fmt.Sprintf("TLSH: %q\n", t.Record.TLSH)) + buf.WriteString(fmt.Sprintf("ja3: %q\n", t.Record.JA3)) + buf.WriteString(fmt.Sprintf("ja3 Digest: %q\n", t.Record.JA3Digest)) + buf.WriteString(fmt.Sprintf("ja3s: %q\n", t.Record.JA3S)) + buf.WriteString(fmt.Sprintf("ja3s Digest: %q\n", t.Record.JA3SDigest)) + for _, certMe := range t.Record.Certificates { + buf.WriteString(fmt.Sprintf("Certificate Issuer: %q\n", certMe.Certificate.Issuer)) + buf.WriteString(fmt.Sprintf("Certificate Subject: %q\n", certMe.Certificate.Subject)) + buf.WriteString(fmt.Sprintf("Certificate is CA: %t\n", certMe.Certificate.IsCA)) + buf.WriteString(fmt.Sprintf("Certificate SHA256: %q\n", certMe.CertHash)) + } + buf.WriteString(fmt.Sprintf("---------------SESSION END--------------------\n")) + return buf.String() +} + +// PopulateClientHello takes a pointer to an etls ClientHelloMsg and writes it to the the TLSSession struct +func (t *TLSSession) PopulateClientHello(h *etls.ClientHelloMsg, cip string, sip string, cp string, sp string, ti time.Time) { + if t.stage < 1 { + t.Record.ClientIP = cip + t.Record.ServerIP = sip + t.Record.ClientPort = cp + t.Record.ServerPort = sp + t.Record.Timestamp = ti + t.handShakeRecord.ETLSHandshakeClientHello = h + t.stage = 1 + } +} + +// PopulateServerHello takes a pointer to an etls ServerHelloMsg and writes it to the TLSSession struct +func (t *TLSSession) PopulateServerHello(h *etls.ServerHelloMsg) { + if t.stage < 2 { + t.handShakeRecord.ETLSHandshakeServerHello = h + t.stage = 2 + } +} + +// PopulateCertificate takes a pointer to an etls ServerHelloMsg and writes it to the TLSSession struct +func (t *TLSSession) PopulateCertificate(c *etls.CertificateMsg) { + if t.stage < 3 { + t.handShakeRecord.ETLSHandshakeCertificate = c + for _, asn1Data := range t.handShakeRecord.ETLSHandshakeCertificate.Certificates { + cert, err := x509.ParseCertificate(asn1Data) + if err != nil { + //return err + } else { + h := sha256.New() + h.Write(cert.Raw) + t.Record.Certificates = append(t.Record.Certificates, certMapElm{Certificate: cert, CertHash: fmt.Sprintf("%x", h.Sum(nil))}) + } + } + } +}