From 1212cf511c0ff002a26a04271f897946f85e2428 Mon Sep 17 00:00:00 2001 From: Jean-Louis Huynen Date: Tue, 29 Jan 2019 17:10:50 +0100 Subject: [PATCH] wip - updates README, cleans up a bit src --- README.md | 25 ++++++++++++++--- d4-tlsf.go | 79 ++---------------------------------------------------- 2 files changed, 24 insertions(+), 80 deletions(-) diff --git a/README.md b/README.md index 292e806..57accb1 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # sensor-d4-tls-fingerprinting -Extracts TLS certificates from pcap files or network interfaces, fingerprints TLS client/server interactions with ja3/ja3s. +Extracts TLS certificates from pcap files or network interfaces (tcpreassembly is done thanks to gopacket), fingerprints TLS client/server interactions with ja3/ja3s and print output in JSON form. # Use This project is currently in its very early stage and relies mainly on a customized version of ![gopacket](http://github.com/google/gopacket "gopacket link") that will be the subject of a pull request later on. ## Install dependencies & go get @@ -10,6 +10,25 @@ $cd $GOPATH/src/github.com/google/gopacket $git remote add fork github.com/gallypette/gopacket $go get github.com/D4-project/sensor-d4-tls-fingerprinting ``` +make allows to compile for amd64 and arm ATM. ## How to use -This early version takes a pcap file in input with the "-r" flag, and outputs the valid x509 certificates it found in current folder. -It speaks networks too with "-i". + +Read from pcap: +``` shell +$ ./d4-tlsf-amd64l -r=file + +``` +Read from interface (promiscious mode): +``` shell +$ ./d4-tlsf-amd64l -i=interface + +``` +Write x509 certificates to folder: +``` shell +$ ./d4-tlsf-amd64l -w=folderName +``` +Write output json inside folder + +``` shell +$ ./d4-tlsf-amd64l -j=folderName +``` diff --git a/d4-tlsf.go b/d4-tlsf.go index fdbb3e0..b519ed4 100644 --- a/d4-tlsf.go +++ b/d4-tlsf.go @@ -1,8 +1,3 @@ -// Copyright 2012 Google, Inc. All rights reserved. -// -// Use of this source code is governed by a BSD-style license -// that can be found in the LICENSE file in the root of the source -// tree. package main import ( @@ -36,10 +31,6 @@ import ( "github.com/google/gopacket/reassembly" ) -var maxcount = flag.Int("c", -1, "Only grab this many packets, then exit") -var decoder = flag.String("decoder", "", "Name of the decoder to use (default: guess from capture)") -var statsevery = flag.Int("stats", 1000, "Output statistics every N packets") -var lazy = flag.Bool("lazy", false, "If true, do lazy decoding") var nodefrag = flag.Bool("nodefrag", false, "If true, do not do IPv4 defrag") var checksum = flag.Bool("checksum", false, "Check TCP checksum") var nooptcheck = flag.Bool("nooptcheck", false, "Do not check TCP options (useful to ignore MSS on captures with TSO)") @@ -58,26 +49,6 @@ var outCerts = flag.String("w", "", "Folder to write certificates into") var outJSON = flag.String("j", "", "Folder to write certificates into, stdin if not set") var jobQ chan TLSSession -var memprofile = flag.String("memprofile", "", "Write memory profile") - -var stats struct { - ipdefrag int - missedBytes int - pkt int - sz int - totalsz int - rejectFsm int - rejectOpt int - rejectConnFsm int - reassembled int - outOfOrderBytes int - outOfOrderPackets int - biggestChunkBytes int - biggestChunkPackets int - overlapBytes int - overlapPackets int -} - type SessionRecord struct { ServerIP string ServerPort string @@ -213,10 +184,8 @@ func (t *tcpStream) Accept(tcp *layers.TCP, ci gopacket.CaptureInfo, dir reassem // FSM if !t.tcpstate.CheckState(tcp, dir) { Error("FSM", "%s: Packet rejected by FSM (state:%s)\n", t.ident, t.tcpstate.String()) - stats.rejectFsm++ if !t.fsmerr { t.fsmerr = true - stats.rejectConnFsm++ } if !*ignorefsmerr { return false @@ -226,7 +195,6 @@ func (t *tcpStream) Accept(tcp *layers.TCP, ci gopacket.CaptureInfo, dir reassem err := t.optchecker.Accept(tcp, ci, dir, nextSeq, start) if err != nil { Error("OptionChecker", "%s: Packet rejected by OptionChecker: %s\n", t.ident, err) - stats.rejectOpt++ if !*nooptcheck { return false } @@ -243,47 +211,12 @@ func (t *tcpStream) Accept(tcp *layers.TCP, ci gopacket.CaptureInfo, dir reassem accept = false } } - if !accept { - stats.rejectOpt++ - } return accept } func (t *tcpStream) ReassembledSG(sg reassembly.ScatterGather, ac reassembly.AssemblerContext) { - dir, start, end, skip := sg.Info() - length, saved := sg.Lengths() - // update stats - sgStats := sg.Stats() - if skip > 0 { - stats.missedBytes += skip - } - stats.sz += length - saved - stats.pkt += sgStats.Packets - if sgStats.Chunks > 1 { - stats.reassembled++ - } - stats.outOfOrderPackets += sgStats.QueuedPackets - stats.outOfOrderBytes += sgStats.QueuedBytes - if length > stats.biggestChunkBytes { - stats.biggestChunkBytes = length - } - if sgStats.Packets > stats.biggestChunkPackets { - stats.biggestChunkPackets = sgStats.Packets - } - if sgStats.OverlapBytes != 0 && sgStats.OverlapPackets == 0 { - fmt.Printf("bytes:%d, pkts:%d\n", sgStats.OverlapBytes, sgStats.OverlapPackets) - panic("Invalid overlap") - } - stats.overlapBytes += sgStats.OverlapBytes - stats.overlapPackets += sgStats.OverlapPackets - - var ident string - if dir == reassembly.TCPDirClientToServer { - ident = fmt.Sprintf("%v %v(%s): ", t.net, t.transport, dir) - } else { - ident = fmt.Sprintf("%v %v(%s): ", t.net.Reverse(), t.transport.Reverse(), dir) - } - Debug("%s: SG reassembled packet with %d bytes (start:%v,end:%v,skip:%d,saved:%d,nb:%d,%d,overlap:%d,%d)\n", ident, length, start, end, skip, saved, sgStats.Packets, sgStats.Chunks, sgStats.OverlapBytes, sgStats.OverlapPackets) + _, _, _, skip := sg.Info() + length, _ := sg.Lengths() if skip == -1 && *allowmissinginit { // this is allowed } else if skip != 0 { @@ -488,7 +421,6 @@ func main() { log.Fatal("No eth decoder") } source := gopacket.NewPacketSource(handle, dec) - source.Lazy = *lazy source.NoCopy = true Info("Starting to read packets\n") count := 0 @@ -534,7 +466,6 @@ func main() { continue // ip packet fragment, we don't have whole packet yet. } if newip4.Length != l { - stats.ipdefrag++ Debug("Decoding re-assembled packet: %s\n", newip4.NextLayerType()) pb, ok := packet.(gopacket.PacketBuilder) if !ok { @@ -557,14 +488,8 @@ func main() { c := Context{ CaptureInfo: packet.Metadata().CaptureInfo, } - stats.totalsz += len(tcp.Payload) assembler.AssembleWithContext(packet.NetworkLayer().NetworkFlow(), tcp, &c) } - if count%*statsevery == 0 { - ref := packet.Metadata().CaptureInfo.Timestamp - flushed, closed := assembler.FlushWithOptions(reassembly.FlushOptions{T: ref.Add(-timeout), TC: ref.Add(-closeTimeout)}) - Debug("Forced flush: %d flushed, %d closed (%s)", flushed, closed, ref) - } var done bool select {