initial PoC
Jean-Louis Huynen
parent
6e65c9c781
commit
7ca0c05606
|
|
@ -4,9 +4,14 @@
|
|||
*.dll
|
||||
*.so
|
||||
*.dylib
|
||||
sensor-d4-tls-fingerprinting
|
||||
|
||||
# Test binary, build with `go test -c`
|
||||
*.test
|
||||
|
||||
# Output of the go coverage tool, specifically when used with LiteIDE
|
||||
*.out
|
||||
|
||||
# Output certificate files & input pcaps
|
||||
*.crt
|
||||
*.pcap
|
||||
|
|
|
|||
15
README.md
15
README.md
|
|
@ -1,2 +1,15 @@
|
|||
# sensor-d4-tls-fingerprinting
|
||||
Extract TLS certificates from pcap files or network interfaces, fingerprint TLS client/server interactions with ja3/ja3s
|
||||
Extracts TLS certificates from pcap files or network interfaces, fingerprints TLS client/server interactions with ja3/ja3s.
|
||||
# Use
|
||||
This project is currently in its very early stage and relies mainly on a customized version of  that will be the subject of a pull request later on.
|
||||
## Install dependencies & go get
|
||||
``` shell
|
||||
$go get github.com/gallypette/gopacket
|
||||
$go get github.com/google/gopacket
|
||||
$cd $GOPATH/src/github.com/google/gopacket
|
||||
$git remote add fork github.com/gallypette/gopacket
|
||||
$go get github.com/D4-project/sensor-d4-tls-fingerprinting
|
||||
```
|
||||
## How to use
|
||||
This early version takes a pcap file in input with the "-r" flag, and outputs the valid x509 certificates it found in current folder.
|
||||
It speaks networks too with "-i".
|
||||
|
|
|
|||
|
|
@ -0,0 +1,406 @@
|
|||
// Copyright 2012 Google, Inc. All rights reserved.
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license
|
||||
// that can be found in the LICENSE file in the root of the source
|
||||
// tree.
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/x509"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/google/gopacket"
|
||||
"github.com/google/gopacket/examples/util"
|
||||
"github.com/google/gopacket/ip4defrag"
|
||||
|
||||
// ATM add a fork to gopacket like this to pull my TLS code:
|
||||
// $go get github.com/gallypette/gopacket
|
||||
// $cd $GOPATH/src/github.com/google/gopacket
|
||||
// $git remote add fork github.com/gallypette/gopacket
|
||||
"github.com/google/gopacket/layers"
|
||||
"github.com/google/gopacket/pcap"
|
||||
"github.com/google/gopacket/reassembly"
|
||||
)
|
||||
|
||||
var maxcount = flag.Int("c", -1, "Only grab this many packets, then exit")
|
||||
var decoder = flag.String("decoder", "", "Name of the decoder to use (default: guess from capture)")
|
||||
var statsevery = flag.Int("stats", 1000, "Output statistics every N packets")
|
||||
var lazy = flag.Bool("lazy", false, "If true, do lazy decoding")
|
||||
var nodefrag = flag.Bool("nodefrag", false, "If true, do not do IPv4 defrag")
|
||||
var checksum = flag.Bool("checksum", false, "Check TCP checksum")
|
||||
var nooptcheck = flag.Bool("nooptcheck", false, "Do not check TCP options (useful to ignore MSS on captures with TSO)")
|
||||
var ignorefsmerr = flag.Bool("ignorefsmerr", false, "Ignore TCP FSM errors")
|
||||
var allowmissinginit = flag.Bool("allowmissinginit", false, "Support streams without SYN/SYN+ACK/ACK sequence")
|
||||
var verbose = flag.Bool("verbose", false, "Be verbose")
|
||||
var debug = flag.Bool("debug", false, "Display debug information")
|
||||
var quiet = flag.Bool("quiet", false, "Be quiet regarding errors")
|
||||
|
||||
// capture
|
||||
var iface = flag.String("i", "eth0", "Interface to read packets from")
|
||||
var fname = flag.String("r", "", "Filename to read from, overrides -i")
|
||||
|
||||
var memprofile = flag.String("memprofile", "", "Write memory profile")
|
||||
|
||||
var stats struct {
|
||||
ipdefrag int
|
||||
missedBytes int
|
||||
pkt int
|
||||
sz int
|
||||
totalsz int
|
||||
rejectFsm int
|
||||
rejectOpt int
|
||||
rejectConnFsm int
|
||||
reassembled int
|
||||
outOfOrderBytes int
|
||||
outOfOrderPackets int
|
||||
biggestChunkBytes int
|
||||
biggestChunkPackets int
|
||||
overlapBytes int
|
||||
overlapPackets int
|
||||
}
|
||||
|
||||
const closeTimeout time.Duration = time.Hour * 24 // Closing inactive: TODO: from CLI
|
||||
const timeout time.Duration = time.Minute * 5 // Pending bytes: TODO: from CLI
|
||||
|
||||
var outputLevel int
|
||||
var errorsMap map[string]uint
|
||||
var errorsMapMutex sync.Mutex
|
||||
var errors uint
|
||||
|
||||
// Too bad for perf that a... is evaluated
|
||||
func Error(t string, s string, a ...interface{}) {
|
||||
errorsMapMutex.Lock()
|
||||
errors++
|
||||
nb, _ := errorsMap[t]
|
||||
errorsMap[t] = nb + 1
|
||||
errorsMapMutex.Unlock()
|
||||
if outputLevel >= 0 {
|
||||
fmt.Printf(s, a...)
|
||||
}
|
||||
}
|
||||
func Info(s string, a ...interface{}) {
|
||||
if outputLevel >= 1 {
|
||||
fmt.Printf(s, a...)
|
||||
}
|
||||
}
|
||||
func Debug(s string, a ...interface{}) {
|
||||
if outputLevel >= 2 {
|
||||
fmt.Printf(s, a...)
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* The TCP factory: returns a new Stream
|
||||
*/
|
||||
type tcpStreamFactory struct {
|
||||
wg sync.WaitGroup
|
||||
}
|
||||
|
||||
func (factory *tcpStreamFactory) New(net, transport gopacket.Flow, tcp *layers.TCP, ac reassembly.AssemblerContext) reassembly.Stream {
|
||||
Debug("* NEW: %s %s\n", net, transport)
|
||||
fsmOptions := reassembly.TCPSimpleFSMOptions{
|
||||
SupportMissingEstablishment: *allowmissinginit,
|
||||
}
|
||||
stream := &tcpStream{
|
||||
net: net,
|
||||
transport: transport,
|
||||
isTLS: true,
|
||||
tcpstate: reassembly.NewTCPSimpleFSM(fsmOptions),
|
||||
ident: fmt.Sprintf("%s:%s", net, transport),
|
||||
optchecker: reassembly.NewTCPOptionCheck(),
|
||||
}
|
||||
return stream
|
||||
}
|
||||
|
||||
func (factory *tcpStreamFactory) WaitGoRoutines() {
|
||||
factory.wg.Wait()
|
||||
}
|
||||
|
||||
/*
|
||||
* The assembler context
|
||||
*/
|
||||
type Context struct {
|
||||
CaptureInfo gopacket.CaptureInfo
|
||||
}
|
||||
|
||||
func (c *Context) GetCaptureInfo() gopacket.CaptureInfo {
|
||||
return c.CaptureInfo
|
||||
}
|
||||
|
||||
/*
|
||||
* TCP stream
|
||||
*/
|
||||
|
||||
/* It's a connection (bidirectional) */
|
||||
type tcpStream struct {
|
||||
tcpstate *reassembly.TCPSimpleFSM
|
||||
fsmerr bool
|
||||
optchecker reassembly.TCPOptionCheck
|
||||
net, transport gopacket.Flow
|
||||
isTLS bool
|
||||
reversed bool
|
||||
urls []string
|
||||
ident string
|
||||
sync.Mutex
|
||||
}
|
||||
|
||||
func (t *tcpStream) Accept(tcp *layers.TCP, ci gopacket.CaptureInfo, dir reassembly.TCPFlowDirection, nextSeq reassembly.Sequence, start *bool, ac reassembly.AssemblerContext) bool {
|
||||
// FSM
|
||||
if !t.tcpstate.CheckState(tcp, dir) {
|
||||
Error("FSM", "%s: Packet rejected by FSM (state:%s)\n", t.ident, t.tcpstate.String())
|
||||
stats.rejectFsm++
|
||||
if !t.fsmerr {
|
||||
t.fsmerr = true
|
||||
stats.rejectConnFsm++
|
||||
}
|
||||
if !*ignorefsmerr {
|
||||
return false
|
||||
}
|
||||
}
|
||||
// Options
|
||||
err := t.optchecker.Accept(tcp, ci, dir, nextSeq, start)
|
||||
if err != nil {
|
||||
Error("OptionChecker", "%s: Packet rejected by OptionChecker: %s\n", t.ident, err)
|
||||
stats.rejectOpt++
|
||||
if !*nooptcheck {
|
||||
return false
|
||||
}
|
||||
}
|
||||
// Checksum
|
||||
accept := true
|
||||
if *checksum {
|
||||
c, err := tcp.ComputeChecksum()
|
||||
if err != nil {
|
||||
Error("ChecksumCompute", "%s: Got error computing checksum: %s\n", t.ident, err)
|
||||
accept = false
|
||||
} else if c != 0x0 {
|
||||
Error("Checksum", "%s: Invalid checksum: 0x%x\n", t.ident, c)
|
||||
accept = false
|
||||
}
|
||||
}
|
||||
if !accept {
|
||||
stats.rejectOpt++
|
||||
}
|
||||
return accept
|
||||
}
|
||||
|
||||
func (t *tcpStream) ReassembledSG(sg reassembly.ScatterGather, ac reassembly.AssemblerContext) {
|
||||
dir, start, end, skip := sg.Info()
|
||||
length, saved := sg.Lengths()
|
||||
// update stats
|
||||
sgStats := sg.Stats()
|
||||
if skip > 0 {
|
||||
stats.missedBytes += skip
|
||||
}
|
||||
stats.sz += length - saved
|
||||
stats.pkt += sgStats.Packets
|
||||
if sgStats.Chunks > 1 {
|
||||
stats.reassembled++
|
||||
}
|
||||
stats.outOfOrderPackets += sgStats.QueuedPackets
|
||||
stats.outOfOrderBytes += sgStats.QueuedBytes
|
||||
if length > stats.biggestChunkBytes {
|
||||
stats.biggestChunkBytes = length
|
||||
}
|
||||
if sgStats.Packets > stats.biggestChunkPackets {
|
||||
stats.biggestChunkPackets = sgStats.Packets
|
||||
}
|
||||
if sgStats.OverlapBytes != 0 && sgStats.OverlapPackets == 0 {
|
||||
fmt.Printf("bytes:%d, pkts:%d\n", sgStats.OverlapBytes, sgStats.OverlapPackets)
|
||||
panic("Invalid overlap")
|
||||
}
|
||||
stats.overlapBytes += sgStats.OverlapBytes
|
||||
stats.overlapPackets += sgStats.OverlapPackets
|
||||
|
||||
var ident string
|
||||
if dir == reassembly.TCPDirClientToServer {
|
||||
ident = fmt.Sprintf("%v %v(%s): ", t.net, t.transport, dir)
|
||||
} else {
|
||||
ident = fmt.Sprintf("%v %v(%s): ", t.net.Reverse(), t.transport.Reverse(), dir)
|
||||
}
|
||||
Debug("%s: SG reassembled packet with %d bytes (start:%v,end:%v,skip:%d,saved:%d,nb:%d,%d,overlap:%d,%d)\n", ident, length, start, end, skip, saved, sgStats.Packets, sgStats.Chunks, sgStats.OverlapBytes, sgStats.OverlapPackets)
|
||||
if skip == -1 && *allowmissinginit {
|
||||
// this is allowed
|
||||
} else if skip != 0 {
|
||||
// Missing bytes in stream: do not even try to parse it
|
||||
return
|
||||
}
|
||||
data := sg.Fetch(length)
|
||||
if t.isTLS {
|
||||
if length > 0 {
|
||||
// We can't rely on TLS length field has there can be several successive Record Layers
|
||||
// We attempt to decode, and if it fails, we keep the slice for later.
|
||||
// Now we attempts TLS decoding
|
||||
tls := &layers.TLS{}
|
||||
var decoded []gopacket.LayerType
|
||||
// First we check if the packet is fragmented
|
||||
p := gopacket.NewDecodingLayerParser(layers.LayerTypeTLS, tls)
|
||||
err := p.DecodeLayers(data, &decoded)
|
||||
if err != nil {
|
||||
// Error("TLS-parser", "Failed to decode TLS: %v\n", err)
|
||||
// Debug("RAW %s\n", hex.Dump(data))
|
||||
sg.KeepFrom(0)
|
||||
} else {
|
||||
//Debug("TLS: %s\n", gopacket.LayerDump(tls))
|
||||
// Debug("TLS: %s\n", gopacket.LayerGoString(tls))
|
||||
if tls.Handshake != nil {
|
||||
for i, tlsrecord := range tls.Handshake {
|
||||
fmt.Printf("TLS record %#d %#v\n", i, tlsrecord.TLSHandshakeMsgType)
|
||||
if tlsrecord.TLSHandshakeMsgType == 11 {
|
||||
certs := make([]*x509.Certificate, len(tlsrecord.TLSHandshakeCertificate.Certificates))
|
||||
for i, asn1Data := range tlsrecord.TLSHandshakeCertificate.Certificates {
|
||||
cert, err := x509.ParseCertificate(asn1Data)
|
||||
if err != nil {
|
||||
panic("tls: failed to parse certificate from server: " + err.Error())
|
||||
}
|
||||
certs[i] = cert
|
||||
err = ioutil.WriteFile(fmt.Sprintf("./cert%d.crt", i), certs[i].Raw, 0644)
|
||||
if err != nil {
|
||||
panic("Could not write to file.")
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (t *tcpStream) ReassemblyComplete(ac reassembly.AssemblerContext) bool {
|
||||
Debug("%s: Connection closed\n", t.ident)
|
||||
// do not remove the connection to allow last ACK
|
||||
return false
|
||||
}
|
||||
|
||||
func main() {
|
||||
defer util.Run()()
|
||||
var handle *pcap.Handle
|
||||
var err error
|
||||
if *debug {
|
||||
outputLevel = 2
|
||||
} else if *verbose {
|
||||
outputLevel = 1
|
||||
} else if *quiet {
|
||||
outputLevel = -1
|
||||
}
|
||||
errorsMap = make(map[string]uint)
|
||||
if *fname != "" {
|
||||
if handle, err = pcap.OpenOffline(*fname); err != nil {
|
||||
log.Fatal("PCAP OpenOffline error:", err)
|
||||
}
|
||||
} else {
|
||||
// Open live on interface
|
||||
if handle, err = pcap.OpenLive(*iface, 65536, true, 0); err != nil {
|
||||
log.Fatal("PCAP OpenOffline error:", err)
|
||||
}
|
||||
defer handle.Close()
|
||||
}
|
||||
if len(flag.Args()) > 0 {
|
||||
bpffilter := strings.Join(flag.Args(), " ")
|
||||
Info("Using BPF filter %q\n", bpffilter)
|
||||
if err = handle.SetBPFFilter(bpffilter); err != nil {
|
||||
log.Fatal("BPF filter error:", err)
|
||||
}
|
||||
}
|
||||
|
||||
var dec gopacket.Decoder
|
||||
var ok bool
|
||||
if dec, ok = gopacket.DecodersByLayerName["Ethernet"]; !ok {
|
||||
log.Fatal("No eth decoder")
|
||||
}
|
||||
source := gopacket.NewPacketSource(handle, dec)
|
||||
source.Lazy = *lazy
|
||||
source.NoCopy = true
|
||||
Info("Starting to read packets\n")
|
||||
count := 0
|
||||
bytes := int64(0)
|
||||
defragger := ip4defrag.NewIPv4Defragmenter()
|
||||
|
||||
streamFactory := &tcpStreamFactory{}
|
||||
streamPool := reassembly.NewStreamPool(streamFactory)
|
||||
assembler := reassembly.NewAssembler(streamPool)
|
||||
|
||||
signalChan := make(chan os.Signal, 1)
|
||||
signal.Notify(signalChan, os.Interrupt)
|
||||
|
||||
for packet := range source.Packets() {
|
||||
count++
|
||||
Debug("PACKET #%d\n", count)
|
||||
data := packet.Data()
|
||||
bytes += int64(len(data))
|
||||
|
||||
// defrag the IPv4 packet if required
|
||||
if !*nodefrag {
|
||||
ip4Layer := packet.Layer(layers.LayerTypeIPv4)
|
||||
if ip4Layer == nil {
|
||||
continue
|
||||
}
|
||||
ip4 := ip4Layer.(*layers.IPv4)
|
||||
l := ip4.Length
|
||||
newip4, err := defragger.DefragIPv4(ip4)
|
||||
if err != nil {
|
||||
log.Fatalln("Error while de-fragmenting", err)
|
||||
} else if newip4 == nil {
|
||||
Debug("Fragment...\n")
|
||||
continue // ip packet fragment, we don't have whole packet yet.
|
||||
}
|
||||
if newip4.Length != l {
|
||||
stats.ipdefrag++
|
||||
Debug("Decoding re-assembled packet: %s\n", newip4.NextLayerType())
|
||||
pb, ok := packet.(gopacket.PacketBuilder)
|
||||
if !ok {
|
||||
panic("Not a PacketBuilder")
|
||||
}
|
||||
nextDecoder := newip4.NextLayerType()
|
||||
nextDecoder.Decode(newip4.Payload, pb)
|
||||
}
|
||||
}
|
||||
|
||||
tcp := packet.Layer(layers.LayerTypeTCP)
|
||||
if tcp != nil {
|
||||
tcp := tcp.(*layers.TCP)
|
||||
if *checksum {
|
||||
err := tcp.SetNetworkLayerForChecksum(packet.NetworkLayer())
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to set network layer for checksum: %s\n", err)
|
||||
}
|
||||
}
|
||||
c := Context{
|
||||
CaptureInfo: packet.Metadata().CaptureInfo,
|
||||
}
|
||||
stats.totalsz += len(tcp.Payload)
|
||||
assembler.AssembleWithContext(packet.NetworkLayer().NetworkFlow(), tcp, &c)
|
||||
}
|
||||
if count%*statsevery == 0 {
|
||||
ref := packet.Metadata().CaptureInfo.Timestamp
|
||||
flushed, closed := assembler.FlushWithOptions(reassembly.FlushOptions{T: ref.Add(-timeout), TC: ref.Add(-closeTimeout)})
|
||||
Debug("Forced flush: %d flushed, %d closed (%s)", flushed, closed, ref)
|
||||
}
|
||||
|
||||
var done bool
|
||||
select {
|
||||
case <-signalChan:
|
||||
fmt.Fprintf(os.Stderr, "\nCaught SIGINT: aborting\n")
|
||||
done = true
|
||||
default:
|
||||
// NOP: continue
|
||||
}
|
||||
if done {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
assembler.FlushAll()
|
||||
streamFactory.WaitGoRoutines()
|
||||
|
||||
}
|
||||
Loading…
Reference in New Issue