From b499781fc6c8f0e7753051f21347dacd2bd64697 Mon Sep 17 00:00:00 2001 From: Jean-Louis Huynen Date: Thu, 11 Apr 2019 16:25:10 +0200 Subject: [PATCH] wip --- README.md | 1 + main.go | 44 +++++++++++++++++++++------- passivessl.sql | 78 ++++++++++++++++++++++++++++++++++++++++++-------- 3 files changed, 101 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 36db9c4..630beb1 100644 --- a/README.md +++ b/README.md @@ -4,4 +4,5 @@ analyzer-d4-passivessl fetch a redis feed of certificate and TLS sessions and ma ```bash go get github.com/gomodule/redigo/redis go get github.com/lib/pq +sudo apt install postgresal-plpython3-10 ``` diff --git a/main.go b/main.go index 04936f9..9ed64a7 100644 --- a/main.go +++ b/main.go @@ -100,12 +100,21 @@ func main() { s := sessionRecord{} _ = json.Unmarshal([]byte(dat), &s) - if len(s.Certificates) > 0 { + if isValid(&s){ + // Create ja3* records + err = insertJA3(&s) + if err != nil { + log.Fatal(fmt.Sprintf("Could not insert JA3 into DB: %q", err)) + } // Insert Session ids, err := insertSession(&s) if err != nil { log.Fatal(fmt.Sprintf("Insert Sessions into DB failed: %q", err)) } + err = insertFuzzyHash(&s, ids) + if err != nil { + log.Fatal(fmt.Sprintf("Insert Fuzzy Hash into DB failed: %q", err)) + } // Attempt to roughly build a chain of trust session := buildChain(&s) @@ -119,17 +128,23 @@ func main() { if err != nil { log.Fatal(fmt.Sprintf("Could not link Certs and Session into DB: %q", err)) } - - // Create ja3* records - err = insertJA3(session) - if err != nil { - log.Fatal(fmt.Sprintf("Could not insert JA3 into DB: %q", err)) - } } } } } +func isValid(s * sessionRecord) bool { + // Relationships + if len(s.Certificates) < 1 || s.JA3Digest == "" || s.JA3 == "" { + return false + } + // Basic informations + if s.ClientPort == "" || s.ServerPort == "" || s.ServerIP == "" || s.ClientIP == "" { + return false + } + return true +} + func insertJA3(s *sessionRecord) error { q := `INSERT INTO "ja3" (hash, raw, type) VALUES ($1, $2, $3) ON CONFLICT DO NOTHING` _, err := db.Exec(q, s.JA3Digest, s.JA3, "ja3") @@ -144,6 +159,15 @@ func insertJA3(s *sessionRecord) error { return nil } +func insertFuzzyHash(s *sessionRecord, ids int64) error { + q := `INSERT INTO "fuzzy_hash" (type, value, "id_sessionRecord") VALUES ($1, $2, $3) ON CONFLICT DO NOTHING` + _, err := db.Exec(q, "TLSH", s.TLSH, ids) + if err != nil { + return err + } + return nil +} + // insertPublicKeys insert each public key of each certificate of a session func insertPublicKey(c x509.Certificate) (string, error) { pub, err := x509.ParsePKIXPublicKey(c.RawSubjectPublicKeyInfo) @@ -321,9 +345,9 @@ func insertCertificates(s *sessionRecord) ([]string, error) { } func insertSession(s *sessionRecord) (int64, error) { - q := `INSERT INTO "sessionRecord" (dst_ip, src_ip, dst_port, src_port, timestamp) VALUES ($1, $2, $3, $4, $5) RETURNING id` + q := `INSERT INTO "sessionRecord" (dst_ip, src_ip, dst_port, src_port, timestamp, hash_ja3) VALUES ($1, $2, $3, $4, $5, $6) RETURNING id` var id int64 - err := db.QueryRow(q, s.ServerIP, s.ClientIP, s.ServerPort, s.ClientPort, s.Timestamp).Scan(&id) + err := db.QueryRow(q, s.ServerIP, s.ClientIP, s.ServerPort, s.ClientPort, s.Timestamp, s.JA3Digest).Scan(&id) if err != nil { return 0, err } @@ -339,7 +363,7 @@ func initRedis() { } func initDB() { - connStr := "user=postgres password=postgres dbname=new_database" + connStr := "user=postgres password=postgres dbname=passive_ssl" err := errors.New("") db, err = sql.Open("postgres", connStr) if err != nil { diff --git a/passivessl.sql b/passivessl.sql index 069c94b..91ce083 100644 --- a/passivessl.sql +++ b/passivessl.sql @@ -4,12 +4,15 @@ -- Project Site: pgmodeler.com.br -- Model Author: --- +SET check_function_bodies = false; +-- ddl-end -- + -- Database creation must be done outside an multicommand file. -- These commands were put in this file only for convenience. --- -- object: new_database | type: DATABASE -- --- -- DROP DATABASE IF EXISTS new_database; --- CREATE DATABASE new_database +-- -- object: passive_ssl | type: DATABASE -- +-- -- DROP DATABASE IF EXISTS passive_ssl; +-- CREATE DATABASE passive_ssl -- ; -- -- ddl-end -- -- @@ -64,7 +67,7 @@ CREATE TABLE public.certificate( is_valid_chain bool NOT NULL DEFAULT false, "notBefore" time, "notAfter" time, - "isSS" bool NOT NULL DEFAULT false, + "is_SS" bool NOT NULL DEFAULT false, CONSTRAINT certificate_pk PRIMARY KEY (hash) ); @@ -104,7 +107,7 @@ CREATE TABLE public."sessionRecord"( src_ip inet NOT NULL, dst_port int4 NOT NULL, src_port int4 NOT NULL, - hash_ja3 bytea, + hash_ja3 bytea NOT NULL, "timestamp" time(0) with time zone, CONSTRAINT "sessionRecord_pk" PRIMARY KEY (id) @@ -130,7 +133,7 @@ ALTER TABLE public.ja3 OWNER TO postgres; -- ALTER TABLE public."sessionRecord" DROP CONSTRAINT IF EXISTS ja3_fk CASCADE; ALTER TABLE public."sessionRecord" ADD CONSTRAINT ja3_fk FOREIGN KEY (hash_ja3) REFERENCES public.ja3 (hash) MATCH FULL -ON DELETE SET NULL ON UPDATE CASCADE; +ON DELETE RESTRICT ON UPDATE CASCADE; -- ddl-end -- -- object: public."many_sessionRecord_has_many_certificate" | type: TABLE -- @@ -162,8 +165,8 @@ ON DELETE RESTRICT ON UPDATE CASCADE; CREATE TABLE public.fuzzy_hash( id bigserial NOT NULL, type text NOT NULL, - value public.hstore NOT NULL, - hash_ja3 bytea, + value text NOT NULL, + "id_sessionRecord" bigint, CONSTRAINT fuzzy_hash_pk PRIMARY KEY (id) ); @@ -258,11 +261,62 @@ CREATE INDEX path_index ON public.certificate WITH (BUFFERING = ON); -- ddl-end -- --- object: ja3_fk | type: CONSTRAINT -- --- ALTER TABLE public.fuzzy_hash DROP CONSTRAINT IF EXISTS ja3_fk CASCADE; -ALTER TABLE public.fuzzy_hash ADD CONSTRAINT ja3_fk FOREIGN KEY (hash_ja3) -REFERENCES public.ja3 (hash) MATCH FULL +-- object: "sessionRecord_fk" | type: CONSTRAINT -- +-- ALTER TABLE public.fuzzy_hash DROP CONSTRAINT IF EXISTS "sessionRecord_fk" CASCADE; +ALTER TABLE public.fuzzy_hash ADD CONSTRAINT "sessionRecord_fk" FOREIGN KEY ("id_sessionRecord") +REFERENCES public."sessionRecord" (id) MATCH FULL ON DELETE SET NULL ON UPDATE CASCADE; -- ddl-end -- +-- object: plpython3u | type: LANGUAGE -- +-- DROP LANGUAGE IF EXISTS plpython3u CASCADE; +CREATE LANGUAGE plpython3u; +-- ddl-end -- +ALTER LANGUAGE plpython3u OWNER TO postgres; +-- ddl-end -- + +-- object: public.tlshc | type: FUNCTION -- +-- DROP FUNCTION IF EXISTS public.tlshc(text,text) CASCADE; +CREATE FUNCTION public.tlshc ( a text, b text) + RETURNS int4 + LANGUAGE plpython3u + VOLATILE + CALLED ON NULL INPUT + SECURITY INVOKER + COST 1 + AS $$ +import tlsh +return tlsh.diff(a, b) +$$; +-- ddl-end -- +ALTER FUNCTION public.tlshc(text,text) OWNER TO postgres; +-- ddl-end -- + +-- object: public.tlsht | type: FUNCTION -- +-- DROP FUNCTION IF EXISTS public.tlsht(IN text,IN text,IN int4,int4) CASCADE; +CREATE FUNCTION public.tlsht (IN filter text, IN hash text, IN threshold int4, maxrows int4) + RETURNS SETOF public.fuzzy_hash + LANGUAGE plpython3u + IMMUTABLE LEAKPROOF + RETURNS NULL ON NULL INPUT + SECURITY INVOKER + COST 1 + ROWS 1000 + AS $$ +import tlsh +param = ["TLSH"] +#param[0] = filter +#plan = plpy.prepare("SELECT * FROM fuzzy_hash WHERE type <> $1", ["text"]) +#rv = plan.execute(param, maxrows) +rv = plpy.execute("SELECT * FROM fuzzy_hash", 1000) +r = [] +for x in rv: + if tlsh.diff(x["value"], hash) < threshold: + r.append(x) +return r +$$; +-- ddl-end -- +ALTER FUNCTION public.tlsht(IN text,IN text,IN int4,int4) OWNER TO postgres; +-- ddl-end -- +