diff --git a/main.go b/main.go index 69aea69..1823abc 100644 --- a/main.go +++ b/main.go @@ -1,50 +1,230 @@ package main -// APACHE 2.0 import ( + "bytes" + "crypto/x509" "database/sql" + "encoding/json" + "errors" "fmt" "io/ioutil" "log" + "strings" + "time" "github.com/gomodule/redigo/redis" _ "github.com/lib/pq" ) +type certMapElm struct { + CertHash string + chain chain + *x509.Certificate +} + +type sessionRecord struct { + ServerIP string + ServerPort string + ClientIP string + ClientPort string + TLSH string + Timestamp time.Time + JA3 string + JA3Digest string + JA3S string + JA3SDigest string + Certificates []certMapElm +} + +type chain struct { + isValid bool + s string +} + +var db *sql.DB +var cr redis.Conn + +var connectRedis = false +var connectDB = true + func main() { - // connect to redis - c, err := redis.Dial("tcp", ":6380", redis.DialDatabase(2)) - defer c.Close() - if err != nil { - panic(err) + if connectDB { + initDB() + defer db.Close() } - // connect to db - connStr := "user=postgres password=postgres dbname=passivessl" - db, err := sql.Open("postgres", connStr) - defer db.Close() - if err != nil { - panic(err) - } + var jsonPath string - // pop redis queue - for { - jsonPath, err := redis.String(c.Do("LPOP", "analyzer:ja3-jl:0894517855f047d2a77b4473d3a9cc5b")) - if err != nil { - log.Fatal("Queue processed") + if connectRedis { + initRedis() + //defer cr.Close() + // pop redis queue + for { + err := errors.New("") + jsonPath, err = redis.String(cr.Do("LPOP", "analyzer:ja3-jl:0894517855f047d2a77b4473d3a9cc5b")) + if err != nil { + log.Fatal("Queue processed") + } } + + } else { + jsonPath = "./test.json" + // read corresponding json file dat, err := ioutil.ReadFile(jsonPath) if err != nil { log.Fatal(err) } + // Unmarshal JSON file + s := sessionRecord{} + _ = json.Unmarshal([]byte(dat), &s) - q := `INSERT INTO sessions (data) VALUES ($1) RETURNING id` - id := 0 - err = db.QueryRow(q, dat).Scan(&id) + // Insert Session + ids, err := insertSession(&s) if err != nil { - panic(err) + log.Fatal(fmt.Sprintf("Insert Sessions into DB failed: %q", err)) } - fmt.Println("New record ID is:", id) + // Attempt to roughly build a chain of trust + session := buildChain(&s) + + // Insert Certificates + idc, err := insertCertificates(session) + if err != nil { + log.Fatal(fmt.Sprintf("Insert Certificate into DB failed: %q", err)) + } + // Launch go routine to create the relationship between certificates and sessions + err = linkSessionCert(ids, idc) + if err != nil { + log.Fatal(fmt.Sprintf("Could not link Certs and Session into DB failed: %q", err)) + } + // Launch go routine to create public keys } } + +// linkSessionCert creates the link between a session and its certificates +func linkSessionCert(ids int64, idc []string) error { + for _, i := range idc { + q := `INSERT INTO "many_sessionRecord_has_many_certificate" ("id_sessionRecord", "hash_certificate") VALUES ($1, $2)` + _, err := db.Query(q, ids, i) + if err != nil { + return err + } + } + return nil +} + +// buildChain attempts to rearrange certificate as a chain of trust from a sessionRecord (that +// contains a slice of certificate). If the chain of trust is build successfully +// it marked as valid, If not root is found or if the chain is broken, it +// does not touch the original slice and mark the chain as invalid. +func buildChain(s *sessionRecord) (*sessionRecord) { + certChain := make([]certMapElm, 0) + + // First we find the leaf + for _, c := range s.Certificates { + fmt.Println(c.Certificate.Issuer.String()) + fmt.Println(c.Certificate.Subject.String()) + fmt.Println(c.Certificate.Subject.String() == c.Certificate.Issuer.String()) + if !c.Certificate.IsCA { + certChain = append(certChain, c) + } + } + // Find the parent of each certificate + for _, _ = range s.Certificates { + for i, _ := range s.Certificates { + if s.Certificates[i].Certificate.Subject.String() == certChain[len(certChain)-1].Issuer.String() { + certChain = append(certChain, s.Certificates[i]) + } + } + } + // Write the new chain + if len(certChain) == len(s.Certificates) { + cstr := make([]string, 0) + for i := len(certChain) - 1; i >= 0; i-- { + certChain[i].chain.isValid = true + cstr = append(cstr, certChain[i].CertHash) + certChain[i].chain.s = strings.Join(cstr, ".") + } + tmp := s + tmp.Certificates = certChain + return tmp + } + return s +} + +func insertCertificate(c certMapElm) (string, error) { + q := `INSERT INTO "certificate" (hash, "is_CA", issuer, subject, cert_chain, is_valid_chain, file_path) VALUES ($1, $2, $3, $4, $5, $6, $7) RETURNING hash` + var hash string + err := db.QueryRow(q, c.CertHash, c.Certificate.IsCA, c.Certificate.Issuer.String(), c.Certificate.Subject.String(), c.chain.s, c.chain.isValid, getFullPath(c.CertHash)).Scan(&hash) + if err != nil { + return "", err + } + return hash, nil +} + +// getFullPath takes a certificate's hash and return the full path to +// its location on disk +func getFullPath(h string) (string) { + return "TODO PATH" +} + +func insertCertificates(s *sessionRecord) ([]string, error) { + var inserted []string + for _, certificate := range s.Certificates { + idc, err := insertCertificate(certificate) + if err != nil { + return inserted, err + } + inserted = append(inserted, idc) + } + return inserted, nil +} + +func insertSession(s *sessionRecord) (int64, error) { + q := `INSERT INTO "sessionRecord" (dst_ip, src_ip, dst_port, src_port, timestamp) VALUES ($1, $2, $3, $4, $5) RETURNING id` + var id int64 + err := db.QueryRow(q, s.ServerIP, s.ClientIP, s.ServerPort, s.ClientPort, s.Timestamp).Scan(&id) + if err != nil { + return 0, err + } + return id, nil +} + +func initRedis() { + err := errors.New("") + cr, err = redis.Dial("tcp", ":6380", redis.DialDatabase(2)) + if err != nil { + panic(err) + } +} + +func initDB() { + connStr := "user=postgres password=postgres dbname=new_database" + err := errors.New("") + db, err = sql.Open("postgres", connStr) + if err != nil { + panic(err) + } +} + +// String returns a string that describes a TLSSession +func (t *sessionRecord) String() string { + var buf bytes.Buffer + buf.WriteString(fmt.Sprintf("---------------SESSION START-------------------\n")) + buf.WriteString(fmt.Sprintf("Time: %v\n", t.Timestamp)) + buf.WriteString(fmt.Sprintf("Client: %v:%v\n", t.ClientIP, t.ClientPort)) + buf.WriteString(fmt.Sprintf("Server: %v:%v\n", t.ServerIP, t.ServerPort)) + buf.WriteString(fmt.Sprintf("TLSH: %q\n", t.TLSH)) + buf.WriteString(fmt.Sprintf("ja3: %q\n", t.JA3)) + buf.WriteString(fmt.Sprintf("ja3 Digest: %q\n", t.JA3Digest)) + buf.WriteString(fmt.Sprintf("ja3s: %q\n", t.JA3S)) + buf.WriteString(fmt.Sprintf("ja3s Digest: %q\n", t.JA3SDigest)) + for _, certMe := range t.Certificates { + buf.WriteString(fmt.Sprintf("Certificate Issuer: %q\n", certMe.Certificate.Issuer)) + buf.WriteString(fmt.Sprintf("Certificate Subject: %q\n", certMe.Certificate.Subject)) + buf.WriteString(fmt.Sprintf("Certificate is CA: %t\n", certMe.Certificate.IsCA)) + buf.WriteString(fmt.Sprintf("Certificate SHA256: %q\n", certMe.CertHash)) + } + buf.WriteString(fmt.Sprintf("---------------SESSION END--------------------\n")) + return buf.String() +} diff --git a/passivessl.sql b/passivessl.sql index 0068ae2..c9a208a 100644 --- a/passivessl.sql +++ b/passivessl.sql @@ -1,66 +1,254 @@ --- --- PostgreSQL database dump --- +-- Database generated with pgModeler (PostgreSQL Database Modeler). +-- pgModeler version: 0.9.1-beta +-- PostgreSQL version: 10.0 +-- Project Site: pgmodeler.com.br +-- Model Author: --- --- Dumped from database version 10.6 (Ubuntu 10.6-0ubuntu0.18.04.1) --- Dumped by pg_dump version 10.6 (Ubuntu 10.6-0ubuntu0.18.04.1) -SET statement_timeout = 0; -SET lock_timeout = 0; -SET idle_in_transaction_session_timeout = 0; -SET client_encoding = 'UTF8'; -SET standard_conforming_strings = on; -SELECT pg_catalog.set_config('search_path', '', false); -SET check_function_bodies = false; -SET client_min_messages = warning; -SET row_security = off; +-- Database creation must be done outside an multicommand file. +-- These commands were put in this file only for convenience. +-- -- object: new_database | type: DATABASE -- +-- -- DROP DATABASE IF EXISTS new_database; +-- CREATE DATABASE new_database +-- ; +-- -- ddl-end -- +-- -SET default_tablespace = ''; +-- object: ltree | type: EXTENSION -- +-- DROP EXTENSION IF EXISTS ltree CASCADE; +CREATE EXTENSION ltree + WITH SCHEMA public; +-- ddl-end -- -SET default_with_oids = false; +-- object: hstore | type: EXTENSION -- +-- DROP EXTENSION IF EXISTS hstore CASCADE; +CREATE EXTENSION hstore + WITH SCHEMA public; +-- ddl-end -- --- --- Name: sessions; Type: TABLE; Schema: public; Owner: postgres --- +-- object: public.public_key | type: TABLE -- +-- DROP TABLE IF EXISTS public.public_key CASCADE; +CREATE TABLE public.public_key( + hash bytea NOT NULL, + type text NOT NULL, + modulus text, + exponent smallint, + modules_size smallint, + CONSTRAINT public_key_pk PRIMARY KEY (hash) -CREATE TABLE public.sessions ( - data jsonb, - id integer NOT NULL ); +-- ddl-end -- +ALTER TABLE public.public_key OWNER TO postgres; +-- ddl-end -- + +-- object: public.certificate | type: TABLE -- +-- DROP TABLE IF EXISTS public.certificate CASCADE; +CREATE TABLE public.certificate( + file_path varchar(4096) NOT NULL, + issuer text, + cert_chain public.ltree, + subject text, + hash bytea NOT NULL, + "is_CA" bool NOT NULL DEFAULT false, + is_valid_chain bool NOT NULL DEFAULT false, + CONSTRAINT certificate_pk PRIMARY KEY (hash) + +); +-- ddl-end -- +ALTER TABLE public.certificate OWNER TO postgres; +-- ddl-end -- + +-- object: public.many_certificate_has_many_public_key | type: TABLE -- +-- DROP TABLE IF EXISTS public.many_certificate_has_many_public_key CASCADE; +CREATE TABLE public.many_certificate_has_many_public_key( + hash_certificate bytea NOT NULL, + hash_public_key bytea NOT NULL, + CONSTRAINT many_certificate_has_many_public_key_pk PRIMARY KEY (hash_certificate,hash_public_key) + +); +-- ddl-end -- + +-- object: certificate_fk | type: CONSTRAINT -- +-- ALTER TABLE public.many_certificate_has_many_public_key DROP CONSTRAINT IF EXISTS certificate_fk CASCADE; +ALTER TABLE public.many_certificate_has_many_public_key ADD CONSTRAINT certificate_fk FOREIGN KEY (hash_certificate) +REFERENCES public.certificate (hash) MATCH FULL +ON DELETE RESTRICT ON UPDATE CASCADE; +-- ddl-end -- + +-- object: public_key_fk | type: CONSTRAINT -- +-- ALTER TABLE public.many_certificate_has_many_public_key DROP CONSTRAINT IF EXISTS public_key_fk CASCADE; +ALTER TABLE public.many_certificate_has_many_public_key ADD CONSTRAINT public_key_fk FOREIGN KEY (hash_public_key) +REFERENCES public.public_key (hash) MATCH FULL +ON DELETE RESTRICT ON UPDATE CASCADE; +-- ddl-end -- + +-- object: public."sessionRecord" | type: TABLE -- +-- DROP TABLE IF EXISTS public."sessionRecord" CASCADE; +CREATE TABLE public."sessionRecord"( + id bigserial NOT NULL, + dst_ip inet NOT NULL, + src_ip inet NOT NULL, + dst_port int4 NOT NULL, + src_port int4 NOT NULL, + hash_ja3 bytea, + "timestamp" time(0) with time zone, + CONSTRAINT "sessionRecord_pk" PRIMARY KEY (id) + +); +-- ddl-end -- +ALTER TABLE public."sessionRecord" OWNER TO postgres; +-- ddl-end -- + +-- object: public.ja3 | type: TABLE -- +-- DROP TABLE IF EXISTS public.ja3 CASCADE; +CREATE TABLE public.ja3( + hash bytea NOT NULL, + raw text, + type smallint NOT NULL, + CONSTRAINT j3a_pk PRIMARY KEY (hash) + +); +-- ddl-end -- +ALTER TABLE public.ja3 OWNER TO postgres; +-- ddl-end -- + +-- object: ja3_fk | type: CONSTRAINT -- +-- ALTER TABLE public."sessionRecord" DROP CONSTRAINT IF EXISTS ja3_fk CASCADE; +ALTER TABLE public."sessionRecord" ADD CONSTRAINT ja3_fk FOREIGN KEY (hash_ja3) +REFERENCES public.ja3 (hash) MATCH FULL +ON DELETE SET NULL ON UPDATE CASCADE; +-- ddl-end -- + +-- object: public."many_sessionRecord_has_many_certificate" | type: TABLE -- +-- DROP TABLE IF EXISTS public."many_sessionRecord_has_many_certificate" CASCADE; +CREATE TABLE public."many_sessionRecord_has_many_certificate"( + "id_sessionRecord" bigint NOT NULL, + hash_certificate bytea NOT NULL, + CONSTRAINT "many_sessionRecord_has_many_certificate_pk" PRIMARY KEY ("id_sessionRecord",hash_certificate) + +); +-- ddl-end -- + +-- object: "sessionRecord_fk" | type: CONSTRAINT -- +-- ALTER TABLE public."many_sessionRecord_has_many_certificate" DROP CONSTRAINT IF EXISTS "sessionRecord_fk" CASCADE; +ALTER TABLE public."many_sessionRecord_has_many_certificate" ADD CONSTRAINT "sessionRecord_fk" FOREIGN KEY ("id_sessionRecord") +REFERENCES public."sessionRecord" (id) MATCH FULL +ON DELETE RESTRICT ON UPDATE CASCADE; +-- ddl-end -- + +-- object: certificate_fk | type: CONSTRAINT -- +-- ALTER TABLE public."many_sessionRecord_has_many_certificate" DROP CONSTRAINT IF EXISTS certificate_fk CASCADE; +ALTER TABLE public."many_sessionRecord_has_many_certificate" ADD CONSTRAINT certificate_fk FOREIGN KEY (hash_certificate) +REFERENCES public.certificate (hash) MATCH FULL +ON DELETE RESTRICT ON UPDATE CASCADE; +-- ddl-end -- + +-- object: public.fuzzy_hash | type: TABLE -- +-- DROP TABLE IF EXISTS public.fuzzy_hash CASCADE; +CREATE TABLE public.fuzzy_hash( + id bigserial NOT NULL, + type text NOT NULL, + value public.hstore NOT NULL, + hash_ja3 bytea, + CONSTRAINT fuzzy_hash_pk PRIMARY KEY (id) + +); +-- ddl-end -- +ALTER TABLE public.fuzzy_hash OWNER TO postgres; +-- ddl-end -- + +-- object: public.software | type: TABLE -- +-- DROP TABLE IF EXISTS public.software CASCADE; +CREATE TABLE public.software( + id serial NOT NULL, + name text NOT NULL, + version text, + CONSTRAINT software_pk PRIMARY KEY (id) + +); +-- ddl-end -- +ALTER TABLE public.software OWNER TO postgres; +-- ddl-end -- + +-- object: public.annotation | type: TABLE -- +-- DROP TABLE IF EXISTS public.annotation CASCADE; +CREATE TABLE public.annotation( + id serial NOT NULL, + hash_ja3 bytea, + confidence smallint, + id_software integer, + CONSTRAINT annotation_pk PRIMARY KEY (id) + +); +-- ddl-end -- +ALTER TABLE public.annotation OWNER TO postgres; +-- ddl-end -- + +-- object: ja3_fk | type: CONSTRAINT -- +-- ALTER TABLE public.annotation DROP CONSTRAINT IF EXISTS ja3_fk CASCADE; +ALTER TABLE public.annotation ADD CONSTRAINT ja3_fk FOREIGN KEY (hash_ja3) +REFERENCES public.ja3 (hash) MATCH FULL +ON DELETE SET NULL ON UPDATE CASCADE; +-- ddl-end -- + +-- object: software_fk | type: CONSTRAINT -- +-- ALTER TABLE public.annotation DROP CONSTRAINT IF EXISTS software_fk CASCADE; +ALTER TABLE public.annotation ADD CONSTRAINT software_fk FOREIGN KEY (id_software) +REFERENCES public.software (id) MATCH FULL +ON DELETE SET NULL ON UPDATE CASCADE; +-- ddl-end -- + +-- object: ja3_trie | type: INDEX -- +-- DROP INDEX IF EXISTS public.ja3_trie CASCADE; +CREATE INDEX CONCURRENTLY ja3_trie ON public.ja3 + USING spgist + ( + raw + ); +-- ddl-end -- + +-- object: hash_index | type: INDEX -- +-- DROP INDEX IF EXISTS public.hash_index CASCADE; +CREATE INDEX hash_index ON public.certificate + USING btree + ( + hash + ); +-- ddl-end -- + +-- object: pk_index | type: INDEX -- +-- DROP INDEX IF EXISTS public.pk_index CASCADE; +CREATE INDEX pk_index ON public.public_key + USING btree + ( + hash + ); +-- ddl-end -- + +-- object: dst_index | type: INDEX -- +-- DROP INDEX IF EXISTS public.dst_index CASCADE; +CREATE INDEX dst_index ON public."sessionRecord" + USING btree + ( + dst_ip + ); +-- ddl-end -- + +-- object: path_index | type: INDEX -- +-- DROP INDEX IF EXISTS public.path_index CASCADE; +CREATE INDEX path_index ON public.certificate + USING gist + ( + cert_chain + ) + WITH (BUFFERING = ON); +-- ddl-end -- + +-- object: ja3_fk | type: CONSTRAINT -- +-- ALTER TABLE public.fuzzy_hash DROP CONSTRAINT IF EXISTS ja3_fk CASCADE; +ALTER TABLE public.fuzzy_hash ADD CONSTRAINT ja3_fk FOREIGN KEY (hash_ja3) +REFERENCES public.ja3 (hash) MATCH FULL +ON DELETE SET NULL ON UPDATE CASCADE; +-- ddl-end -- -ALTER TABLE public.sessions OWNER TO postgres; - --- --- Name: sessions_id_seq; Type: SEQUENCE; Schema: public; Owner: postgres --- - -CREATE SEQUENCE public.sessions_id_seq - AS integer - START WITH 1 - INCREMENT BY 1 - NO MINVALUE - NO MAXVALUE - CACHE 1; - - -ALTER TABLE public.sessions_id_seq OWNER TO postgres; - --- --- Name: sessions_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: postgres --- - -ALTER SEQUENCE public.sessions_id_seq OWNED BY public.sessions.id; - - --- --- Name: sessions id; Type: DEFAULT; Schema: public; Owner: postgres --- - -ALTER TABLE ONLY public.sessions ALTER COLUMN id SET DEFAULT nextval('public.sessions_id_seq'::regclass); - - --- --- PostgreSQL database dump complete --- -