wip - use postgres

master
Jean-Louis Huynen 2019-04-05 14:41:26 +02:00
parent 2f88b3653e
commit 0cc7180688
No known key found for this signature in database
GPG Key ID: 4CDEBABACE9A5DC9
2 changed files with 448 additions and 80 deletions

226
main.go
View File

@ -1,50 +1,230 @@
package main
// APACHE 2.0
import (
"bytes"
"crypto/x509"
"database/sql"
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"log"
"strings"
"time"
"github.com/gomodule/redigo/redis"
_ "github.com/lib/pq"
)
type certMapElm struct {
CertHash string
chain chain
*x509.Certificate
}
type sessionRecord struct {
ServerIP string
ServerPort string
ClientIP string
ClientPort string
TLSH string
Timestamp time.Time
JA3 string
JA3Digest string
JA3S string
JA3SDigest string
Certificates []certMapElm
}
type chain struct {
isValid bool
s string
}
var db *sql.DB
var cr redis.Conn
var connectRedis = false
var connectDB = true
func main() {
// connect to redis
c, err := redis.Dial("tcp", ":6380", redis.DialDatabase(2))
defer c.Close()
if err != nil {
panic(err)
if connectDB {
initDB()
defer db.Close()
}
// connect to db
connStr := "user=postgres password=postgres dbname=passivessl"
db, err := sql.Open("postgres", connStr)
defer db.Close()
if err != nil {
panic(err)
}
var jsonPath string
// pop redis queue
for {
jsonPath, err := redis.String(c.Do("LPOP", "analyzer:ja3-jl:0894517855f047d2a77b4473d3a9cc5b"))
if err != nil {
log.Fatal("Queue processed")
if connectRedis {
initRedis()
//defer cr.Close()
// pop redis queue
for {
err := errors.New("")
jsonPath, err = redis.String(cr.Do("LPOP", "analyzer:ja3-jl:0894517855f047d2a77b4473d3a9cc5b"))
if err != nil {
log.Fatal("Queue processed")
}
}
} else {
jsonPath = "./test.json"
// read corresponding json file
dat, err := ioutil.ReadFile(jsonPath)
if err != nil {
log.Fatal(err)
}
// Unmarshal JSON file
s := sessionRecord{}
_ = json.Unmarshal([]byte(dat), &s)
q := `INSERT INTO sessions (data) VALUES ($1) RETURNING id`
id := 0
err = db.QueryRow(q, dat).Scan(&id)
// Insert Session
ids, err := insertSession(&s)
if err != nil {
panic(err)
log.Fatal(fmt.Sprintf("Insert Sessions into DB failed: %q", err))
}
fmt.Println("New record ID is:", id)
// Attempt to roughly build a chain of trust
session := buildChain(&s)
// Insert Certificates
idc, err := insertCertificates(session)
if err != nil {
log.Fatal(fmt.Sprintf("Insert Certificate into DB failed: %q", err))
}
// Launch go routine to create the relationship between certificates and sessions
err = linkSessionCert(ids, idc)
if err != nil {
log.Fatal(fmt.Sprintf("Could not link Certs and Session into DB failed: %q", err))
}
// Launch go routine to create public keys
}
}
// linkSessionCert creates the link between a session and its certificates
func linkSessionCert(ids int64, idc []string) error {
for _, i := range idc {
q := `INSERT INTO "many_sessionRecord_has_many_certificate" ("id_sessionRecord", "hash_certificate") VALUES ($1, $2)`
_, err := db.Query(q, ids, i)
if err != nil {
return err
}
}
return nil
}
// buildChain attempts to rearrange certificate as a chain of trust from a sessionRecord (that
// contains a slice of certificate). If the chain of trust is build successfully
// it marked as valid, If not root is found or if the chain is broken, it
// does not touch the original slice and mark the chain as invalid.
func buildChain(s *sessionRecord) (*sessionRecord) {
certChain := make([]certMapElm, 0)
// First we find the leaf
for _, c := range s.Certificates {
fmt.Println(c.Certificate.Issuer.String())
fmt.Println(c.Certificate.Subject.String())
fmt.Println(c.Certificate.Subject.String() == c.Certificate.Issuer.String())
if !c.Certificate.IsCA {
certChain = append(certChain, c)
}
}
// Find the parent of each certificate
for _, _ = range s.Certificates {
for i, _ := range s.Certificates {
if s.Certificates[i].Certificate.Subject.String() == certChain[len(certChain)-1].Issuer.String() {
certChain = append(certChain, s.Certificates[i])
}
}
}
// Write the new chain
if len(certChain) == len(s.Certificates) {
cstr := make([]string, 0)
for i := len(certChain) - 1; i >= 0; i-- {
certChain[i].chain.isValid = true
cstr = append(cstr, certChain[i].CertHash)
certChain[i].chain.s = strings.Join(cstr, ".")
}
tmp := s
tmp.Certificates = certChain
return tmp
}
return s
}
func insertCertificate(c certMapElm) (string, error) {
q := `INSERT INTO "certificate" (hash, "is_CA", issuer, subject, cert_chain, is_valid_chain, file_path) VALUES ($1, $2, $3, $4, $5, $6, $7) RETURNING hash`
var hash string
err := db.QueryRow(q, c.CertHash, c.Certificate.IsCA, c.Certificate.Issuer.String(), c.Certificate.Subject.String(), c.chain.s, c.chain.isValid, getFullPath(c.CertHash)).Scan(&hash)
if err != nil {
return "", err
}
return hash, nil
}
// getFullPath takes a certificate's hash and return the full path to
// its location on disk
func getFullPath(h string) (string) {
return "TODO PATH"
}
func insertCertificates(s *sessionRecord) ([]string, error) {
var inserted []string
for _, certificate := range s.Certificates {
idc, err := insertCertificate(certificate)
if err != nil {
return inserted, err
}
inserted = append(inserted, idc)
}
return inserted, nil
}
func insertSession(s *sessionRecord) (int64, error) {
q := `INSERT INTO "sessionRecord" (dst_ip, src_ip, dst_port, src_port, timestamp) VALUES ($1, $2, $3, $4, $5) RETURNING id`
var id int64
err := db.QueryRow(q, s.ServerIP, s.ClientIP, s.ServerPort, s.ClientPort, s.Timestamp).Scan(&id)
if err != nil {
return 0, err
}
return id, nil
}
func initRedis() {
err := errors.New("")
cr, err = redis.Dial("tcp", ":6380", redis.DialDatabase(2))
if err != nil {
panic(err)
}
}
func initDB() {
connStr := "user=postgres password=postgres dbname=new_database"
err := errors.New("")
db, err = sql.Open("postgres", connStr)
if err != nil {
panic(err)
}
}
// String returns a string that describes a TLSSession
func (t *sessionRecord) String() string {
var buf bytes.Buffer
buf.WriteString(fmt.Sprintf("---------------SESSION START-------------------\n"))
buf.WriteString(fmt.Sprintf("Time: %v\n", t.Timestamp))
buf.WriteString(fmt.Sprintf("Client: %v:%v\n", t.ClientIP, t.ClientPort))
buf.WriteString(fmt.Sprintf("Server: %v:%v\n", t.ServerIP, t.ServerPort))
buf.WriteString(fmt.Sprintf("TLSH: %q\n", t.TLSH))
buf.WriteString(fmt.Sprintf("ja3: %q\n", t.JA3))
buf.WriteString(fmt.Sprintf("ja3 Digest: %q\n", t.JA3Digest))
buf.WriteString(fmt.Sprintf("ja3s: %q\n", t.JA3S))
buf.WriteString(fmt.Sprintf("ja3s Digest: %q\n", t.JA3SDigest))
for _, certMe := range t.Certificates {
buf.WriteString(fmt.Sprintf("Certificate Issuer: %q\n", certMe.Certificate.Issuer))
buf.WriteString(fmt.Sprintf("Certificate Subject: %q\n", certMe.Certificate.Subject))
buf.WriteString(fmt.Sprintf("Certificate is CA: %t\n", certMe.Certificate.IsCA))
buf.WriteString(fmt.Sprintf("Certificate SHA256: %q\n", certMe.CertHash))
}
buf.WriteString(fmt.Sprintf("---------------SESSION END--------------------\n"))
return buf.String()
}

View File

@ -1,66 +1,254 @@
--
-- PostgreSQL database dump
--
-- Database generated with pgModeler (PostgreSQL Database Modeler).
-- pgModeler version: 0.9.1-beta
-- PostgreSQL version: 10.0
-- Project Site: pgmodeler.com.br
-- Model Author: ---
-- Dumped from database version 10.6 (Ubuntu 10.6-0ubuntu0.18.04.1)
-- Dumped by pg_dump version 10.6 (Ubuntu 10.6-0ubuntu0.18.04.1)
SET statement_timeout = 0;
SET lock_timeout = 0;
SET idle_in_transaction_session_timeout = 0;
SET client_encoding = 'UTF8';
SET standard_conforming_strings = on;
SELECT pg_catalog.set_config('search_path', '', false);
SET check_function_bodies = false;
SET client_min_messages = warning;
SET row_security = off;
-- Database creation must be done outside an multicommand file.
-- These commands were put in this file only for convenience.
-- -- object: new_database | type: DATABASE --
-- -- DROP DATABASE IF EXISTS new_database;
-- CREATE DATABASE new_database
-- ;
-- -- ddl-end --
--
SET default_tablespace = '';
-- object: ltree | type: EXTENSION --
-- DROP EXTENSION IF EXISTS ltree CASCADE;
CREATE EXTENSION ltree
WITH SCHEMA public;
-- ddl-end --
SET default_with_oids = false;
-- object: hstore | type: EXTENSION --
-- DROP EXTENSION IF EXISTS hstore CASCADE;
CREATE EXTENSION hstore
WITH SCHEMA public;
-- ddl-end --
--
-- Name: sessions; Type: TABLE; Schema: public; Owner: postgres
--
-- object: public.public_key | type: TABLE --
-- DROP TABLE IF EXISTS public.public_key CASCADE;
CREATE TABLE public.public_key(
hash bytea NOT NULL,
type text NOT NULL,
modulus text,
exponent smallint,
modules_size smallint,
CONSTRAINT public_key_pk PRIMARY KEY (hash)
CREATE TABLE public.sessions (
data jsonb,
id integer NOT NULL
);
-- ddl-end --
ALTER TABLE public.public_key OWNER TO postgres;
-- ddl-end --
-- object: public.certificate | type: TABLE --
-- DROP TABLE IF EXISTS public.certificate CASCADE;
CREATE TABLE public.certificate(
file_path varchar(4096) NOT NULL,
issuer text,
cert_chain public.ltree,
subject text,
hash bytea NOT NULL,
"is_CA" bool NOT NULL DEFAULT false,
is_valid_chain bool NOT NULL DEFAULT false,
CONSTRAINT certificate_pk PRIMARY KEY (hash)
);
-- ddl-end --
ALTER TABLE public.certificate OWNER TO postgres;
-- ddl-end --
-- object: public.many_certificate_has_many_public_key | type: TABLE --
-- DROP TABLE IF EXISTS public.many_certificate_has_many_public_key CASCADE;
CREATE TABLE public.many_certificate_has_many_public_key(
hash_certificate bytea NOT NULL,
hash_public_key bytea NOT NULL,
CONSTRAINT many_certificate_has_many_public_key_pk PRIMARY KEY (hash_certificate,hash_public_key)
);
-- ddl-end --
-- object: certificate_fk | type: CONSTRAINT --
-- ALTER TABLE public.many_certificate_has_many_public_key DROP CONSTRAINT IF EXISTS certificate_fk CASCADE;
ALTER TABLE public.many_certificate_has_many_public_key ADD CONSTRAINT certificate_fk FOREIGN KEY (hash_certificate)
REFERENCES public.certificate (hash) MATCH FULL
ON DELETE RESTRICT ON UPDATE CASCADE;
-- ddl-end --
-- object: public_key_fk | type: CONSTRAINT --
-- ALTER TABLE public.many_certificate_has_many_public_key DROP CONSTRAINT IF EXISTS public_key_fk CASCADE;
ALTER TABLE public.many_certificate_has_many_public_key ADD CONSTRAINT public_key_fk FOREIGN KEY (hash_public_key)
REFERENCES public.public_key (hash) MATCH FULL
ON DELETE RESTRICT ON UPDATE CASCADE;
-- ddl-end --
-- object: public."sessionRecord" | type: TABLE --
-- DROP TABLE IF EXISTS public."sessionRecord" CASCADE;
CREATE TABLE public."sessionRecord"(
id bigserial NOT NULL,
dst_ip inet NOT NULL,
src_ip inet NOT NULL,
dst_port int4 NOT NULL,
src_port int4 NOT NULL,
hash_ja3 bytea,
"timestamp" time(0) with time zone,
CONSTRAINT "sessionRecord_pk" PRIMARY KEY (id)
);
-- ddl-end --
ALTER TABLE public."sessionRecord" OWNER TO postgres;
-- ddl-end --
-- object: public.ja3 | type: TABLE --
-- DROP TABLE IF EXISTS public.ja3 CASCADE;
CREATE TABLE public.ja3(
hash bytea NOT NULL,
raw text,
type smallint NOT NULL,
CONSTRAINT j3a_pk PRIMARY KEY (hash)
);
-- ddl-end --
ALTER TABLE public.ja3 OWNER TO postgres;
-- ddl-end --
-- object: ja3_fk | type: CONSTRAINT --
-- ALTER TABLE public."sessionRecord" DROP CONSTRAINT IF EXISTS ja3_fk CASCADE;
ALTER TABLE public."sessionRecord" ADD CONSTRAINT ja3_fk FOREIGN KEY (hash_ja3)
REFERENCES public.ja3 (hash) MATCH FULL
ON DELETE SET NULL ON UPDATE CASCADE;
-- ddl-end --
-- object: public."many_sessionRecord_has_many_certificate" | type: TABLE --
-- DROP TABLE IF EXISTS public."many_sessionRecord_has_many_certificate" CASCADE;
CREATE TABLE public."many_sessionRecord_has_many_certificate"(
"id_sessionRecord" bigint NOT NULL,
hash_certificate bytea NOT NULL,
CONSTRAINT "many_sessionRecord_has_many_certificate_pk" PRIMARY KEY ("id_sessionRecord",hash_certificate)
);
-- ddl-end --
-- object: "sessionRecord_fk" | type: CONSTRAINT --
-- ALTER TABLE public."many_sessionRecord_has_many_certificate" DROP CONSTRAINT IF EXISTS "sessionRecord_fk" CASCADE;
ALTER TABLE public."many_sessionRecord_has_many_certificate" ADD CONSTRAINT "sessionRecord_fk" FOREIGN KEY ("id_sessionRecord")
REFERENCES public."sessionRecord" (id) MATCH FULL
ON DELETE RESTRICT ON UPDATE CASCADE;
-- ddl-end --
-- object: certificate_fk | type: CONSTRAINT --
-- ALTER TABLE public."many_sessionRecord_has_many_certificate" DROP CONSTRAINT IF EXISTS certificate_fk CASCADE;
ALTER TABLE public."many_sessionRecord_has_many_certificate" ADD CONSTRAINT certificate_fk FOREIGN KEY (hash_certificate)
REFERENCES public.certificate (hash) MATCH FULL
ON DELETE RESTRICT ON UPDATE CASCADE;
-- ddl-end --
-- object: public.fuzzy_hash | type: TABLE --
-- DROP TABLE IF EXISTS public.fuzzy_hash CASCADE;
CREATE TABLE public.fuzzy_hash(
id bigserial NOT NULL,
type text NOT NULL,
value public.hstore NOT NULL,
hash_ja3 bytea,
CONSTRAINT fuzzy_hash_pk PRIMARY KEY (id)
);
-- ddl-end --
ALTER TABLE public.fuzzy_hash OWNER TO postgres;
-- ddl-end --
-- object: public.software | type: TABLE --
-- DROP TABLE IF EXISTS public.software CASCADE;
CREATE TABLE public.software(
id serial NOT NULL,
name text NOT NULL,
version text,
CONSTRAINT software_pk PRIMARY KEY (id)
);
-- ddl-end --
ALTER TABLE public.software OWNER TO postgres;
-- ddl-end --
-- object: public.annotation | type: TABLE --
-- DROP TABLE IF EXISTS public.annotation CASCADE;
CREATE TABLE public.annotation(
id serial NOT NULL,
hash_ja3 bytea,
confidence smallint,
id_software integer,
CONSTRAINT annotation_pk PRIMARY KEY (id)
);
-- ddl-end --
ALTER TABLE public.annotation OWNER TO postgres;
-- ddl-end --
-- object: ja3_fk | type: CONSTRAINT --
-- ALTER TABLE public.annotation DROP CONSTRAINT IF EXISTS ja3_fk CASCADE;
ALTER TABLE public.annotation ADD CONSTRAINT ja3_fk FOREIGN KEY (hash_ja3)
REFERENCES public.ja3 (hash) MATCH FULL
ON DELETE SET NULL ON UPDATE CASCADE;
-- ddl-end --
-- object: software_fk | type: CONSTRAINT --
-- ALTER TABLE public.annotation DROP CONSTRAINT IF EXISTS software_fk CASCADE;
ALTER TABLE public.annotation ADD CONSTRAINT software_fk FOREIGN KEY (id_software)
REFERENCES public.software (id) MATCH FULL
ON DELETE SET NULL ON UPDATE CASCADE;
-- ddl-end --
-- object: ja3_trie | type: INDEX --
-- DROP INDEX IF EXISTS public.ja3_trie CASCADE;
CREATE INDEX CONCURRENTLY ja3_trie ON public.ja3
USING spgist
(
raw
);
-- ddl-end --
-- object: hash_index | type: INDEX --
-- DROP INDEX IF EXISTS public.hash_index CASCADE;
CREATE INDEX hash_index ON public.certificate
USING btree
(
hash
);
-- ddl-end --
-- object: pk_index | type: INDEX --
-- DROP INDEX IF EXISTS public.pk_index CASCADE;
CREATE INDEX pk_index ON public.public_key
USING btree
(
hash
);
-- ddl-end --
-- object: dst_index | type: INDEX --
-- DROP INDEX IF EXISTS public.dst_index CASCADE;
CREATE INDEX dst_index ON public."sessionRecord"
USING btree
(
dst_ip
);
-- ddl-end --
-- object: path_index | type: INDEX --
-- DROP INDEX IF EXISTS public.path_index CASCADE;
CREATE INDEX path_index ON public.certificate
USING gist
(
cert_chain
)
WITH (BUFFERING = ON);
-- ddl-end --
-- object: ja3_fk | type: CONSTRAINT --
-- ALTER TABLE public.fuzzy_hash DROP CONSTRAINT IF EXISTS ja3_fk CASCADE;
ALTER TABLE public.fuzzy_hash ADD CONSTRAINT ja3_fk FOREIGN KEY (hash_ja3)
REFERENCES public.ja3 (hash) MATCH FULL
ON DELETE SET NULL ON UPDATE CASCADE;
-- ddl-end --
ALTER TABLE public.sessions OWNER TO postgres;
--
-- Name: sessions_id_seq; Type: SEQUENCE; Schema: public; Owner: postgres
--
CREATE SEQUENCE public.sessions_id_seq
AS integer
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
ALTER TABLE public.sessions_id_seq OWNER TO postgres;
--
-- Name: sessions_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: postgres
--
ALTER SEQUENCE public.sessions_id_seq OWNED BY public.sessions.id;
--
-- Name: sessions id; Type: DEFAULT; Schema: public; Owner: postgres
--
ALTER TABLE ONLY public.sessions ALTER COLUMN id SET DEFAULT nextval('public.sessions_id_seq'::regclass);
--
-- PostgreSQL database dump complete
--