From 2683ab692993c4db9c07d6fb1f4231e25c20fe01 Mon Sep 17 00:00:00 2001 From: Jean-Louis Huynen Date: Wed, 10 Apr 2019 16:04:54 +0200 Subject: [PATCH] wip --- main.go | 223 ++++++++++++++++++++++++++++++++++--------------- passivessl.sql | 3 +- 2 files changed, 156 insertions(+), 70 deletions(-) diff --git a/main.go b/main.go index 1285d82..04936f9 100644 --- a/main.go +++ b/main.go @@ -46,6 +46,7 @@ type sessionRecord struct { type chain struct { isValid bool + isSS bool s string } @@ -77,38 +78,72 @@ func main() { } } else { - jsonPath = "./test.json" - - // read corresponding json file - dat, err := ioutil.ReadFile(jsonPath) + var path []string + path = append(path, "./sessions") + path = append(path, "") + files, err := ioutil.ReadDir(path[0]) if err != nil { log.Fatal(err) } - // Unmarshal JSON file - s := sessionRecord{} - _ = json.Unmarshal([]byte(dat), &s) - // Insert Session - ids, err := insertSession(&s) - if err != nil { - log.Fatal(fmt.Sprintf("Insert Sessions into DB failed: %q", err)) - } - // Attempt to roughly build a chain of trust - session := buildChain(&s) + for _, f := range files { - // Insert Certificates - idc, err := insertCertificates(session) - if err != nil { - log.Fatal(fmt.Sprintf("Insert Certificate into DB failed: %q", err)) - } - // Create the relationship between certificates and sessions - err = linkSessionCerts(ids, idc) - if err != nil { - log.Fatal(fmt.Sprintf("Could not link Certs and Session into DB: %q", err)) + path[1] = f.Name() + jsonPath = strings.Join(path, "/") + + // read corresponding json file + dat, err := ioutil.ReadFile(jsonPath) + if err != nil { + log.Fatal(err) + } + // Unmarshal JSON file + s := sessionRecord{} + _ = json.Unmarshal([]byte(dat), &s) + + if len(s.Certificates) > 0 { + // Insert Session + ids, err := insertSession(&s) + if err != nil { + log.Fatal(fmt.Sprintf("Insert Sessions into DB failed: %q", err)) + } + // Attempt to roughly build a chain of trust + session := buildChain(&s) + + // Insert Certificates + idc, err := insertCertificates(session) + if err != nil { + log.Fatal(fmt.Sprintf("Insert Certificate into DB failed: %q", err)) + } + // Create the relationship between certificates and sessions + err = linkSessionCerts(ids, idc) + if err != nil { + log.Fatal(fmt.Sprintf("Could not link Certs and Session into DB: %q", err)) + } + + // Create ja3* records + err = insertJA3(session) + if err != nil { + log.Fatal(fmt.Sprintf("Could not insert JA3 into DB: %q", err)) + } + } } } } +func insertJA3(s *sessionRecord) error { + q := `INSERT INTO "ja3" (hash, raw, type) VALUES ($1, $2, $3) ON CONFLICT DO NOTHING` + _, err := db.Exec(q, s.JA3Digest, s.JA3, "ja3") + if err != nil { + return err + } + q = `INSERT INTO "ja3" (hash, raw, type) VALUES ($1, $2, $3) ON CONFLICT DO NOTHING` + _, err = db.Exec(q, s.JA3SDigest, s.JA3S, "ja3s") + if err != nil { + return err + } + return nil +} + // insertPublicKeys insert each public key of each certificate of a session func insertPublicKey(c x509.Certificate) (string, error) { pub, err := x509.ParsePKIXPublicKey(c.RawSubjectPublicKeyInfo) @@ -119,22 +154,22 @@ func insertPublicKey(c x509.Certificate) (string, error) { switch pub := pub.(type) { case *rsa.PublicKey: - q := `INSERT INTO "public_key" (hash, type, modulus, exponent, modulus_size) VALUES ($1, $2, $3, $4, $5)` - _, err := db.Query(q, hash, "RSA", (*BigNumber)(pub.N), pub.E, pub.Size()) + q := `INSERT INTO "public_key" (hash, type, modulus, exponent, modulus_size) VALUES ($1, $2, $3, $4, $5) ON CONFLICT DO NOTHING` + _, err := db.Exec(q, hash, "RSA", (*BigNumber)(pub.N), pub.E, pub.Size()) if err != nil { - return hash, nil + return hash, err } case *dsa.PublicKey: - q := `INSERT INTO "public_key" (hash, type, "G", "P", "Q", "Y") VALUES ($1, $2, $3, $4, $5, $6)` - _, err := db.Query(q, hash, "DSA", (*BigNumber)(pub.Parameters.G), (*BigNumber)(pub.Parameters.P), (*BigNumber)(pub.Parameters.Q), (*BigNumber)(pub.Y)) + q := `INSERT INTO "public_key" (hash, type, "G", "P", "Q", "Y") VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT DO NOTHING` + _, err := db.Exec(q, hash, "DSA", (*BigNumber)(pub.Parameters.G), (*BigNumber)(pub.Parameters.P), (*BigNumber)(pub.Parameters.Q), (*BigNumber)(pub.Y)) if err != nil { - return hash, nil + return hash, err } case *ecdsa.PublicKey: - q := `INSERT INTO "public_key" (hash, type, "Y", "X", "P", "N", "B", "bitsize", "Gx", "Gy") VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)` - _, err := db.Query(q, hash, "ECDSA", pub.Y, pub.X, pub.Curve.Params().P, pub.Curve.Params().N, pub.Curve.Params().B, pub.Curve.Params().BitSize, pub.Curve.Params().Gx, pub.Curve.Params().Gy) + q := `INSERT INTO "public_key" (hash, type, "Y", "X", "P", "N", "B", "bitsize", "Gx", "Gy", "curve_name") VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) ON CONFLICT DO NOTHING` + _, err := db.Exec(q, hash, "ECDSA", (*BigNumber)(pub.Y), (*BigNumber)(pub.X), (*BigNumber)(pub.Curve.Params().P), (*BigNumber)(pub.Curve.Params().N), (*BigNumber)(pub.Curve.Params().B), pub.Curve.Params().BitSize, (*BigNumber)(pub.Curve.Params().Gx), (*BigNumber)(pub.Curve.Params().Gy), pub.Params().Name) if err != nil { - return hash, nil + return hash, err } default: return hash, fmt.Errorf("PKIx: could not determine the type of key %g", pub) @@ -149,8 +184,8 @@ func (bn *BigNumber) Value() (driver.Value, error) { // linkSessionCerts creates the link between a session and its certificates func linkSessionCerts(ids int64, idc []string) error { for _, i := range idc { - q := `INSERT INTO "many_sessionRecord_has_many_certificate" ("id_sessionRecord", "hash_certificate") VALUES ($1, $2)` - _, err := db.Query(q, ids, i) + q := `INSERT INTO "many_sessionRecord_has_many_certificate" ("id_sessionRecord", "hash_certificate") VALUES ($1, $2) ON CONFLICT DO NOTHING ` + _, err := db.Exec(q, ids, i) if err != nil { return err } @@ -164,57 +199,107 @@ func linkSessionCerts(ids int64, idc []string) error { // does not touch the original slice and mark the chain as invalid. func buildChain(s *sessionRecord) *sessionRecord { certChain := make([]certMapElm, 0) + // First find if there are any duplicates certificate + dedup := unique(s.Certificates) + deduplen := len(dedup) - // First we find the leaf - for _, c := range s.Certificates { - fmt.Println(c.Certificate.Issuer.String()) - fmt.Println(c.Certificate.Subject.String()) - fmt.Println(c.Certificate.Subject.String() == c.Certificate.Issuer.String()) - if !c.Certificate.IsCA { - certChain = append(certChain, c) - } - } - // Find the parent of each certificate - for _, _ = range s.Certificates { - for i, _ := range s.Certificates { - if s.Certificates[i].Certificate.Subject.String() == certChain[len(certChain)-1].Issuer.String() { - certChain = append(certChain, s.Certificates[i]) + if deduplen > 1 { + // Then we find the leaf + removed := 0 + for i, c := range dedup { + if !c.Certificate.IsCA { + certChain = append(certChain, c) + // Remove this element from the list + dedup = append(dedup[:i-removed], dedup[i+1-removed:]...) + removed++ } } - } - // Write the new chain - if len(certChain) == len(s.Certificates) { - cstr := make([]string, 0) - for i := len(certChain) - 1; i >= 0; i-- { - certChain[i].chain.isValid = true - cstr = append(cstr, certChain[i].CertHash) - certChain[i].chain.s = strings.Join(cstr, ".") + + cursor := len(certChain) + // Find the parent of each certificate + for i := 0; i < cursor; i++ { + p, success, isSS := findParent(dedup, certChain[i]) + // if we found a root, no need to go any further + if isSS { + p.chain.isSS = true + certChain = append(certChain, p) + break + } + if success { + p.chain.isSS = false + certChain = append(certChain, p) + } + cursor = len(certChain) } - tmp := s - tmp.Certificates = certChain - return tmp + + // Write the new chain if it's valid + if len(certChain) >= deduplen-1 { + cstr := make([]string, 0) + for i := len(certChain) - 1; i >= 0; i-- { + certChain[i].chain.isValid = true + if !certChain[i].chain.isSS { + cstr = append(cstr, certChain[i].CertHash) + certChain[i].chain.s = strings.Join(cstr, ".") + } + } + tmp := s + tmp.Certificates = certChain + return tmp + } + // Only one cert in the chain + } else { + s.Certificates[0].chain = chain{true, s.Certificates[0].Certificate.Issuer.String() == s.Certificates[0].Certificate.Subject.String(), s.Certificates[0].CertHash} } + return s } +func findParent(dedup []certMapElm, c certMapElm) (certMapElm, bool, bool) { + // A Root or SSC signs itself + if c.Certificate.Subject.String() == c.Certificate.Issuer.String() { + return c, true, true + } + + // A leaf or a node has a parent + for _, candidate := range dedup { + if candidate.Certificate.Subject.String() == c.Certificate.Issuer.String() { + return candidate, true, false + } + } + + return c, false, false +} + +func unique(s []certMapElm) []certMapElm { + keys := make(map[string]bool) + var list []certMapElm + for _, entry := range s { + if _, value := keys[entry.CertHash]; !value { + keys[entry.CertHash] = true + list = append(list, entry) + } + } + return list +} + func insertCertificate(c certMapElm) (string, error) { - q := `INSERT INTO "certificate" (hash, "is_CA", issuer, subject, cert_chain, is_valid_chain, file_path) VALUES ($1, $2, $3, $4, $5, $6, $7) RETURNING hash` - var hash string - err := db.QueryRow(q, c.CertHash, c.Certificate.IsCA, c.Certificate.Issuer.String(), c.Certificate.Subject.String(), c.chain.s, c.chain.isValid, getFullPath(c.CertHash)).Scan(&hash) + q := `INSERT INTO "certificate" (hash, "is_CA", "is_SS", issuer, subject, cert_chain, is_valid_chain, file_path) VALUES ($1, $2, $3, $4, $5, $6, $7, $8) ON CONFLICT DO NOTHING` + _, err := db.Exec(q, c.CertHash, c.Certificate.IsCA, c.chain.isSS, c.Certificate.Issuer.String(), c.Certificate.Subject.String(), c.chain.s, c.chain.isValid, getFullPath(c.CertHash)) if err != nil { - return hash, err + return c.CertHash, err } key, err := insertPublicKey(*c.Certificate) if err != nil { - return hash, err + return c.CertHash, err } - - q = `INSERT INTO "many_certificate_has_many_public_key" ("hash_certificate", "hash_public_key") VALUES ($1, $2)` - _, err = db.Query(q, hash, key) + fmt.Println(c.CertHash) + fmt.Println(key) + q = `INSERT INTO "many_certificate_has_many_public_key" ("hash_certificate", "hash_public_key") VALUES ($1, $2) ON CONFLICT DO NOTHING ` + _, err = db.Exec(q, c.CertHash, key) if err != nil { - return hash, err + return c.CertHash, err } - return hash, nil + return c.CertHash, nil } // getFullPath takes a certificate's hash and return the full path to diff --git a/passivessl.sql b/passivessl.sql index 63423f0..069c94b 100644 --- a/passivessl.sql +++ b/passivessl.sql @@ -64,6 +64,7 @@ CREATE TABLE public.certificate( is_valid_chain bool NOT NULL DEFAULT false, "notBefore" time, "notAfter" time, + "isSS" bool NOT NULL DEFAULT false, CONSTRAINT certificate_pk PRIMARY KEY (hash) ); @@ -117,7 +118,7 @@ ALTER TABLE public."sessionRecord" OWNER TO postgres; CREATE TABLE public.ja3( hash bytea NOT NULL, raw text, - type smallint NOT NULL, + type varchar(16) NOT NULL, CONSTRAINT j3a_pk PRIMARY KEY (hash) );