chg: [sql] remove unnecessary not null constraint

chg: [sql] remove plpython3u clutter
chg: [doc] not only pgsql10
2020-11-18 16:16:46 +01:00 · 2020-11-18 15:21:11 +01:00 · 2020-11-18 14:47:14 +01:00 · 2020-01-10 13:14:09 +01:00 · 2019-11-25 10:21:19 +01:00 · 2019-10-14 11:15:23 +02:00
8 changed files with 939 additions and 308 deletions
--- a/.gitignore
+++ b/.gitignore
@ -4,6 +4,7 @@
 *.dll
 *.so
 *.dylib
+analyzer-d4-passivessl

 # Test binary, build with `go test -c`
 *.test
--- a/README.md
+++ b/README.md
@ -4,7 +4,7 @@ analyzer-d4-passivessl fetch a redis feed of certificate and TLS sessions and ma
 ```bash
 go get github.com/gomodule/redigo/redis
 go get github.com/lib/pq
-sudo apt install postgresql-plpython3-10
+sudo apt install postgresql-plpython3-[your psql version]
 ```

 # Config
--- a/go.mod
+++ b/go.mod
@ -0,0 +1,12 @@
+module github.com/D4-project/analyzer-d4-passivessl
+
+go 1.13
+
+require (
+	github.com/D4-project/d4-golang-utils v0.0.0-20200108150548-740f16240125
+	github.com/gallypette/certificate-transparency-go v1.0.21
+	github.com/gomodule/redigo v2.0.0+incompatible
+	github.com/google/certificate-transparency-go v1.1.0 // indirect
+	github.com/lib/pq v1.3.0
+	golang.org/x/crypto v0.0.0-20200109152110-61a87790db17 // indirect
+)
--- a/go.sum
+++ b/go.sum
@ -0,0 +1,323 @@
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
+contrib.go.opencensus.io/exporter/stackdriver v0.12.1/go.mod h1:iwB6wGarfphGGe/e5CWqyUk/cLzKnWsOKPVW3no6OTw=
+contrib.go.opencensus.io/resource v0.1.1/go.mod h1:F361eGI91LCmW1I/Saf+rX0+OFcigGlFvXwEGEnkRLA=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/D4-project/d4-golang-utils v0.0.0-20200108150548-740f16240125 h1:iv+hcdT+M0XJIDEoCtvk9HVvI8PgvbQNBtbEfCczCRI=
+github.com/D4-project/d4-golang-utils v0.0.0-20200108150548-740f16240125/go.mod h1:2rq8KBQnNNDocwc/49cnpaqoQA/komoSHKom7ynvqJc=
+github.com/OpenPeeDeeP/depguard v1.0.0/go.mod h1:7/4sitnI9YlQgTLLk734QlzXT8DuHVnAyztLplQjk+o=
+github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
+github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
+github.com/aws/aws-sdk-go v1.19.18/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
+github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
+github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
+github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
+github.com/census-instrumentation/opencensus-proto v0.2.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
+github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
+github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
+github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
+github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
+github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
+github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
+github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
+github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
+github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
+github.com/fatih/color v1.6.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
+github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/gallypette/certificate-transparency-go v1.0.21 h1:Bcpz95r7OMUMen6knu3Pyd7g1RNFE5k5mft7yeQ0aQI=
+github.com/gallypette/certificate-transparency-go v1.0.21/go.mod h1:BKLYYnvX0ezkK7zk0rD7pVvFSUOg6wmhKi9lRN/tgU0=
+github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
+github.com/go-critic/go-critic v0.3.5-0.20190526074819-1df300866540/go.mod h1:+sE8vrLDS2M0pZkBk0wy6+nLdKexVDrl/jBqQOTDThA=
+github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-lintpack/lintpack v0.5.2/go.mod h1:NwZuYi2nUHho8XEIZ6SIxihrnPoqBTDqfpXvXAN0sXM=
+github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
+github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8=
+github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
+github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/go-toolsmith/astcast v1.0.0/go.mod h1:mt2OdQTeAQcY4DQgPSArJjHCcOwlX+Wl/kwN+LbLGQ4=
+github.com/go-toolsmith/astcopy v1.0.0/go.mod h1:vrgyG+5Bxrnz4MZWPF+pI4R8h3qKRjjyvV/DSez4WVQ=
+github.com/go-toolsmith/astequal v0.0.0-20180903214952-dcb477bfacd6/go.mod h1:H+xSiq0+LtiDC11+h1G32h7Of5O3CYFJ99GVbS5lDKY=
+github.com/go-toolsmith/astequal v1.0.0/go.mod h1:H+xSiq0+LtiDC11+h1G32h7Of5O3CYFJ99GVbS5lDKY=
+github.com/go-toolsmith/astfmt v0.0.0-20180903215011-8f8ee99c3086/go.mod h1:mP93XdblcopXwlyN4X4uodxXQhldPGZbcEJIimQHrkg=
+github.com/go-toolsmith/astfmt v1.0.0/go.mod h1:cnWmsOAuq4jJY6Ct5YWlVLmcmLMn1JUPuQIHCY7CJDw=
+github.com/go-toolsmith/astinfo v0.0.0-20180906194353-9809ff7efb21/go.mod h1:dDStQCHtmZpYOmjRP/8gHHnCCch3Zz3oEgCdZVdtweU=
+github.com/go-toolsmith/astp v0.0.0-20180903215135-0af7e3c24f30/go.mod h1:SV2ur98SGypH1UjcPpCatrV5hPazG6+IfNHbkDXBRrk=
+github.com/go-toolsmith/astp v1.0.0/go.mod h1:RSyrtpVlfTFGDYRbrjyWP1pYu//tSFcvdYrA8meBmLI=
+github.com/go-toolsmith/pkgload v0.0.0-20181119091011-e9e65178eee8/go.mod h1:WoMrjiy4zvdS+Bg6z9jZH82QXwkcgCBX6nOfnmdaHks=
+github.com/go-toolsmith/pkgload v1.0.0/go.mod h1:5eFArkbO80v7Z0kdngIxsRXRMTaX4Ilcwuh3clNrQJc=
+github.com/go-toolsmith/strparse v1.0.0/go.mod h1:YI2nUKP9YGZnL/L1/DLFBfixrcjslWct4wyljWhSRy8=
+github.com/go-toolsmith/typep v1.0.0/go.mod h1:JSQCQMUPdRlMZFswiq3TGpNp1GMktqkR2Ns5AIQkATU=
+github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
+github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
+github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/mock v1.0.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2/go.mod h1:k9Qvh+8juN+UKMCS/3jFtGICgW8O96FVaZsaxdzDkR4=
+github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a/go.mod h1:ryS0uhF+x9jgbj/N71xsEqODy9BN81/GonCZiOzirOk=
+github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6/go.mod h1:DbHgvLiFKX1Sh2T1w8Q/h4NAI8MHIpzCdnBUDTXU3I0=
+github.com/golangci/go-misc v0.0.0-20180628070357-927a3d87b613/go.mod h1:SyvUF2NxV+sN8upjjeVYr5W7tyxaT1JVtvhKhOn2ii8=
+github.com/golangci/go-tools v0.0.0-20190318055746-e32c54105b7c/go.mod h1:unzUULGw35sjyOYjUt0jMTXqHlZPpPc6e+xfO4cd6mM=
+github.com/golangci/goconst v0.0.0-20180610141641-041c5f2b40f3/go.mod h1:JXrF4TWy4tXYn62/9x8Wm/K/dm06p8tCKwFRDPZG/1o=
+github.com/golangci/gocyclo v0.0.0-20180528134321-2becd97e67ee/go.mod h1:ozx7R9SIwqmqf5pRP90DhR2Oay2UIjGuKheCBCNwAYU=
+github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98/go.mod h1:9qCChq59u/eW8im404Q2WWTrnBUQKjpNYKMbU4M7EFU=
+github.com/golangci/golangci-lint v1.17.2-0.20190910081718-bad04bb7378f/go.mod h1:kaqo8l0OZKYPtjNmG4z4HrWLgcYNIJ9B9q3LWri9uLg=
+github.com/golangci/gosec v0.0.0-20190211064107-66fb7fc33547/go.mod h1:0qUabqiIQgfmlAmulqxyiGkkyF6/tOGSnY2cnPVwrzU=
+github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc/go.mod h1:e5tpTHCfVze+7EpLEozzMB3eafxo2KT5veNg1k6byQU=
+github.com/golangci/lint-1 v0.0.0-20190420132249-ee948d087217/go.mod h1:66R6K6P6VWk9I95jvqGxkqJxVWGFy9XlDwLwVz1RCFg=
+github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca/go.mod h1:tvlJhZqDe4LMs4ZHD0oMUlt9G2LWuDGoisJTBzLMV9o=
+github.com/golangci/misspell v0.0.0-20180809174111-950f5d19e770/go.mod h1:dEbvlSfYbMQDtrpRMQU675gSDLDNa8sCPPChZ7PhiVA=
+github.com/golangci/prealloc v0.0.0-20180630174525-215b22d4de21/go.mod h1:tf5+bzsHdTM0bsB7+8mt0GUMvjCgwLpTapNZHU8AajI=
+github.com/golangci/revgrep v0.0.0-20180526074752-d9c87f5ffaf0/go.mod h1:qOQCunEYvmd/TLamH+7LlVccLvUH5kZNhbCgTHoBbp4=
+github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4/go.mod h1:Izgrg8RkN3rCIMLGE9CyYmU9pY2Jer6DgANEnZ/L/cQ=
+github.com/gomodule/redigo v2.0.0+incompatible h1:K/R+8tc58AaqLkqG2Ol3Qk+DR/TlNuhuh457pBFPtt0=
+github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4=
+github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/certificate-transparency-go v1.1.0 h1:10MlrYzh5wfkToxWI4yJzffsxLfxcEDlOATMx/V9Kzw=
+github.com/google/certificate-transparency-go v1.1.0/go.mod h1:i+Q7XY+ArBveOUT36jiHGfuSK1fHICIg6sUkRxPAbCs=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
+github.com/google/monologue v0.0.0-20190606152607-4b11a32b5934/go.mod h1:6NTfaQoUpg5QmPsCUWLR3ig33FHrKXhTtWzF0DVdmuk=
+github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/trillian v1.2.2-0.20190612132142-05461f4df60a/go.mod h1:YPmUVn5NGwgnDUgqlVyFGMTgaWlnSvH7W5p+NdOG8UA=
+github.com/google/trillian-examples v0.0.0-20190603134952-4e75ba15216c/go.mod h1:WgL3XZ3pA8/9cm7yxqWrZE6iZkESB2ItGxy5Fo6k2lk=
+github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
+github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
+github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE=
+github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
+github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
+github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
+github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/hcl v0.0.0-20180404174102-ef8a98b0bbce/go.mod h1:oZtUIOe8dh44I2q6ScRibXws4Ajl+d+nod3AaR9vL5w=
+github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
+github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
+github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
+github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
+github.com/juju/ratelimit v1.0.1/go.mod h1:qapgC/Gy+xNh9UxzV13HGGl/6UXNN+ct+vwSgWNm/qk=
+github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
+github.com/kisielk/gotool v0.0.0-20161130080628-0de1eaf82fa3/go.mod h1:jxZFDH7ILpTPQTk+E2s+z4CUas9lVNjIuKR4c5/zKgM=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
+github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
+github.com/klauspost/cpuid v0.0.0-20180405133222-e7e905edc00e/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
+github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
+github.com/letsencrypt/pkcs11key v2.0.1-0.20170608213348-396559074696+incompatible/go.mod h1:iGYXKqDXt0cpBthCHdr9ZdsQwyGlYFh/+8xa4WzIQ34=
+github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/lib/pq v1.3.0 h1:/qkRGz8zljWiDcFvgpwUpwIAPu3r07TDvs3Rws+o/pU=
+github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
+github.com/magiconair/properties v1.7.6/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
+github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
+github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
+github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
+github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
+github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
+github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw=
+github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/miekg/pkcs11 v1.0.2/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs=
+github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936/go.mod h1:r1VsdOzOPt1ZSrGZWFoNhsAedKnEd6r9Np1+5blZCWk=
+github.com/mitchellh/mapstructure v0.0.0-20180220230111-00c29f56e238/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8=
+github.com/mozilla/tls-observatory v0.0.0-20180409132520-8791a200eb40/go.mod h1:SrKMQvPiws7F7iqYp8/TX+IhxCYhzr6N/1yb8cwHsGk=
+github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/nbutton23/zxcvbn-go v0.0.0-20160627004424-a22cb81b2ecd/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU=
+github.com/nbutton23/zxcvbn-go v0.0.0-20171102151520-eafdab6b0663/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU=
+github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
+github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
+github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/pelletier/go-toml v1.1.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
+github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
+github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
+github.com/prometheus/client_golang v0.9.4/go.mod h1:oCXIBxdI62A4cR6aTRJCgetEjecSIYzOEaeAn4iYEpM=
+github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
+github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
+github.com/quasilyte/go-consistent v0.0.0-20190521200055-c6f3937de18c/go.mod h1:5STLWrekHfjyYwxBRVRXNOSewLJ3PWfDJd1VyTS21fI=
+github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
+github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/rs/cors v1.6.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
+github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
+github.com/ryanuber/go-glob v0.0.0-20170128012129-256dc444b735/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc=
+github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
+github.com/shirou/gopsutil v0.0.0-20180427012116-c95755e4bcd7/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
+github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc=
+github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk=
+github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ=
+github.com/sirupsen/logrus v1.0.5/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
+github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
+github.com/sourcegraph/go-diff v0.5.1/go.mod h1:j2dHj3m8aZgQO8lMTcTnBcXkRRRqi34cd2MNlA9u1mE=
+github.com/spf13/afero v1.1.0/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
+github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
+github.com/spf13/cast v1.2.0/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg=
+github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
+github.com/spf13/cobra v0.0.2/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
+github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
+github.com/spf13/jwalterweatherman v0.0.0-20180109140146-7c0cea34c8ec/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
+github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
+github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/viper v1.0.2/go.mod h1:A8kyI5cUJhb8N+3pkfONlcEcZbueH6nhAm0Fq7SrnBM=
+github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/timakin/bodyclose v0.0.0-20190721030226-87058b9bfcec/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk=
+github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
+github.com/tomasen/realip v0.0.0-20180522021738-f0c99a92ddce/go.mod h1:o8v6yHRoik09Xen7gje4m9ERNah1d1PPsVq1VEx9vE4=
+github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
+github.com/ultraware/funlen v0.0.1/go.mod h1:Dp4UiAus7Wdb9KUZsYWZEWiRzGuM2kXM1lPbfaF6xhA=
+github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
+github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
+github.com/valyala/fasthttp v1.2.0/go.mod h1:4vX61m6KN+xDduDNwXrhIAVZaZaZiQ1luJk8LWSxF3s=
+github.com/valyala/quicktemplate v1.1.1/go.mod h1:EH+4AkTd43SvgIbQHYu59/cJyxDoOVRUAfrukLPuGJ4=
+github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
+github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
+github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
+go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
+go.etcd.io/etcd v3.3.13+incompatible/go.mod h1:yaeTdrJi5lOmYerz05bd8+V7KubZs8YSFZfzsF9A6aI=
+go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
+go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
+go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
+go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
+go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
+golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20200109152110-61a87790db17 h1:nVJ3guKA9qdkEQ3TUdXI9QSINo2CUPM/cySEvw2w8I0=
+golang.org/x/crypto v0.0.0-20200109152110-61a87790db17/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/net v0.0.0-20170915142106-8351a756f30f/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20171026204733-164713f0dfce/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.0.0-20170915090833-1cbadb444a80/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/tools v0.0.0-20170915040203-e531a2a1c15f/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20181117154741-2ddaf7f79a09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190110163146-51295c7ec13a/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190121143147-24cd39ecf745/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190311215038-5c2858a9cfe5/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190322203728-c1a832b0ad89/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190521203540-521d6ed310dd/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190909030654-5b82db07426d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
+google.golang.org/api v0.5.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
+google.golang.org/api v0.6.0/go.mod h1:btoxGiFvQNVUZQ8W08zLtrVS08CNpINPEfxXxgJL1Q4=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190605220351-eb0b1bdb6ae6/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
+google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
+gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
+gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/cheggaaa/pb.v1 v1.0.28/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
+gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
+gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo=
+gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
+gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
+gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed/go.mod h1:Xkxe497xwlCKkIaQYRfC7CSLworTXY9RMqwhhCm+8Nc=
+mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b/go.mod h1:2odslEg/xrtNQqCYg2/jCoyKnw3vv5biOc3JnIcYfL4=
+mvdan.cc/unparam v0.0.0-20190209190245-fbb59629db34/go.mod h1:H6SUd1XjIs+qQCyskXg5OFSrilMRUkD8ePJpHKDPaeY=
+sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0=
--- a/main.go
+++ b/main.go
@ -1,12 +1,15 @@
 package main

 import (
+	"archive/tar"
+	"bufio"
 	"bytes"
+	"compress/gzip"
 	"crypto/dsa"
 	"crypto/ecdsa"
 	"crypto/rsa"
+	"crypto/sha1"
 	"crypto/sha256"
-	"crypto/x509"
 	"database/sql"
 	"database/sql/driver"
 	"encoding/json"
@ -17,16 +20,19 @@ import (
 	"io/ioutil"
 	"log"
 	"math/big"
-	"net"
 	"os"
 	"os/signal"
-	"regexp"
+	"path/filepath"
 	"strconv"
 	"strings"
 	"time"

+	"github.com/gallypette/certificate-transparency-go/x509"
+
 	"github.com/gomodule/redigo/redis"
 	_ "github.com/lib/pq"
+
+	config "github.com/D4-project/d4-golang-utils/config"
 )

 type (
@ -41,9 +47,12 @@ type (
 		postgresPort string
 		postgresDB   string
 		certPath     string
+		format       string
+		recursive    bool
+		tarball      bool
 	}

-	BigNumber big.Int
+	bigNumber big.Int

 	certMapElm struct {
 		CertHash string
@ -73,10 +82,13 @@ type (
 )

 var (
-	db           *sql.DB
-	confdir      = flag.String("c", "conf.sample", "configuration directory")
-	connectRedis = true
-	cr           redis.Conn
+	db        *sql.DB
+	confdir   = flag.String("c", "conf.sample", "configuration directory")
+	recursive = flag.Bool("r", false, "should it open the directory recursively")
+	tarball   = flag.Bool("t", false, "is it a tar archive")
+	format    = flag.String("f", "json", "certificate file format [json, crt, der]")
+	pull      = flag.Bool("p", false, "pull from redis?")
+	cr        redis.Conn
 )

 func main() {
@ -119,25 +131,9 @@ func main() {
 		*confdir = strings.TrimSuffix(*confdir, "\\")
 	}

-	// Parse Redis Config
-	tmp := readConfFile(*confdir, "redis")
-	ss := strings.Split(string(tmp), "/")
-	if len(ss) <= 1 {
-		log.Fatal("Missing Database in Redis config: should be host:port/database_name")
-	}
-	c.redisDB, _ = strconv.Atoi(ss[1])
-	var ret bool
-	ret, ss[0] = isNet(ss[0])
-	if !ret {
-		sss := strings.Split(string(ss[0]), ":")
-		c.redisHost = sss[0]
-		c.redisPort = sss[1]
-	}
-	c.redisQueue = string(readConfFile(*confdir, "redis_queue"))
-
 	// Parse DB Config
-	tmp = readConfFile(*confdir, "postgres")
-	ss = strings.Split(string(tmp), "/")
+	tmp := config.ReadConfigFile(*confdir, "postgres")
+	ss := strings.Split(string(tmp), "/")
 	if len(ss) <= 1 {
 		log.Fatal("Missing Database in Postgres config: should be user:pwd@host:port/database_name")
 	}
@ -152,7 +148,7 @@ func main() {
 	}
 	c.postgresUser = sssu[0]
 	c.postgresPWD = sssu[1]
-	ret, ssh := isNet(sssat[1])
+	ret, ssh := config.IsNet(sssat[1])
 	if !ret {
 		sssh := strings.Split(string(ssh), ":")
 		c.postgresHost = sssh[0]
@ -160,7 +156,12 @@ func main() {
 	}

 	// Parse Certificate Folder
-	c.certPath = string(readConfFile(*confdir, "certfolder"))
+	if !*pull {
+		c.certPath = string(config.ReadConfigFile(*confdir, "certfolder"))
+	}
+	c.recursive = *recursive
+	c.tarball = *tarball
+	c.format = *format

 	// DB
 	initDB(c.postgresUser, c.postgresPWD, c.postgresHost, c.postgresPort, c.postgresDB)
@ -168,55 +169,243 @@ func main() {

 	var jsonPath string

-	// Redis
-	if connectRedis {
+	if *pull { // Redis
+		// Parse Redis Config
+		tmp := config.ReadConfigFile(*confdir, "redis")
+		ss := strings.Split(string(tmp), "/")
+		if len(ss) <= 1 {
+			log.Fatal("Missing Database in Redis config: should be host:port/database_name")
+		}
+		c.redisDB, _ = strconv.Atoi(ss[1])
+		var ret bool
+		ret, ss[0] = config.IsNet(ss[0])
+		if !ret {
+			sss := strings.Split(string(ss[0]), ":")
+			c.redisHost = sss[0]
+			c.redisPort = sss[1]
+		}
+		c.redisQueue = string(config.ReadConfigFile(*confdir, "redis_queue"))
 		initRedis(c.redisHost, c.redisPort, c.redisDB)
 		defer cr.Close()
 		// pop redis queue
 		for {
 			err := errors.New("")
-			jsonPath, err = redis.String(cr.Do("LPOP", "analyzer:"+c.redisQueue))
+			jsonPath, err = redis.String(cr.Do("LPOP", "analyzer:ja3-jl:"+c.redisQueue))
 			if err != nil {
-				log.Fatal("Queue processed")
+				log.Fatal(err)
 			}
-			processFile(c.certPath, jsonPath)
+			err = filepath.Walk(jsonPath,
+				func(path string, info os.FileInfo, err error) error {
+					if !info.IsDir() {
+						fd, err := os.Open(path)
+						if err != nil {
+							return err
+						}
+						bf := bufio.NewReader(fd)
+						fmt.Println(path)
+						processFile(bf, path, c.format)
+						// Exit Signal Handle
+						select {
+						case <-s:
+							fmt.Println("Exiting...")
+							os.Exit(0)
+						default:
+						}
+					}
+					return nil
+				})
+			if err != nil {
+				log.Fatal(err)
+			}
+		}
+	} else { // Files
+		if c.recursive {
+			err := filepath.Walk(c.certPath,
+				func(path string, info os.FileInfo, err error) error {
+					if err != nil {
+						return err
+					}
+					if !info.IsDir() {
+						fd, err := os.Open(path)
+						if err != nil {
+							log.Fatal(err)
+						}
+						bf := bufio.NewReader(fd)
+						fmt.Println(path)
+						processFile(bf, path, c.format)
+					}
+					return nil
+				})
+			if err != nil {
+				log.Println(err)
+			}
+		} else if c.tarball {
+			fd, err := os.Stat(c.certPath)
+			if err != nil {
+				log.Fatal(err)
+			}
+			switch mode := fd.Mode(); {
+			case mode.IsDir():
+				log.Fatal("With -t=true flag, you need to input a tarball")
+			case mode.IsRegular():
+				processTar(c.certPath, jsonPath, c.format)
+				break
+			}
+		} else {
+			var path []string
+			path = append(path, "./sessions")
+			path = append(path, "")
+			files, err := ioutil.ReadDir(path[0])
+			if err != nil {
+				log.Fatal(err)
+			}
+			for _, f := range files {
+				path[1] = f.Name()
+				jsonPath = strings.Join(path, "/")
+				fd, err := os.Open(jsonPath)
+				if err != nil {
+					log.Fatal(err)
+				}
+				bf := bufio.NewReader(fd)
+				processFile(bf, jsonPath, c.format)

-			// Exit Signal Handle
-			select {
-			case <-s:
-				fmt.Println("Exiting...")
-				os.Exit(0)
-			default:
-				continue
+				// Exit Signal Handle
+				select {
+				case <-s:
+					fmt.Println("Exiting...")
+					os.Exit(0)
+				}
 			}
 		}

-	} else {
-		var path []string
-		path = append(path, "./sessions")
-		path = append(path, "")
-		files, err := ioutil.ReadDir(path[0])
-		if err != nil {
-			log.Fatal(err)
-		}
-		for _, f := range files {
-			path[1] = f.Name()
-			jsonPath = strings.Join(path, "/")
-			processFile(c.certPath, jsonPath)
+	}
+}

-			// Exit Signal Handle
-			select {
-			case <-s:
-				fmt.Println("Exiting...")
-				os.Exit(0)
-			}
+func processTar(fp string, p string, f string) error {
+	fd, err := os.Open(fp)
+	if err != nil {
+		log.Fatal(err)
+	}
+	bf := bufio.NewReader(fd)
+	gzr, err := gzip.NewReader(bf)
+	if err != nil {
+		return err
+	}
+	defer gzr.Close()
+	tr := tar.NewReader(gzr)
+	for {
+		header, err := tr.Next()
+		switch {
+		// if no more files are found return
+		case err == io.EOF:
+			return nil
+		// return any other error
+		case err != nil:
+			return err
+		// if the header is nil, just skip it (not sure how this happens)
+		case header == nil:
+			continue
+		}
+		switch header.Typeflag {
+		case tar.TypeDir:
+			continue
+		case tar.TypeReg:
+			processFile(tr, fp, f)
 		}
 	}
 }

-func processFile(fp string, p string) bool {
+func processFile(r io.Reader, fp string, f string) {
+	switch f {
+	case "json":
+		processJSON(r, fp)
+		break
+	case "der":
+		processDER(r, fp)
+		break
+	case "crt":
+		processCRT(r, fp)
+		break
+	}
+}
+
+func processDER(r io.Reader, p string) bool {
+	// read corresponding der file
+	dat, err := ioutil.ReadAll(r)
+	if err != nil {
+		log.Fatal(err)
+	}
+	cert, err := x509.ParseCertificate(dat)
+	if err != nil {
+		// Not stopping on Non Fatal Errors
+		switch err := err.(type) {
+		case x509.NonFatalErrors:
+			// Stopping on Unknown PK Algo
+			if cert.PublicKeyAlgorithm == 0 {
+				fmt.Println("Unknown Public Key Algorithm, skipping key (most likely GOST R 14)")
+				break
+			}
+			goto I
+		default:
+			fmt.Println("failed to parse certificate: " + err.Error())
+			return false
+		}
+	}
+
+I:
+	// Cert elem
+	h := sha1.New()
+	h.Write(cert.Raw)
+	c := certMapElm{Certificate: cert, CertHash: fmt.Sprintf("%x", h.Sum(nil))}
+	// Insert Certificate
+	err = insertLeafCertificate(p, c)
+	if err != nil {
+		// Not stopping on failed insertion
+		log.Println(fmt.Sprintf("Insert Certificate into DB failed: %q", err))
+	}
+
+	return true
+}
+
+func insertLeafCertificate(fp string, c certMapElm) error {
+	key, err := insertPublicKey(*c.Certificate)
+	if err != nil {
+		// Not stopping on Non Fatal Errors
+		switch err := err.(type) {
+		case x509.NonFatalErrors:
+			goto J
+		default:
+			fmt.Println("failed to Insert Key: " + err.Error())
+			return err
+		}
+	}
+J:
+	// q := `INSERT INTO "certificate" (hash, "is_CA", "is_SS", issuer, subject, cert_chain, is_valid_chain, atrest_path) VALUES ($1, $2, $3, $4, $5, $6, $7, $8) ON CONFLICT DO NOTHING`
+	q := `INSERT INTO "certificate" (hash, "is_CA", "is_SS", issuer, subject, cert_chain, is_valid_chain, atrest_path) VALUES ($1, $2, $3, $4, $5, $6, $7, $8) ON CONFLICT (hash) DO UPDATE SET atrest_path = excluded.atrest_path`
+	_, err = db.Exec(q, c.CertHash, c.Certificate.IsCA, false, c.Certificate.Issuer.String(), c.Certificate.Subject.String(), nil, false, fp)
+	fmt.Println(fp)
+	if err != nil {
+		return err
+	}
+	//fmt.Println(c.CertHash)
+	//fmt.Println(key)
+	q = `INSERT INTO "many_certificate_has_many_public_key" ("hash_certificate", "hash_public_key") VALUES ($1, $2) ON CONFLICT DO NOTHING `
+	_, err = db.Exec(q, c.CertHash, key)
+	if err != nil {
+		fmt.Println(c.CertHash)
+		return err
+	}
+	return nil
+}
+
+func processCRT(r io.Reader, fp string) bool {
+
+	return true
+}
+
+func processJSON(r io.Reader, fp string) bool {
 	// read corresponding json file
-	dat, err := ioutil.ReadFile(p)
+	dat, err := ioutil.ReadAll(r)
 	if err != nil {
 		log.Fatal(err)
 	}
@ -296,25 +485,30 @@ func insertPublicKey(c x509.Certificate) (string, error) {
 	pub, err := x509.ParsePKIXPublicKey(c.RawSubjectPublicKeyInfo)
 	hash := fmt.Sprintf("%x", sha256.Sum256(c.RawSubjectPublicKeyInfo))
 	if err != nil {
-		return hash, nil
+		return hash, err
 	}

 	switch pub := pub.(type) {
 	case *rsa.PublicKey:
 		q := `INSERT INTO "public_key" (hash, type, modulus, exponent, modulus_size) VALUES ($1, $2, $3, $4, $5) ON CONFLICT DO NOTHING`
-		_, err := db.Exec(q, hash, "RSA", (*BigNumber)(pub.N), pub.E, pub.Size())
+		_, err := db.Exec(q, hash, "RSA", (*bigNumber)(pub.N), pub.E, pub.Size())
 		if err != nil {
 			return hash, err
 		}
+		//  else {
+		// Adds the moduli into Redis for analysis
+		// cr.Send("HMSET", hash, "moduli", (*BigNumber)(pub.N))
+		//cr.Send("LPUSH", "albums", "1")
+		// }
 	case *dsa.PublicKey:
 		q := `INSERT INTO "public_key" (hash, type, "G", "P", "Q", "Y") VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT DO NOTHING`
-		_, err := db.Exec(q, hash, "DSA", (*BigNumber)(pub.Parameters.G), (*BigNumber)(pub.Parameters.P), (*BigNumber)(pub.Parameters.Q), (*BigNumber)(pub.Y))
+		_, err := db.Exec(q, hash, "DSA", (*bigNumber)(pub.Parameters.G), (*bigNumber)(pub.Parameters.P), (*bigNumber)(pub.Parameters.Q), (*bigNumber)(pub.Y))
 		if err != nil {
 			return hash, err
 		}
 	case *ecdsa.PublicKey:
 		q := `INSERT INTO "public_key" (hash, type, "Y", "X", "P", "N", "B", "bitsize", "Gx", "Gy", "curve_name") VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) ON CONFLICT DO NOTHING`
-		_, err := db.Exec(q, hash, "ECDSA", (*BigNumber)(pub.Y), (*BigNumber)(pub.X), (*BigNumber)(pub.Curve.Params().P), (*BigNumber)(pub.Curve.Params().N), (*BigNumber)(pub.Curve.Params().B), pub.Curve.Params().BitSize, (*BigNumber)(pub.Curve.Params().Gx), (*BigNumber)(pub.Curve.Params().Gy), pub.Params().Name)
+		_, err := db.Exec(q, hash, "ECDSA", (*bigNumber)(pub.Y), (*bigNumber)(pub.X), (*bigNumber)(pub.Curve.Params().P), (*bigNumber)(pub.Curve.Params().N), (*bigNumber)(pub.Curve.Params().B), pub.Curve.Params().BitSize, (*bigNumber)(pub.Curve.Params().Gx), (*bigNumber)(pub.Curve.Params().Gy), pub.Params().Name)
 		if err != nil {
 			return hash, err
 		}
@ -324,7 +518,7 @@ func insertPublicKey(c x509.Certificate) (string, error) {
 	return hash, nil
 }

-func (bn *BigNumber) Value() (driver.Value, error) {
+func (bn *bigNumber) Value() (driver.Value, error) {
 	return driver.Value((*big.Int)(bn).Text(10)), nil
 }

@ -430,7 +624,7 @@ func unique(s []certMapElm) []certMapElm {
 }

 func insertCertificate(fp string, c certMapElm) (string, error) {
-	q := `INSERT INTO "certificate" (hash, "is_CA", "is_SS", issuer, subject, cert_chain, is_valid_chain, file_path) VALUES ($1, $2, $3, $4, $5, $6, $7, $8) ON CONFLICT DO NOTHING`
+	q := `INSERT INTO "certificate" (hash, "is_CA", "is_SS", issuer, subject, cert_chain, is_valid_chain, atrest_path) VALUES ($1, $2, $3, $4, $5, $6, $7, $8) ON CONFLICT DO NOTHING`
 	_, err := db.Exec(q, c.CertHash, c.Certificate.IsCA, c.chain.isSS, c.Certificate.Issuer.String(), c.Certificate.Subject.String(), c.chain.s, c.chain.isValid, getFullPath(fp, c.CertHash))
 	if err != nil {
 		return c.CertHash, err
@ -439,8 +633,8 @@ func insertCertificate(fp string, c certMapElm) (string, error) {
 	if err != nil {
 		return c.CertHash, err
 	}
-	fmt.Println(c.CertHash)
-	fmt.Println(key)
+	//fmt.Println(c.CertHash)
+	//fmt.Println(key)
 	q = `INSERT INTO "many_certificate_has_many_public_key" ("hash_certificate", "hash_public_key") VALUES ($1, $2) ON CONFLICT DO NOTHING `
 	_, err = db.Exec(q, c.CertHash, key)
 	if err != nil {
@ -452,7 +646,7 @@ func insertCertificate(fp string, c certMapElm) (string, error) {
 // getFullPath takes a certificate's hash and return the full path to
 // its location on disk
 func getFullPath(c string, h string) string {
-	return c+h
+	return c + h
 }

 func insertCertificates(fp string, s *sessionRecord) ([]string, error) {
@ -515,86 +709,3 @@ func (t *sessionRecord) String() string {
 	buf.WriteString(fmt.Sprintf("---------------SESSION  END--------------------\n"))
 	return buf.String()
 }
-
-func isNet(host string) (bool, string) {
-	// DNS regex
-	validDNS := regexp.MustCompile(`^(([a-zA-Z]{1})|([a-zA-Z]{1}[a-zA-Z]{1})|([a-zA-Z]{1}[0-9]{1})|([0-9]{1}[a-zA-Z]{1})|([a-zA-Z0-9][a-zA-Z0-9-_]{1,61}[a-zA-Z0-9]))\.([a-zA-Z]{2,6}|[a-zA-Z0-9-]{2,30}\.[a-zA-Z
- ]{2,3})$`)
-	// Check ipv6
-	if strings.HasPrefix(host, "[") {
-		// Parse an IP-Literal in RFC 3986 and RFC 6874.
-		// E.g., "[fe80::1]:80".
-		i := strings.LastIndex(host, "]")
-		if i < 0 {
-			log.Fatal("Unmatched [ in destination config")
-			return false, ""
-		}
-		if !validPort(host[i+1:]) {
-			log.Fatal("No valid port specified")
-			return false, ""
-		}
-		// trim brackets
-		if net.ParseIP(strings.Trim(host[:i+1], "[]")) != nil {
-			log.Fatal(fmt.Sprintf("Server IP: %s, Server Port: %s\n", host[:i+1], host[i+1:]))
-			return true, host
-		}
-	} else {
-		// Ipv4 or DNS name
-		ss := strings.Split(string(host), ":")
-		if len(ss) > 1 {
-			if !validPort(":" + ss[1]) {
-				log.Fatal("No valid port specified")
-				return false, ""
-			}
-			if net.ParseIP(ss[0]) != nil {
-				log.Fatal(fmt.Sprintf("Server IP: %s, Server Port: %s\n", ss[0], ss[1]))
-				return true, host
-			} else if validDNS.MatchString(ss[0]) {
-				log.Fatal(fmt.Sprintf("DNS: %s, Server Port: %s\n", ss[0], ss[1]))
-				return true, host
-			}
-		}
-	}
-	return false, host
-}
-
-// Reusing code from net.url
-// validOptionalPort reports whether port is either an empty string
-// or matches /^:\d*$/
-func validPort(port string) bool {
-	if port == "" {
-		return false
-	}
-	if port[0] != ':' {
-		return false
-	}
-	for _, b := range port[1:] {
-		if b < '0' || b > '9' {
-			return false
-		}
-	}
-	return true
-}
-
-func readConfFile(p string, fileName string) []byte {
-	f, err := os.OpenFile("./"+p+"/"+fileName, os.O_RDWR|os.O_CREATE, 0666)
-	defer f.Close()
-	if err != nil {
-		log.Fatal(err)
-	}
-	data := make([]byte, 100)
-	count, err := f.Read(data)
-	if err != nil {
-		if err != io.EOF {
-			log.Fatal(err)
-		}
-	}
-	if count == 0 {
-		log.Fatal(fileName + " is empty.")
-	}
-	if err := f.Close(); err != nil {
-		log.Fatal(err)
-	}
-	// trim \n if present
-	return bytes.TrimSuffix(data[:count], []byte("\n"))
-}
--- a/passivessl.sql
+++ b/passivessl.sql
@ -1,32 +1,41 @@
 -- Database generated with pgModeler (PostgreSQL Database Modeler).
-- pgModeler  version: 0.9.1-beta
+-- pgModeler  version: 0.9.1
 -- PostgreSQL version: 10.0
-- Project Site: pgmodeler.com.br
+-- Project Site: pgmodeler.io
 -- Model Author: ---

 SET check_function_bodies = false;
 -- ddl-end --

-
-- Database creation must be done outside an multicommand file.
-- These commands were put in this file only for convenience.
+-- Database creation must be done outside a multicommand file.
+-- These commands were put in this file only as a convenience.
 -- -- object: passive_ssl | type: DATABASE --
 -- -- DROP DATABASE IF EXISTS passive_ssl;
 -- CREATE DATABASE passive_ssl
-- ;
+-- 	ENCODING = 'UTF8'
+-- 	LC_COLLATE = 'en_US.UTF-8'
+-- 	LC_CTYPE = 'en_US.UTF-8'
+-- 	TABLESPACE = pg_default
+-- 	OWNER = postgres;
 -- -- ddl-end --
 -- 

 -- object: ltree | type: EXTENSION --
 -- DROP EXTENSION IF EXISTS ltree CASCADE;
 CREATE EXTENSION ltree
-      WITH SCHEMA public;
+      WITH SCHEMA public
+      VERSION '1.1';
+-- ddl-end --
+COMMENT ON EXTENSION ltree IS 'data type for hierarchical tree-like structures';
 -- ddl-end --

 -- object: hstore | type: EXTENSION --
 -- DROP EXTENSION IF EXISTS hstore CASCADE;
 CREATE EXTENSION hstore
-      WITH SCHEMA public;
+      WITH SCHEMA public
+      VERSION '1.4';
+-- ddl-end --
+COMMENT ON EXTENSION hstore IS 'data type for storing sets of (key, value) pairs';
 -- ddl-end --

 -- object: public.public_key | type: TABLE --
@ -35,8 +44,8 @@ CREATE TABLE public.public_key(
 	hash bytea NOT NULL,
 	type text NOT NULL,
 	modulus text,
-	exponent int4,
-	modulus_size int4,
+	exponent integer,
+	modulus_size integer,
 	"P" numeric,
 	"Q" numeric,
 	"G" numeric,
@ -44,10 +53,11 @@ CREATE TABLE public.public_key(
 	"X" numeric,
 	"N" numeric,
 	"B" numeric,
-	bitsize int4,
-	curve_name varchar(256),
+	bitsize integer,
+	curve_name character varying(256),
 	"Gx" numeric,
 	"Gy" numeric,
+	private bytea,
 	CONSTRAINT public_key_pk PRIMARY KEY (hash)

 );
@ -58,20 +68,46 @@ ALTER TABLE public.public_key OWNER TO postgres;
 -- object: public.certificate | type: TABLE --
 -- DROP TABLE IF EXISTS public.certificate CASCADE;
 CREATE TABLE public.certificate(
-	file_path varchar(4096) NOT NULL,
+	mounted_path character varying(4096),
 	issuer text,
-	cert_chain public.ltree,
+	cert_chain ltree,
 	subject text,
 	hash bytea NOT NULL,
-	"is_CA" bool NOT NULL DEFAULT false,
-	is_valid_chain bool NOT NULL DEFAULT false,
+	"is_CA" boolean NOT NULL DEFAULT false,
+	is_valid_chain boolean NOT NULL DEFAULT false,
 	"notBefore" time,
 	"notAfter" time,
-	"is_SS" bool NOT NULL DEFAULT false,
+	"is_SS" boolean NOT NULL DEFAULT false,
+	"Signature" bytea,
+	"SignatureAlgorithm" text,
+	"Version" integer,
+	"DNSnames" text[],
+	emails text[],
+	"IPaddresses" inet[],
+	"URIs" text[],
+	"PermittedDNS" text[],
+	"ExcludedDNS" text[],
+	"PermittedIPRanges" cidr[],
+	"ExcludedIPRanges" cidr[],
+	"PermittedEmailAddresses" text[],
+	"ExcludedEmailAddresses" text[],
+	"PermittedURIDomains" text[],
+	"ExcludedURIDomains" text[],
+	fs_type smallint DEFAULT 0,
+	atrest_path character varying(4096),
 	CONSTRAINT certificate_pk PRIMARY KEY (hash)

 );
 -- ddl-end --
+COMMENT ON COLUMN public.certificate.mounted_path IS 'Where to access the file when mounted (check fs_type to know the state)';
+-- ddl-end --
+COMMENT ON COLUMN public.certificate.fs_type IS 'How to access the raw certificate:
+0 - mount point
+1 - tar.gz
+2 - squashfs';
+-- ddl-end --
+COMMENT ON COLUMN public.certificate.atrest_path IS 'Where to access the file when unmounted (check fs_type to know how to mount)';
+-- ddl-end --
 ALTER TABLE public.certificate OWNER TO postgres;
 -- ddl-end --

@ -84,31 +120,33 @@ CREATE TABLE public.many_certificate_has_many_public_key(

 );
 -- ddl-end --
-
-- object: certificate_fk | type: CONSTRAINT --
-- ALTER TABLE public.many_certificate_has_many_public_key DROP CONSTRAINT IF EXISTS certificate_fk CASCADE;
-ALTER TABLE public.many_certificate_has_many_public_key ADD CONSTRAINT certificate_fk FOREIGN KEY (hash_certificate)
-REFERENCES public.certificate (hash) MATCH FULL
-ON DELETE RESTRICT ON UPDATE CASCADE;
+ALTER TABLE public.many_certificate_has_many_public_key OWNER TO postgres;
 -- ddl-end --

-- object: public_key_fk | type: CONSTRAINT --
-- ALTER TABLE public.many_certificate_has_many_public_key DROP CONSTRAINT IF EXISTS public_key_fk CASCADE;
-ALTER TABLE public.many_certificate_has_many_public_key ADD CONSTRAINT public_key_fk FOREIGN KEY (hash_public_key)
-REFERENCES public.public_key (hash) MATCH FULL
-ON DELETE RESTRICT ON UPDATE CASCADE;
+-- object: public."sessionRecord_id_seq" | type: SEQUENCE --
+-- DROP SEQUENCE IF EXISTS public."sessionRecord_id_seq" CASCADE;
+CREATE SEQUENCE public."sessionRecord_id_seq"
+	INCREMENT BY 1
+	MINVALUE 1
+	MAXVALUE 9223372036854775807
+	START WITH 1
+	CACHE 1
+	NO CYCLE
+	OWNED BY NONE;
+-- ddl-end --
+ALTER SEQUENCE public."sessionRecord_id_seq" OWNER TO postgres;
 -- ddl-end --

 -- object: public."sessionRecord" | type: TABLE --
 -- DROP TABLE IF EXISTS public."sessionRecord" CASCADE;
 CREATE TABLE public."sessionRecord"(
-	id bigserial NOT NULL,
+	id bigint NOT NULL DEFAULT nextval('public."sessionRecord_id_seq"'::regclass),
 	dst_ip inet NOT NULL,
 	src_ip inet NOT NULL,
-	dst_port int4 NOT NULL,
-	src_port int4 NOT NULL,
+	dst_port integer NOT NULL,
+	src_port integer NOT NULL,
 	hash_ja3 bytea NOT NULL,
-	"timestamp" time(0) with time zone,
+	"timestamp" timestamp(0) with time zone,
 	CONSTRAINT "sessionRecord_pk" PRIMARY KEY (id)

 );
@ -121,7 +159,7 @@ ALTER TABLE public."sessionRecord" OWNER TO postgres;
 CREATE TABLE public.ja3(
 	hash bytea NOT NULL,
 	raw text,
-	type varchar(16) NOT NULL,
+	type character varying(16) NOT NULL,
 	CONSTRAINT j3a_pk PRIMARY KEY (hash)

 );
@ -129,13 +167,6 @@ CREATE TABLE public.ja3(
 ALTER TABLE public.ja3 OWNER TO postgres;
 -- ddl-end --

-- object: ja3_fk | type: CONSTRAINT --
-- ALTER TABLE public."sessionRecord" DROP CONSTRAINT IF EXISTS ja3_fk CASCADE;
-ALTER TABLE public."sessionRecord" ADD CONSTRAINT ja3_fk FOREIGN KEY (hash_ja3)
-REFERENCES public.ja3 (hash) MATCH FULL
-ON DELETE RESTRICT ON UPDATE CASCADE;
-- ddl-end --
-
 -- object: public."many_sessionRecord_has_many_certificate" | type: TABLE --
 -- DROP TABLE IF EXISTS public."many_sessionRecord_has_many_certificate" CASCADE;
 CREATE TABLE public."many_sessionRecord_has_many_certificate"(
@ -145,25 +176,27 @@ CREATE TABLE public."many_sessionRecord_has_many_certificate"(

 );
 -- ddl-end --
-
-- object: "sessionRecord_fk" | type: CONSTRAINT --
-- ALTER TABLE public."many_sessionRecord_has_many_certificate" DROP CONSTRAINT IF EXISTS "sessionRecord_fk" CASCADE;
-ALTER TABLE public."many_sessionRecord_has_many_certificate" ADD CONSTRAINT "sessionRecord_fk" FOREIGN KEY ("id_sessionRecord")
-REFERENCES public."sessionRecord" (id) MATCH FULL
-ON DELETE RESTRICT ON UPDATE CASCADE;
+ALTER TABLE public."many_sessionRecord_has_many_certificate" OWNER TO postgres;
 -- ddl-end --

-- object: certificate_fk | type: CONSTRAINT --
-- ALTER TABLE public."many_sessionRecord_has_many_certificate" DROP CONSTRAINT IF EXISTS certificate_fk CASCADE;
-ALTER TABLE public."many_sessionRecord_has_many_certificate" ADD CONSTRAINT certificate_fk FOREIGN KEY (hash_certificate)
-REFERENCES public.certificate (hash) MATCH FULL
-ON DELETE RESTRICT ON UPDATE CASCADE;
+-- object: public.fuzzy_hash_id_seq | type: SEQUENCE --
+-- DROP SEQUENCE IF EXISTS public.fuzzy_hash_id_seq CASCADE;
+CREATE SEQUENCE public.fuzzy_hash_id_seq
+	INCREMENT BY 1
+	MINVALUE 1
+	MAXVALUE 9223372036854775807
+	START WITH 1
+	CACHE 1
+	NO CYCLE
+	OWNED BY NONE;
+-- ddl-end --
+ALTER SEQUENCE public.fuzzy_hash_id_seq OWNER TO postgres;
 -- ddl-end --

 -- object: public.fuzzy_hash | type: TABLE --
 -- DROP TABLE IF EXISTS public.fuzzy_hash CASCADE;
 CREATE TABLE public.fuzzy_hash(
-	id bigserial NOT NULL,
+	id bigint NOT NULL DEFAULT nextval('public.fuzzy_hash_id_seq'::regclass),
 	type text NOT NULL,
 	value text NOT NULL,
 	"id_sessionRecord" bigint,
@ -174,10 +207,24 @@ CREATE TABLE public.fuzzy_hash(
 ALTER TABLE public.fuzzy_hash OWNER TO postgres;
 -- ddl-end --

+-- object: public.software_id_seq | type: SEQUENCE --
+-- DROP SEQUENCE IF EXISTS public.software_id_seq CASCADE;
+CREATE SEQUENCE public.software_id_seq
+	INCREMENT BY 1
+	MINVALUE 1
+	MAXVALUE 2147483647
+	START WITH 1
+	CACHE 1
+	NO CYCLE
+	OWNED BY NONE;
+-- ddl-end --
+ALTER SEQUENCE public.software_id_seq OWNER TO postgres;
+-- ddl-end --
+
 -- object: public.software | type: TABLE --
 -- DROP TABLE IF EXISTS public.software CASCADE;
 CREATE TABLE public.software(
-	id serial NOT NULL,
+	id integer NOT NULL DEFAULT nextval('public.software_id_seq'::regclass),
 	name text NOT NULL,
 	version text,
 	CONSTRAINT software_pk PRIMARY KEY (id)
@ -187,10 +234,24 @@ CREATE TABLE public.software(
 ALTER TABLE public.software OWNER TO postgres;
 -- ddl-end --

+-- object: public.annotation_id_seq | type: SEQUENCE --
+-- DROP SEQUENCE IF EXISTS public.annotation_id_seq CASCADE;
+CREATE SEQUENCE public.annotation_id_seq
+	INCREMENT BY 1
+	MINVALUE 1
+	MAXVALUE 2147483647
+	START WITH 1
+	CACHE 1
+	NO CYCLE
+	OWNED BY NONE;
+-- ddl-end --
+ALTER SEQUENCE public.annotation_id_seq OWNER TO postgres;
+-- ddl-end --
+
 -- object: public.annotation | type: TABLE --
 -- DROP TABLE IF EXISTS public.annotation CASCADE;
 CREATE TABLE public.annotation(
-	id serial NOT NULL,
+	id integer NOT NULL DEFAULT nextval('public.annotation_id_seq'::regclass),
 	hash_ja3 bytea,
 	confidence smallint,
 	id_software integer,
@ -201,6 +262,219 @@ CREATE TABLE public.annotation(
 ALTER TABLE public.annotation OWNER TO postgres;
 -- ddl-end --

+-- object: ja3_trie | type: INDEX --
+-- DROP INDEX IF EXISTS public.ja3_trie CASCADE;
+CREATE INDEX ja3_trie ON public.ja3
+	USING spgist
+	(
+	  raw
+	)
+	WITH (FILLFACTOR = 90);
+-- ddl-end --
+
+-- object: hash_index | type: INDEX --
+-- DROP INDEX IF EXISTS public.hash_index CASCADE;
+CREATE INDEX hash_index ON public.certificate
+	USING btree
+	(
+	  hash
+	)
+	WITH (FILLFACTOR = 90);
+-- ddl-end --
+
+-- object: pk_index | type: INDEX --
+-- DROP INDEX IF EXISTS public.pk_index CASCADE;
+CREATE INDEX pk_index ON public.public_key
+	USING btree
+	(
+	  hash
+	)
+	WITH (FILLFACTOR = 90);
+-- ddl-end --
+
+-- object: dst_index | type: INDEX --
+-- DROP INDEX IF EXISTS public.dst_index CASCADE;
+CREATE INDEX dst_index ON public."sessionRecord"
+	USING btree
+	(
+	  dst_ip
+	)
+	WITH (FILLFACTOR = 90);
+-- ddl-end --
+
+-- object: path_index | type: INDEX --
+-- DROP INDEX IF EXISTS public.path_index CASCADE;
+CREATE INDEX path_index ON public.certificate
+	USING gist
+	(
+	  cert_chain
+	)
+	WITH (FILLFACTOR = 90);
+-- ddl-end --
+
+-- DROP LANGUAGE IF EXISTS plpython3u CASCADE;
+CREATE  LANGUAGE plpython3u;
+-- ddl-end --
+ALTER LANGUAGE plpython3u OWNER TO postgres;
+-- ddl-end --
+
+-- object: public.tlshc | type: FUNCTION --
+-- DROP FUNCTION IF EXISTS public.tlshc(text,text) CASCADE;
+CREATE FUNCTION public.tlshc ( a text,  b text)
+	RETURNS integer
+	LANGUAGE plpython3u
+	VOLATILE 
+	CALLED ON NULL INPUT
+	SECURITY INVOKER
+	COST 1
+	AS $$
+
+import tlsh
+return tlsh.diff(a, b)
+
+$$;
+-- ddl-end --
+ALTER FUNCTION public.tlshc(text,text) OWNER TO postgres;
+-- ddl-end --
+
+-- object: public.tlsht | type: FUNCTION --
+-- DROP FUNCTION IF EXISTS public.tlsht(text,text,integer,integer) CASCADE;
+CREATE FUNCTION public.tlsht ( filter text,  hash text,  threshold integer,  maxrows integer)
+	RETURNS SETOF public.fuzzy_hash
+	LANGUAGE plpython3u
+	IMMUTABLE LEAKPROOF
+	STRICT
+	SECURITY INVOKER
+	COST 1
+	ROWS 1000
+	AS $$
+
+import tlsh
+plan = plpy.prepare("SELECT * FROM fuzzy_hash WHERE type <> $1", ["text"])
+rv = plan.execute(["filter"], maxrows)
+r = []
+for x in rv:
+    if tlsh.diff(x["value"], hash) < threshold:
+        r.append(x)
+return r
+
+$$;
+-- ddl-end --
+ALTER FUNCTION public.tlsht(text,text,integer,integer) OWNER TO postgres;
+-- ddl-end --
+
+-- object: pg_catalog.plpython3_validator_cp | type: FUNCTION --
+-- DROP FUNCTION IF EXISTS pg_catalog.plpython3_validator_cp(oid) CASCADE;
+CREATE FUNCTION pg_catalog.plpython3_validator_cp ( _param1 oid)
+	RETURNS void
+	LANGUAGE c
+	VOLATILE 
+	STRICT
+	SECURITY INVOKER
+	COST 1
+	AS '$libdir/plpython3', 'plpython3_validator';
+-- ddl-end --
+ALTER FUNCTION pg_catalog.plpython3_validator_cp(oid) OWNER TO postgres;
+-- ddl-end --
+
+-- object: pg_catalog.plpython3_call_handler_cp | type: FUNCTION --
+-- DROP FUNCTION IF EXISTS pg_catalog.plpython3_call_handler_cp() CASCADE;
+CREATE FUNCTION pg_catalog.plpython3_call_handler_cp ()
+	RETURNS language_handler
+	LANGUAGE c
+	VOLATILE 
+	CALLED ON NULL INPUT
+	SECURITY INVOKER
+	COST 1
+	AS '$libdir/plpython3', 'plpython3_call_handler';
+-- ddl-end --
+ALTER FUNCTION pg_catalog.plpython3_call_handler_cp() OWNER TO postgres;
+-- ddl-end --
+
+-- object: pg_catalog.plpython3_inline_handler_cp | type: FUNCTION --
+-- DROP FUNCTION IF EXISTS pg_catalog.plpython3_inline_handler_cp(internal) CASCADE;
+CREATE FUNCTION pg_catalog.plpython3_inline_handler_cp ( _param1 internal)
+	RETURNS void
+	LANGUAGE c
+	VOLATILE 
+	STRICT
+	SECURITY INVOKER
+	COST 1
+	AS '$libdir/plpython3', 'plpython3_inline_handler';
+-- ddl-end --
+ALTER FUNCTION pg_catalog.plpython3_inline_handler_cp(internal) OWNER TO postgres;
+-- ddl-end --
+
+-- object: public.zgrep | type: FUNCTION --
+-- DROP FUNCTION IF EXISTS public.zgrep(text,text) CASCADE;
+CREATE FUNCTION public.zgrep ( a text,  b text)
+	RETURNS integer
+	LANGUAGE plpython3u
+	VOLATILE 
+	CALLED ON NULL INPUT
+	SECURITY INVOKER
+	COST 1
+	AS $$
+
+import tlsh
+return tlsh.diff(a, b)
+
+$$;
+-- ddl-end --
+ALTER FUNCTION public.zgrep(text,text) OWNER TO postgres;
+-- ddl-end --
+
+-- object: p_index | type: INDEX --
+-- DROP INDEX IF EXISTS public.p_index CASCADE;
+CREATE INDEX  CONCURRENTLY p_index ON public.public_key
+	USING btree
+	(
+	  "P"
+	);
+-- ddl-end --
+
+-- object: certificate_fk | type: CONSTRAINT --
+-- ALTER TABLE public.many_certificate_has_many_public_key DROP CONSTRAINT IF EXISTS certificate_fk CASCADE;
+ALTER TABLE public.many_certificate_has_many_public_key ADD CONSTRAINT certificate_fk FOREIGN KEY (hash_certificate)
+REFERENCES public.certificate (hash) MATCH FULL
+ON DELETE CASCADE ON UPDATE CASCADE;
+-- ddl-end --
+
+-- object: public_key_fk | type: CONSTRAINT --
+-- ALTER TABLE public.many_certificate_has_many_public_key DROP CONSTRAINT IF EXISTS public_key_fk CASCADE;
+ALTER TABLE public.many_certificate_has_many_public_key ADD CONSTRAINT public_key_fk FOREIGN KEY (hash_public_key)
+REFERENCES public.public_key (hash) MATCH FULL
+ON DELETE CASCADE ON UPDATE CASCADE;
+-- ddl-end --
+
+-- object: ja3_fk | type: CONSTRAINT --
+-- ALTER TABLE public."sessionRecord" DROP CONSTRAINT IF EXISTS ja3_fk CASCADE;
+ALTER TABLE public."sessionRecord" ADD CONSTRAINT ja3_fk FOREIGN KEY (hash_ja3)
+REFERENCES public.ja3 (hash) MATCH FULL
+ON DELETE CASCADE ON UPDATE CASCADE;
+-- ddl-end --
+
+-- object: "sessionRecord_fk" | type: CONSTRAINT --
+-- ALTER TABLE public."many_sessionRecord_has_many_certificate" DROP CONSTRAINT IF EXISTS "sessionRecord_fk" CASCADE;
+ALTER TABLE public."many_sessionRecord_has_many_certificate" ADD CONSTRAINT "sessionRecord_fk" FOREIGN KEY ("id_sessionRecord")
+REFERENCES public."sessionRecord" (id) MATCH FULL
+ON DELETE CASCADE ON UPDATE CASCADE;
+-- ddl-end --
+
+-- object: certificate_fk | type: CONSTRAINT --
+-- ALTER TABLE public."many_sessionRecord_has_many_certificate" DROP CONSTRAINT IF EXISTS certificate_fk CASCADE;
+ALTER TABLE public."many_sessionRecord_has_many_certificate" ADD CONSTRAINT certificate_fk FOREIGN KEY (hash_certificate)
+REFERENCES public.certificate (hash) MATCH FULL
+ON DELETE CASCADE ON UPDATE CASCADE;
+-- ddl-end --
+
+-- object: "sessionRecord_fk" | type: CONSTRAINT --
+-- ALTER TABLE public.fuzzy_hash DROP CONSTRAINT IF EXISTS "sessionRecord_fk" CASCADE;
+ALTER TABLE public.fuzzy_hash ADD CONSTRAINT "sessionRecord_fk" FOREIGN KEY ("id_sessionRecord")
+REFERENCES public."sessionRecord" (id) MATCH FULL
+ON DELETE SET NULL ON UPDATE CASCADE;
+-- ddl-end --
+
 -- object: ja3_fk | type: CONSTRAINT --
 -- ALTER TABLE public.annotation DROP CONSTRAINT IF EXISTS ja3_fk CASCADE;
 ALTER TABLE public.annotation ADD CONSTRAINT ja3_fk FOREIGN KEY (hash_ja3)
@ -215,105 +489,4 @@ REFERENCES public.software (id) MATCH FULL
 ON DELETE SET NULL ON UPDATE CASCADE;
 -- ddl-end --

-- object: ja3_trie | type: INDEX --
-- DROP INDEX IF EXISTS public.ja3_trie CASCADE;
-CREATE INDEX  CONCURRENTLY ja3_trie ON public.ja3
-	USING spgist
-	(
-	  raw
-	);
-- ddl-end --
-
-- object: hash_index | type: INDEX --
-- DROP INDEX IF EXISTS public.hash_index CASCADE;
-CREATE INDEX hash_index ON public.certificate
-	USING btree
-	(
-	  hash
-	);
-- ddl-end --
-
-- object: pk_index | type: INDEX --
-- DROP INDEX IF EXISTS public.pk_index CASCADE;
-CREATE INDEX pk_index ON public.public_key
-	USING btree
-	(
-	  hash
-	);
-- ddl-end --
-
-- object: dst_index | type: INDEX --
-- DROP INDEX IF EXISTS public.dst_index CASCADE;
-CREATE INDEX dst_index ON public."sessionRecord"
-	USING btree
-	(
-	  dst_ip
-	);
-- ddl-end --
-
-- object: path_index | type: INDEX --
-- DROP INDEX IF EXISTS public.path_index CASCADE;
-CREATE INDEX path_index ON public.certificate
-	USING gist
-	(
-	  cert_chain
-	)
-	WITH (BUFFERING = ON);
-- ddl-end --
-
-- object: "sessionRecord_fk" | type: CONSTRAINT --
-- ALTER TABLE public.fuzzy_hash DROP CONSTRAINT IF EXISTS "sessionRecord_fk" CASCADE;
-ALTER TABLE public.fuzzy_hash ADD CONSTRAINT "sessionRecord_fk" FOREIGN KEY ("id_sessionRecord")
-REFERENCES public."sessionRecord" (id) MATCH FULL
-ON DELETE SET NULL ON UPDATE CASCADE;
-- ddl-end --
-
-- object: plpython3u | type: LANGUAGE --
-- DROP LANGUAGE IF EXISTS plpython3u CASCADE;
-CREATE  LANGUAGE plpython3u;
-- ddl-end --
-ALTER LANGUAGE plpython3u OWNER TO postgres;
-- ddl-end --
-
-- object: public.tlshc | type: FUNCTION --
-- DROP FUNCTION IF EXISTS public.tlshc(text,text) CASCADE;
-CREATE FUNCTION public.tlshc ( a text,  b text)
-	RETURNS int4
-	LANGUAGE plpython3u
-	VOLATILE 
-	CALLED ON NULL INPUT
-	SECURITY INVOKER
-	COST 1
-	AS $$
-import tlsh
-return tlsh.diff(a, b)
-$$;
-- ddl-end --
-ALTER FUNCTION public.tlshc(text,text) OWNER TO postgres;
-- ddl-end --
-
-- object: public.tlsht | type: FUNCTION --
-- DROP FUNCTION IF EXISTS public.tlsht(IN text,IN text,IN int4,IN int4) CASCADE;
-CREATE FUNCTION public.tlsht (IN filter text, IN hash text, IN threshold int4, IN maxrows int4)
-	RETURNS SETOF public.fuzzy_hash
-	LANGUAGE plpython3u
-	IMMUTABLE LEAKPROOF
-	RETURNS NULL ON NULL INPUT
-	SECURITY INVOKER
-	COST 1
-	ROWS 1000
-	AS $$
-import tlsh
-plan = plpy.prepare("SELECT * FROM fuzzy_hash WHERE type <> $1", ["text"])
-rv = plan.execute(["filter"], maxrows)
-r = []
-for x in rv:
-    if tlsh.diff(x["value"], hash) < threshold:
-        r.append(x)
-return r
-$$;
-- ddl-end --
-ALTER FUNCTION public.tlsht(IN text,IN text,IN int4,IN int4) OWNER TO postgres;
-- ddl-end --
-

--- a/passivesslCreate.sql
+++ b/passivesslCreate.sql
@ -0,0 +1,6 @@
+CREATE DATABASE p2
+ 	ENCODING = 'UTF8'
+ 	LC_COLLATE = 'en_US.UTF-8'
+ 	LC_CTYPE = 'en_US.UTF-8'
+ 	TABLESPACE = pg_default
+ 	OWNER = postgres;
--- a/populate.sh
+++ b/populate.sh
@ -0,0 +1,5 @@
+#!/bin/bash
+# Set PGPASSWORD first
+ export PGPASSWORD=postgres
+psql -hlocalhost -p5432 -Upostgres -f passivesslCreate.sql
+psql -hlocalhost -p5432 -Upostgres -d p2 -f passivessl.sql
Author	SHA1	Message	Date
Jean-Louis Huynen	d5811ae08e	chg: [sql] remove unnecessary not null constraint	2020-11-18 16:16:46 +01:00
Jean-Louis Huynen	7a6f8505a7	chg: [sql] remove plpython3u clutter	2020-11-18 15:21:11 +01:00
Jean-Louis Huynen	779c204b1b	chg: [doc] not only pgsql10	2020-11-18 14:47:14 +01:00
Jean-Louis Huynen	7bffb828e7	chg [modules] going modular	2020-01-10 13:14:09 +01:00
Jean-Louis Huynen	ecb6fbf452	chg: [passivessl] import was borked	2019-11-25 10:21:19 +01:00
Jean-Louis Huynen	af4e61cf3d	add: [usability] script to execute passivessl.sql	2019-10-14 11:15:23 +02:00
Jean-Louis Huynen	13b163173d	chg: [db_schema] more info on certs, archives, + private in pubK	2019-10-14 11:14:27 +02:00
Jean-Louis Huynen	7367a5382e	add: [ingester] Tarball support	2019-09-19 15:46:38 +02:00
Jean-Louis Huynen	4e9cdc17a9	chg: [insert] brainfart	2019-09-18 09:37:39 +02:00
Jean-Louis Huynen	cd604827ce	chg: [parsing] correct more parsing errors	2019-09-17 16:33:58 +02:00
Jean-Louis Huynen	2c1a7a62b0	add: [install] create DB structure	2019-09-17 16:31:24 +02:00
Jean-Louis Huynen	0ec06e5b66	add: [importing] walk + DER + fix #4	2019-09-17 14:41:40 +02:00
Jean-Louis Huynen	33f0b333ae	fix: [#2 + sleep] correctly handle sleep	2019-06-12 16:18:12 +02:00
Jean-Louis Huynen	110cc29d18	chg: [exit] sleep for 30 seconds instead of exiting on emtpy queue	2019-06-12 15:59:28 +02:00
Jean-Louis Huynen	46dbd20e4b	fix: user: timez / timestampz typo in DB model	2019-04-26 14:40:36 +02:00