From d66a51e53779ebc07fc83d5750362df7d11adb68 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Fri, 11 Jun 2021 20:55:48 +0200 Subject: [PATCH] new: [crl] Genreate domains and IPs directly from Mozilla intermediate list --- generate_all.sh | 2 +- lists/crl-hostname/list.json | 300 +++++++++++++++++++++++ lists/crl-ip-hostname/list.json | 392 ------------------------------- lists/crl-ip/list.json | 319 +++++++++++++++++++++++++ requirements.txt | 3 + tools/generate-crl-ip-domains.py | 116 +++++++++ tools/generate-crl-ip-list.py | 41 ---- 7 files changed, 739 insertions(+), 434 deletions(-) create mode 100644 lists/crl-hostname/list.json delete mode 100644 lists/crl-ip-hostname/list.json create mode 100644 lists/crl-ip/list.json create mode 100755 tools/generate-crl-ip-domains.py delete mode 100755 tools/generate-crl-ip-list.py diff --git a/generate_all.sh b/generate_all.sh index 5d83b6e..0d8d5d4 100755 --- a/generate_all.sh +++ b/generate_all.sh @@ -9,7 +9,7 @@ python3 generate-amazon-aws.py python3 generate-cisco.py python3 generate-cloudflare.py python3 generate-covid.py -python3 generate-crl-ip-list.py +python3 generate-crl-ip-domains.py python3 generate-disposal.py # TODO: Google page on Wikipedia does not exist anymore # Suggestion came to use a passivetotal whois search for org:Google LLC diff --git a/lists/crl-hostname/list.json b/lists/crl-hostname/list.json new file mode 100644 index 0000000..b1bab37 --- /dev/null +++ b/lists/crl-hostname/list.json @@ -0,0 +1,300 @@ +{ + "description": "Domains that belongs to CRL or OCSP", + "list": [ + "atospki", + "caps.fujixerox.co.jp", + "cdp-ldap.intranet.eon.com", + "cdp-ldap.intranet.uniper.energy", + "cdp.elektronicznypodpis.pl", + "cdp1.disig.sk", + "cdp1.pca.dfn.de", + "cdp1.public-trust.com", + "cdp2.disig.sk", + "cdp2.pca.dfn.de", + "cert.managedpki.com", + "certificates.godaddy.com", + "certificates.starfieldtech.com", + "certigna.ocsp.certigna.fr", + "certigna.ocsp.dhimyotis.com", + "certum.crl.sheca.com", + "ch.siemens.com", + "cl.siemens.com", + "cl.siemens.net", + "commercial.ocsp.identrust.com", + "corppki", + "crl-1.trust.teliasonera.com", + "crl-2.trust.teliasonera.com", + "crl-3.trust.teliasonera.com", + "crl-cpki.telekom.de", + "crl.acs.altech.co.za", + "crl.adacom.com", + "crl.affirmtrust.com", + "crl.anf.es", + "crl.buypass.no", + "crl.ca.pki.africa", + "crl.ca.vodafone.com", + "crl.camerfirma.com", + "crl.certigna.fr", + "crl.certsign.ro", + "crl.certum.pl", + "crl.cfca.com.cn", + "crl.chambersign.org", + "crl.comodo.net", + "crl.comodoca.com", + "crl.d-trust.net", + "crl.dhimyotis.com", + "crl.digicert-cn.com", + "crl.digicert-validation.com", + "crl.digicert.cn", + "crl.e-szigno.hu", + "crl.e-tugra.com", + "crl.eid.belgium.be", + "crl.emsign.com", + "crl.ensuredca.com", + "crl.entrust.net", + "crl.firmaprofesional.com", + "crl.gdca.com.cn", + "crl.global.sheca.com", + "crl.globalsign.com", + "crl.globalsign.net", + "crl.godaddy.com", + "crl.harica.gr", + "crl.identrust.com", + "crl.izenpe.com", + "crl.luxtrust.lu", + "crl.microsoft.com", + "crl.msctrustgate.com", + "crl.netsolssl.com", + "crl.omniroot.com", + "crl.pki.belgium.be", + "crl.pki.goog", + "crl.pkioverheid.nl", + "crl.quovadisglobal.com", + "crl.root-x1.letsencrypt.org", + "crl.rootca1.amazontrust.com", + "crl.rootca2.amazontrust.com", + "crl.rootca3.amazontrust.com", + "crl.rootca4.amazontrust.com", + "crl.rootg2.amazontrust.com", + "crl.sbca.telesec.de", + "crl.securetrust.com", + "crl.sslcom.cn", + "crl.starfieldtech.com", + "crl.swisssign.net", + "crl.symauth.jp", + "crl.trust-provider.com", + "crl.trustcor.ca", + "crl.trustwave.com", + "crl.usertrust.com", + "crl.verisign.co.jp", + "crl.verisign.com", + "crl.verisign.com.au", + "crl.ws.symantec.com", + "crl05.actalis.it", + "crl1.camerfirma.com", + "crl1.e-tugra.com", + "crl1.hongkongpost.gov.hk", + "crl1.netlock.hu", + "crl2.netlock.hu", + "crl3.digicert.com", + "crl3.netlock.hu", + "crl4.digicert.com", + "crls.ssl.com", + "crlv1.harica.gr", + "depo.kamusm.gov.tr", + "directory.d-trust.net", + "directory.s-trust.de", + "directory.swisssign.net", + "domorganisatieservicesocsp-g3.pkioverheid.nl", + "domserver2020ocsp.pkioverheid.nl", + "eca.hinet.net", + "eon-group-ca-2-2013.ocsp.d-trust.net", + "epki.com.tw", + "epscd.catcert.net", + "epscd2.catcert.net", + "evrootocsp.pkioverheid.nl", + "gold-ev-g2.ocsp.swisssign.net", + "grcl2.crl.telesec.de", + "grcl2.ocsp.telesec.de", + "httpcrl.trust.telia.com", + "isrg.trustid.ocsp.identrust.com", + "ldap-cpki.telekom.de", + "ldap.actalis.it", + "ldap.certsign.ro", + "ldap.identrust.com", + "ldap.sbca.telesec.de", + "ldap05.actalis.it", + "ldap2.sheca.com", + "ldapfnmt.cert.fnmt.es", + "mscrl.microsoft.com", + "o.ss2.us", + "ocsp-rca.navercorp.com", + "ocsp.accv.es", + "ocsp.affirmtrust.com", + "ocsp.anf.es", + "ocsp.buypass.com", + "ocsp.ca.pki.africa", + "ocsp.camerfirma.com", + "ocsp.catcert.cat", + "ocsp.certsign.ro", + "ocsp.cfca.com.cn", + "ocsp.comodoca.com", + "ocsp.comodoca2.com", + "ocsp.comodoca3.com", + "ocsp.comodoca4.com", + "ocsp.dcocsp.cn", + "ocsp.digicert-cn.com", + "ocsp.digicert-validation.com", + "ocsp.digicert.cn", + "ocsp.digicert.com", + "ocsp.e-tugra.com", + "ocsp.eca.hinet.net", + "ocsp.eid.belgium.be", + "ocsp.elektronicznypodpis.pl", + "ocsp.emsign.com", + "ocsp.ensuredca.com", + "ocsp.entrust.net", + "ocsp.firmaprofesional.com", + "ocsp.global.sheca.com", + "ocsp.globalsign.com", + "ocsp.globaltrust.eu", + "ocsp.godaddy.com", + "ocsp.harica.gr", + "ocsp.identrust.com", + "ocsp.izenpe.com", + "ocsp.netsolssl.com", + "ocsp.omniroot.com", + "ocsp.pca.dfn.de", + "ocsp.pki-services.siemens.com", + "ocsp.pki.goog", + "ocsp.quovadisglobal.com", + "ocsp.root-x1.letsencrypt.org", + "ocsp.root.cartaodecidadao.pt", + "ocsp.rootca1.amazontrust.com", + "ocsp.rootca2.amazontrust.com", + "ocsp.rootca3.amazontrust.com", + "ocsp.rootca4.amazontrust.com", + "ocsp.rootg2.amazontrust.com", + "ocsp.securetrust.com", + "ocsp.starfieldtech.com", + "ocsp.swisssign.net", + "ocsp.taica.com.tw", + "ocsp.telekom.de", + "ocsp.telesec.de", + "ocsp.trust-provider.com", + "ocsp.trust.telia.com", + "ocsp.trust.teliasonera.com", + "ocsp.trustcor.ca", + "ocsp.trustwave.com", + "ocsp.usertrust.com", + "ocsp.verisign.com", + "ocsp.wisekey.com", + "ocsp0336.telesec.de", + "ocsp04.telesec.de", + "ocsp05.actalis.it", + "ocsp1.hongkongpost.gov.hk", + "ocsp1.netlock.hu", + "ocsp2.gdca.com.cn", + "ocsp2.globalsign.com", + "ocsp2.netlock.hu", + "ocsp3.gdca.com.cn", + "ocsp3.netlock.hu", + "ocsp3.sheca.com", + "ocsp4.gdca.com.cn", + "ocsp5.gdca.com.cn", + "ocsp6.gdca.com.cn", + "ocspape.cert.fnmt.es", + "ocspfnmtrcmca.cert.fnmt.es", + "ocspfnmtssr.cert.fnmt.es", + "ocsps.ssl.com", + "ocspsslkoks1.kamusm.gov.tr", + "oneocsp.microsoft.com", + "onsitecrl.certisign.com.br", + "onsitecrl.niftetrust.com", + "onsitecrl.s-trust.de", + "onsitecrl.trustitalia.it", + "onsitecrl.trustwise.com", + "onsitecrl.verisign.com", + "pecs1.unisys.com", + "pki-crl.atos.net", + "pki-crl.symauth.com", + "pki-ldap.atos.net", + "pki-ocsp.atos.net", + "pki-ocsp.symauth.com", + "pki-ocsp.verisign.com", + "pki.cartaodecidadao.pt", + "pki.intranet.eon.com", + "pki.intranet.uniper.energy", + "pki.telesec.de", + "pki0336.telesec.de", + "pkicdp.uniperapps.com", + "pkildp.unisys.com", + "pkirep.unisys.com", + "platinum-g2.ocsp.swisssign.net", + "portal.actalis.it", + "public.ocsp.identrust.com", + "public.wisekey.com", + "rca.navercorp.com", + "repository.secomtrust.net", + "root-c3-ca2-2009.ocsp.d-trust.net", + "root-c3-ca2-ev-2009.ocsp.d-trust.net", + "root-ca-3-2013.ocsp.d-trust.net", + "rootca.twca.com.tw", + "rootca2009-crl1.e-szigno.hu", + "rootca2009-crl2.e-szigno.hu", + "rootca2009-crl3.e-szigno.hu", + "rootca2009-ocsp1.e-szigno.hu", + "rootca2009-ocsp2.e-szigno.hu", + "rootca2009-ocsp3.e-szigno.hu", + "rootca2017-crl1.e-szigno.hu", + "rootca2017-crl2.e-szigno.hu", + "rootca2017-crl3.e-szigno.hu", + "rootca2017-ocsp1.e-szigno.hu", + "rootca2017-ocsp2.e-szigno.hu", + "rootca2017-ocsp3.e-szigno.hu", + "rootcar2-ocsp.disig.sk", + "rootocsp-g3.pkioverheid.nl", + "rootocsp.twca.com.tw", + "rootocsp2009.e-szigno.hu", + "s.ss2.us", + "s.symcb.com", + "s.symcd.com", + "scrootca1.ocsp.secomtrust.net", + "scrootca2.ocsp.secomtrust.net", + "service.globaltrust.eu", + "servicios.firmaprofesional.com", + "ssl.taica.com.tw", + "sslcom.crl.certum.pl", + "sslcom.ocsp-certum.com", + "ssp-crl-ldap.verisign.com", + "ssp-crl.symauth.com", + "ssp-crl.verisign.com", + "ssp-ocsp.symauth.com", + "ssp-ocsp.verisign.com", + "subca.crl.certum.pl", + "subca.ocsp-certum.com", + "trustidcaas.ocsp.identrust.com", + "uispki.unisys.com", + "uniper-group-ca-2-2015.ocsp.d-trust.net", + "uniper-group-ca-3-2020.ocsp.d-trust.net", + "validation.identrust.com", + "www.accv.es", + "www.anf.es", + "www.cert.fnmt.es", + "www.certigna.fr", + "www.d-trust.net", + "www.dhimyotis.com", + "www.gdca.com.cn", + "www.microsoft.com", + "www2.public-trust.com", + "x1.c.lencr.org" + ], + "matching_attributes": [ + "hostname", + "domain", + "domain|ip" + ], + "name": "CRL and OCSP domains", + "type": "string", + "version": 20210612 +} diff --git a/lists/crl-ip-hostname/list.json b/lists/crl-ip-hostname/list.json deleted file mode 100644 index b387dcc..0000000 --- a/lists/crl-ip-hostname/list.json +++ /dev/null @@ -1,392 +0,0 @@ -{ - "description": "CRL Warninglist from threatstop (https://github.com/threatstop/crl-ocsp-whitelist/)", - "list": [ - "104.16.89.188", - "104.16.90.188", - "104.16.91.188", - "104.16.92.188", - "104.16.93.188", - "104.17.102.175", - "104.17.103.175", - "104.17.104.175", - "104.17.105.175", - "104.17.106.175", - "104.215.29.84", - "104.215.54.174", - "104.41.179.244", - "104.91.166.106", - "104.91.166.112", - "104.91.166.82", - "104.91.166.89", - "104.91.166.96", - "104.91.166.98", - "109.70.240.114", - "113.52.156.18", - "116.92.128.12", - "116.92.128.34", - "119.145.171.206", - "119.145.171.215", - "121.50.63.210", - "121.50.63.211", - "13.114.126.114", - "13.33.164.100", - "13.33.164.105", - "13.33.164.164", - "13.33.164.223", - "13.33.164.236", - "13.33.164.37", - "13.33.164.7", - "13.33.164.93", - "13.78.114.232", - "133.242.48.24", - "133.242.50.38", - "133.242.68.56", - "151.101.46.133", - "153.120.128.154", - "153.127.215.13", - "153.127.216.172", - "153.149.154.120", - "153.149.17.219", - "153.149.96.48", - "153.149.98.42", - "155.207.94.23", - "155.207.94.25", - "172.217.1.46", - "172.217.4.243", - "178.255.83.1", - "18.194.140.191", - "184.73.226.63", - "185.102.40.212", - "185.102.40.23", - "185.33.53.5", - "185.62.162.144", - "185.62.162.145", - "185.69.225.3", - "185.69.225.4", - "192.35.177.117", - "192.35.177.153", - "192.35.177.155", - "193.104.0.178", - "193.104.0.210", - "193.140.71.141", - "193.140.71.35", - "193.27.6.240", - "193.42.222.125", - "194.140.12.241", - "194.140.59.23", - "194.145.83.75", - "194.145.83.79", - "194.30.48.30", - "195.77.23.39", - "195.77.23.49", - "195.80.175.18", - "195.80.175.39", - "195.80.175.7", - "195.95.167.129", - "195.95.167.162", - "195.95.167.163", - "2001:4420:aa01:ff01:210:241:69:194", - "2001:4542:2064:7::1010", - "2001:4542:2064:7::1013", - "2001:559:19:5400::173e:e30b", - "2001:559:19:5400::173e:e319", - "2001:559:19:5400::173e:e361", - "2001:559:19:5400::173e:e36a", - "2001:559:19:5400::173e:e378", - "2001:559:19:5400::173e:e380", - "2001:559:19:5c96::201a", - "2001:559:19:5c98::201a", - "2001:559:19:6483::201a", - "2001:559:19:648f::201a", - "2001:559:19:e000::b854:f46a", - "2001:b031:1306:ff00::1010", - "2001:b031:1306:ff00::1013", - "202.32.255.81", - "202.32.255.82", - "210.151.42.156", - "210.241.69.194", - "210.71.154.56", - "210.74.41.123", - "210.74.41.181", - "212.142.249.49", - "212.175.187.26", - "212.175.187.27", - "212.175.187.59", - "212.31.61.102", - "212.31.61.106", - "213.162.193.244", - "213.162.193.245", - "213.229.84.216", - "213.61.227.196", - "216.58.216.78", - "217.150.144.194", - "217.150.144.200", - "217.150.144.202", - "217.170.186.113", - "217.170.186.115", - "219.127.237.69", - "219.87.64.165", - "219.87.64.186", - "23.215.104.10", - "23.215.104.113", - "23.215.104.16", - "23.215.104.19", - "23.215.104.27", - "23.215.104.35", - "23.215.104.49", - "23.215.104.65", - "23.215.105.96", - "23.34.78.114", - "23.4.43.27", - "23.5.251.27", - "23.54.187.27", - "23.62.227.64", - "23.62.227.72", - "23.62.227.9", - "2600:1407:21:2a1::1b01", - "2600:1407:21:2b3::1b01", - "2600:9000:2044:4800:3:6aa6:6180:21", - "2600:9000:2044:a200:3:6aa6:6180:21", - "2600:9000:2044:ae00:3:6aa6:6180:21", - "2600:9000:2044:bc00:3:6aa6:6180:21", - "2600:9000:2044:e200:3:6aa6:6180:21", - "2600:9000:2044:ec00:3:6aa6:6180:21", - "2600:9000:2044:f800:3:6aa6:6180:21", - "2600:9000:2044:fc00:3:6aa6:6180:21", - "2606:4700::6810:59bc", - "2606:4700::6810:5abc", - "2606:4700::6810:5bbc", - "2606:4700::6810:5cbc", - "2606:4700::6810:5dbc", - "2606:4700::6811:66af", - "2606:4700::6811:67af", - "2606:4700::6811:68af", - "2606:4700::6811:69af", - "2606:4700::6811:6aaf", - "2607:f8b0:4009:80d::200e", - "2607:f8b0:4009:815::2013", - "2607:f8b0:4009:816::200e", - "2620:108:700f::22d4:f675", - "2620:108:700f::22d6:45ab", - "2620:108:700f::3426:765e", - "2a00:17f0:1300:3285::2", - "2a00:17f0:1300:3285::3", - "2a02:1788:2fd::b2ff:5301", - "2a04:4e42:2c::645", - "2a04:4e42:b::645", - "35.163.43.72", - "46.137.168.218", - "46.137.183.10", - "46.29.101.81", - "46.29.101.82", - "46.29.101.83", - "46.29.101.84", - "50.63.243.228", - "50.63.243.229", - "50.63.243.230", - "52.207.77.222", - "52.219.73.78", - "52.222.217.106", - "52.222.217.144", - "52.222.217.59", - "52.222.217.88", - "52.239.142.228", - "54.199.233.192", - "59.106.216.193", - "60.250.3.135", - "60.250.3.156", - "61.114.186.157", - "61.203.134.55", - "62.96.224.138", - "66.225.197.197", - "72.21.91.29", - "80.79.96.210", - "80.79.96.44", - "82.223.54.157", - "86.109.121.18", - "88.87.212.233", - "88.87.212.243", - "91.120.239.74", - "91.121.147.17", - "91.194.146.110", - "91.198.11.52", - "91.198.11.79", - "91.198.11.87", - "91.83.236.157", - "93.92.105.115", - "93.92.105.23", - "aces.ocsp.identrust.com", - "cdn.d-trust-cloudcrl.net", - "cdp.elektronicznypodpis.pl", - "cdp1.disig.sk", - "cdp2.disig.sk", - "commercial.ocsp.identrust.com", - "crl-ssl.certificat2.com", - "crl.affirmtrust.com", - "crl.buypass.no", - "crl.camerfirma.com", - "crl.certsign.ro", - "crl.cfca.com.cn", - "crl.comodoca.com", - "crl.d-trust.net", - "crl.e-tugra.com", - "crl.entrust.net", - "crl.firmaprofesional.com", - "crl.gdca.com.cn", - "crl.globalsign.com", - "crl.godaddy.com", - "crl.igc-g3.certinomis.com", - "crl.infocert.it", - "crl.izenpe.com", - "crl.luxtrust.lu", - "crl.managedpki.com", - "crl.netsolssl.com", - "crl.pki.goog", - "crl.quovadisglobal.com", - "crl.sbca.telesec.de", - "crl.serverpass.telesec.de", - "crl.starfieldtech.com", - "crl.swisssign.net", - "crl.trust-provider.com", - "crl.trustcor.ca", - "crl.trustwave.com", - "crl.usertrust.com", - "crl09.actalis.it", - "crl1.camerfirma.com", - "crl1.e-tugra.com", - "crl1.hongkongpost.gov.hk", - "crl1.netlock.hu", - "crl2.firmaprofesional.com", - "crl2.netlock.hu", - "crl3.digicert.com", - "crl3.netlock.hu", - "crl4.digicert.com", - "crls.ssl.com", - "crlv1.harica.gr", - "depo.kamusm.gov.tr", - "epscd.catcert.net", - "ev.ocsp.quovadisglobal.com", - "ev2.ocsp.secomtrust.net", - "evcrl1.managedpki.com", - "evocsp1.managedpki.com", - "evsslocsp.twca.com.tw", - "fe.symcb.com", - "fe.symcd.com", - "fi.symcb.com", - "fi.symcd.com", - "fj.symcb.com", - "fj.symcd.com", - "g2ocsp.managedpki.com", - "g3ocsp.managedpki.com", - "gca.nat.gov.tw", - "gk.symcb.com", - "gk.symcd.com", - "gm.symcb.com", - "gm.symcd.com", - "gn.symcb.com", - "gn.symcd.com", - "gold-ev-g2.ocsp.swisssign.net", - "igc-g3.certinomis.com", - "jcsitlssignpublicca-ocsp.managedpki.ne.jp", - "ocsp-ssl.certificat2.com", - "ocsp.accv.es", - "ocsp.affirmtrust.com", - "ocsp.buypass.com", - "ocsp.buypass.no", - "ocsp.camerfirma.com", - "ocsp.catcert.cat", - "ocsp.certsign.ro", - "ocsp.cfca.com.cn", - "ocsp.comodoca.com", - "ocsp.digicert.com", - "ocsp.e-tugra.com", - "ocsp.entrust.net", - "ocsp.epki.external.trustcor.ca", - "ocsp.ev.hinet.net", - "ocsp.firmaprofesional.com", - "ocsp.godaddy.com", - "ocsp.harica.gr", - "ocsp.int-x3.letsencrypt.org", - "ocsp.izenpe.com", - "ocsp.netsolssl.com", - "ocsp.ovcf.ca3.infocert.it", - "ocsp.pki.goog", - "ocsp.quovadisglobal.com", - "ocsp.sca0a.amazontrust.com", - "ocsp.sca1a.amazontrust.com", - "ocsp.sca2a.amazontrust.com", - "ocsp.sca3a.amazontrust.com", - "ocsp.sca4a.amazontrust.com", - "ocsp.serverpass.telesec.de", - "ocsp.starfieldtech.com", - "ocsp.trust-provider.com", - "ocsp.trustcor.ca", - "ocsp.trustwave.com", - "ocsp.usertrust.com", - "ocsp.wisekey.com", - "ocsp03.sbca.telesec.de", - "ocsp09.actalis.it", - "ocsp1.hongkongpost.gov.hk", - "ocsp1.netlock.hu", - "ocsp1.trustisfps.com", - "ocsp2.globalsign.com", - "ocsp2.netlock.hu", - "ocsp2.wisekey.com", - "ocsp3.gdca.com.cn", - "ocsp3.netlock.hu", - "ocspap.cert.fnmt.es", - "ocsps.ssl.com", - "ocspssls1.kamusm.gov.tr", - "pki-crl.atos.net", - "pki-ocsp.atos.net", - "public.wisekey.com", - "repo1.secomtrust.net", - "repository.ev.hinet.net", - "rtcrl.managedpki.ne.jp", - "sh.symcb.com", - "sh.symcd.com", - "silver-server-g2.ocsp.swisssign.net", - "sn.symcb.com", - "sn.symcd.com", - "sr.symcb.com", - "sr.symcd.com", - "ss.symcb.com", - "ss.symcd.com", - "ssl-c3-ca1-2009.ocsp.d-trust.net", - "ssl-c3-ca1-ev-2009.ocsp.d-trust.net", - "ssl.ocsp.luxtrust.lu", - "sslca2014-crl1.e-szigno.hu", - "sslca2014-crl2.e-szigno.hu", - "sslca2014-crl3.e-szigno.hu", - "sslca2014-ocsp1.e-szigno.hu", - "sslca2014-ocsp2.e-szigno.hu", - "sslca2014-ocsp3.e-szigno.hu", - "sslserver.twca.com.tw", - "subcar2i2-ocsp.disig.sk", - "sureseries-crl.cybertrust.ne.jp", - "sureseries-ocsp.cybertrust.ne.jp", - "tf.symcb.com", - "tf.symcd.com", - "ti.symcb.com", - "ti.symcd.com", - "tq.symcb.com", - "tq.symcd.com", - "validation.identrust.com", - "www.accv.es", - "www.cert.fnmt.es", - "www.certinomis.com", - "www.certsign.ro", - "www.trustis.com" - ], - "matching_attributes": [ - "hostname", - "domain", - "ip-dst", - "ip-src", - "url", - "domain|ip" - ], - "name": "CRL Warninglist", - "type": "string", - "version": 20210604 -} diff --git a/lists/crl-ip/list.json b/lists/crl-ip/list.json new file mode 100644 index 0000000..f6d48c1 --- /dev/null +++ b/lists/crl-ip/list.json @@ -0,0 +1,319 @@ +{ + "description": "IP addresses that belongs to CRL or OCSP", + "list": [ + "10.55.52.11", + "100.24.223.135", + "103.140.139.132", + "104.18.20.226", + "104.18.21.226", + "104.89.32.83", + "104.89.37.9", + "107.162.183.49", + "109.197.245.4", + "109.70.240.125", + "109.70.240.128", + "109.70.240.130", + "116.92.128.12", + "116.92.128.37", + "117.25.133.185", + "117.25.156.164", + "120.82.199.11", + "120.82.199.6", + "122.228.74.136", + "122.228.74.138", + "122.228.95.142", + "122.228.95.183", + "125.209.222.101", + "125.209.222.102", + "13.32.11.154", + "13.32.11.157", + "13.32.11.164", + "13.32.11.176", + "13.32.11.185", + "13.32.11.218", + "13.32.11.229", + "13.32.11.230", + "13.32.11.33", + "13.32.11.60", + "13.32.11.63", + "13.32.11.71", + "13.32.2.121", + "13.32.2.32", + "13.32.2.37", + "13.32.2.59", + "13.32.2.62", + "13.32.2.63", + "13.32.2.72", + "13.32.2.73", + "13.32.2.74", + "13.32.2.92", + "13.32.2.94", + "14.143.1.164", + "151.139.128.14", + "152.199.19.160", + "155.207.94.23", + "155.207.94.25", + "172.217.23.227", + "174.138.99.83", + "180.168.84.131", + "180.168.84.137", + "182.76.145.36", + "184.51.10.83", + "185.33.53.5", + "185.62.162.145", + "185.69.225.3", + "192.124.249.22", + "192.124.249.23", + "192.124.249.24", + "192.124.249.31", + "192.124.249.36", + "192.124.249.41", + "192.35.177.153", + "192.35.177.23", + "192.35.177.69", + "193.104.0.116", + "193.104.0.178", + "193.104.0.184", + "193.104.0.210", + "193.140.71.142", + "193.140.71.35", + "193.17.0.203", + "193.17.0.208", + "193.174.13.106", + "193.174.13.86", + "193.27.6.217", + "193.27.6.240", + "193.42.222.125", + "194.138.20.140", + "194.138.21.194", + "194.138.21.32", + "194.140.12.241", + "194.140.59.23", + "194.145.83.75", + "194.145.83.94", + "194.237.208.172", + "194.237.208.174", + "194.252.124.241", + "194.55.113.71", + "194.55.116.61", + "195.77.23.39", + "195.77.23.41", + "195.77.23.49", + "195.80.175.17", + "195.80.175.39", + "195.80.175.7", + "195.95.167.161", + "195.95.167.162", + "195.95.167.163", + "196.43.243.143", + "200.219.128.77", + "2001:2030:0:6::50ef:9449", + "2001:2030:0:6::50ef:c810", + "2001:2030:0:6::50ef:c819", + "2001:2030:0:6::50ef:c81a", + "2001:2030:0:6::50ef:c828", + "2001:2030:0:6::50ef:c831", + "2001:4542:2064:7::1013", + "2001:4542:2064:7::2005", + "2001:4de0:ac19::1:b:1a", + "2001:4de0:ac19::1:b:1b", + "2001:4de0:ac19::1:b:2a", + "2001:4de0:ac19::1:b:2b", + "2001:4de0:ac19::1:b:3a", + "2001:4de0:ac19::1:b:3b", + "2001:638:714:2809:3::1", + "2001:638:714:2809:3::7", + "2001:648:2800:a94:155:207:94:23", + "2001:648:2800:a94:155:207:94:25", + "2001:b031:1306:ff00::1013", + "2001:b031:1306:ff00::2005", + "202.32.181.22", + "202.65.20.176", + "203.26.77.30", + "204.79.197.203", + "210.66.125.97", + "210.71.154.6", + "210.74.41.123", + "210.74.41.181", + "212.174.7.27", + "212.175.187.26", + "212.175.187.27", + "212.210.63.17", + "212.5.219.10", + "212.5.219.17", + "212.5.219.18", + "212.5.219.42", + "212.5.219.58", + "212.5.219.64", + "212.5.219.65", + "212.5.219.72", + "212.5.219.73", + "212.5.219.8", + "212.5.219.9", + "213.162.193.244", + "213.162.193.245", + "213.61.227.196", + "216.168.246.31", + "216.168.246.41", + "217.124.154.30", + "217.124.154.50", + "217.150.144.163", + "217.150.144.200", + "217.150.144.234", + "217.170.186.113", + "217.170.186.115", + "219.80.58.97", + "219.87.64.165", + "23.51.123.27", + "240e:f7:c010:106:3::3fc", + "2600:1f18:232d:c200:280b:13d7:3f1d:c9e6", + "2600:1f18:232d:c201:30ba:778a:fc78:3c4a", + "2600:1f18:232d:c202:28b9:3732:152e:5f29", + "2600:9000:206e:2800:1d:123a:d0c0:93a1", + "2600:9000:206e:4200:1d:123a:d0c0:93a1", + "2600:9000:206e:4e00:3:6aa6:6180:21", + "2600:9000:206e:6c00:3:6aa6:6180:21", + "2600:9000:206e:7e00:1d:123a:d0c0:93a1", + "2600:9000:206e:8600:3:6aa6:6180:21", + "2600:9000:206e:8a00:1d:123a:d0c0:93a1", + "2600:9000:206e:9600:1d:123a:d0c0:93a1", + "2600:9000:206e:a000:3:6aa6:6180:21", + "2600:9000:206e:a200:3:6aa6:6180:21", + "2600:9000:206e:ac00:1d:123a:d0c0:93a1", + "2600:9000:206e:bc00:3:6aa6:6180:21", + "2600:9000:206e:c800:1d:123a:d0c0:93a1", + "2600:9000:206e:d600:1d:123a:d0c0:93a1", + "2600:9000:206e:de00:3:6aa6:6180:21", + "2600:9000:206e:e800:3:6aa6:6180:21", + "2606:4700::6812:14e2", + "2606:4700::6812:15e2", + "2620:108:700f::22d2:a6e7", + "2620:108:700f::22d5:d07f", + "2620:108:700f::2353:356a", + "2620:108:700f::23a5:9612", + "2620:108:700f::23a5:eb9c", + "2620:108:700f::2ceb:b9d0", + "2620:108:700f::3427:5e5a", + "2620:108:700f::3428:b514", + "2620:108:700f::3429:fe62", + "2a00:12a8:1100:e::d405:db12", + "2a00:12a8:1100:e::d405:db2a", + "2a00:12a8:1100:e::d405:db41", + "2a00:12a8:1100:e::d405:db48", + "2a00:1450:4014:80d::2003", + "2a00:17f0:1300:3285::2", + "2a00:17f0:1300:3285::3", + "2a02:26f0:11a::5f65:171b", + "2a02:26f0:11a::5f65:17b8", + "2a02:26f0:11a::5f65:17e0", + "2a02:26f0:11a::5f65:17e9", + "2a02:26f0:11a::5f65:17f0", + "2a02:26f0:1700:1a3::201a", + "2a02:26f0:1700:1aa::201a", + "2a02:26f0:1700:1ab::356e", + "2a02:26f0:1700:1b3::356e", + "2a02:26f0:1700:380::21cc", + "2a02:26f0:1700:389::1b01", + "2a02:26f0:1700:38a::21cc", + "2a02:26f0:1700:38b::1b01", + "34.237.184.165", + "34.250.14.212", + "34.77.53.190", + "46.29.127.179", + "46.29.127.181", + "46.29.127.182", + "47.246.43.168", + "47.246.43.172", + "47.246.43.203", + "47.246.43.209", + "47.73.67.26", + "52.177.240.188", + "52.210.206.107", + "52.219.75.222", + "52.6.97.148", + "54.76.92.234", + "54.77.250.123", + "60.250.3.135", + "61.114.177.151", + "61.114.186.157", + "62.239.7.4", + "62.71.3.136", + "62.96.224.137", + "62.96.224.138", + "62.96.224.156", + "64.18.25.27", + "64.18.25.30", + "64.18.26.163", + "79.133.177.225", + "79.133.177.226", + "79.133.177.227", + "79.133.177.228", + "79.133.177.229", + "79.133.177.230", + "79.133.177.231", + "79.133.177.232", + "80.158.50.254", + "80.158.59.63", + "80.158.61.91", + "80.231.126.181", + "80.231.126.182", + "80.231.126.183", + "80.231.126.184", + "80.231.126.185", + "80.231.126.186", + "80.239.148.73", + "80.239.200.16", + "80.239.200.25", + "80.239.200.26", + "80.239.200.40", + "80.239.200.49", + "80.79.96.44", + "80.79.97.38", + "80.79.98.61", + "82.223.54.157", + "83.137.118.12", + "83.137.118.21", + "83.137.118.28", + "83.137.118.5", + "84.53.161.112", + "84.53.161.114", + "84.53.161.25", + "84.53.161.35", + "84.53.161.80", + "84.53.161.90", + "86.109.121.18", + "90.160.140.202", + "90.160.140.204", + "90.160.140.205", + "90.160.140.230", + "90.160.140.232", + "91.120.239.74", + "91.194.146.110", + "91.194.146.119", + "91.198.11.87", + "91.198.183.20", + "91.199.212.51", + "91.83.236.157", + "93.184.220.29", + "99.86.241.101", + "99.86.241.12", + "99.86.241.50", + "99.86.241.53", + "99.86.245.108", + "99.86.245.175", + "99.86.245.201", + "99.86.245.211", + "99.86.245.53", + "99.86.245.63", + "99.86.245.67", + "99.86.245.92" + ], + "matching_attributes": [ + "ip-src", + "ip-dst", + "domain|ip" + ], + "name": "CRL and OCSP IP addresses", + "type": "cidr", + "version": 20210612 +} diff --git a/requirements.txt b/requirements.txt index 6c4b5cd..dbb7dfe 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,6 @@ beautifulsoup4==4.9.1 pyOpenSSL==19.1.0 python-dateutil==2.8.1 requests==2.24.0 +dnspython +pyasn1 +pyasn1-modules diff --git a/tools/generate-crl-ip-domains.py b/tools/generate-crl-ip-domains.py new file mode 100755 index 0000000..1466742 --- /dev/null +++ b/tools/generate-crl-ip-domains.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +import csv +import logging +import multiprocessing.dummy +import urllib.parse +from OpenSSL.crypto import FILETYPE_PEM, load_certificate, X509 +from pyasn1.codec.der.decoder import decode as asn1_decoder +from pyasn1_modules.rfc2459 import CRLDistPointsSyntax, AuthorityInfoAccessSyntax +from typing import List, Set +from dns.resolver import Resolver, NoAnswer, NXDOMAIN +from dns.exception import Timeout +from generator import download_to_file, get_version, write_to_file, get_abspath_source_file + + +def get_domain(url: str) -> str: + return urllib.parse.urlparse(url).hostname + + +def get_crl_ocsp_domains(cert: X509) -> List[str]: + crl_ocsp_domains = [] + for i in range(0, cert.get_extension_count()): + extension = cert.get_extension(i) + short_name = extension.get_short_name() + if short_name == b'crlDistributionPoints': + decoded, _ = asn1_decoder(extension.get_data(), asn1Spec=CRLDistPointsSyntax()) + for crl in decoded: + for generalName in crl.getComponentByName('distributionPoint').getComponentByName('fullName'): + crl_url = generalName.getComponentByName('uniformResourceIdentifier') + domain = get_domain(str(crl_url)) + if domain: + crl_ocsp_domains.append(domain) + + elif short_name == b'authorityInfoAccess': + decoded, _ = asn1_decoder(extension.get_data(), asn1Spec=AuthorityInfoAccessSyntax()) + for section in decoded: + if str(section.getComponentByName('accessMethod')) == '1.3.6.1.5.5.7.48.1': # ocsp + ocsp_url = section.getComponentByName('accessLocation').getComponentByName( + 'uniformResourceIdentifier') + domain = get_domain(str(ocsp_url)) + if domain: + crl_ocsp_domains.append(domain) + + return crl_ocsp_domains + + +def get_ips_from_domain(domain: str) -> Set[str]: + resolver = Resolver() + resolver.timeout = 5 + resolver.lifetime = 5 + + ips = set() + + try: + for rdata in resolver.query(domain, 'A'): + ips.add(str(rdata)) + except (NoAnswer, NXDOMAIN, Timeout): + pass + try: + for rdata in resolver.query(domain, 'AAAA'): + ips.add(str(rdata)) + except (NoAnswer, NXDOMAIN, Timeout): + pass + + return ips + + +def get_ips_from_domains(domains) -> Set[str]: + resolver = Resolver() + resolver.timeout = 5 + resolver.lifetime = 5 + + p = multiprocessing.dummy.Pool(10) + ips = set() + for ips_for_domain in p.map(get_ips_from_domain, domains): + ips.update(ips_for_domain) + return ips + + +def process(file): + crl_ocsp_domains = set() + with open(get_abspath_source_file(file), 'r') as f_in: + for obj in csv.DictReader(f_in): + try: + pem = obj['PEM Info'].strip("'").replace('\r', '').replace('\n\n', '\n') + cert = load_certificate(FILETYPE_PEM, pem) + crl_ocsp_domains.update(get_crl_ocsp_domains(cert)) + except Exception: + logging.exception("Could not process certificate") + + warninglist = { + 'name': 'CRL and OCSP domains', + 'version': get_version(), + 'description': 'Domains that belongs to CRL or OCSP', + 'list': crl_ocsp_domains, + 'matching_attributes': ["hostname", "domain", "domain|ip"], + 'type': 'string', + } + write_to_file(warninglist, "crl-hostname") + + warninglist = { + 'name': 'CRL and OCSP IP addresses', + 'version': get_version(), + 'description': 'IP addresses that belongs to CRL or OCSP', + 'list': get_ips_from_domains(crl_ocsp_domains), + 'matching_attributes': ["ip-src", "ip-dst", "domain|ip"], + 'type': 'cidr', + } + write_to_file(warninglist, "crl-ip") + + +if __name__ == '__main__': + CA_known_intermediate_url = 'https://ccadb-public.secure.force.com/mozilla/PublicAllIntermediateCertsWithPEMCSV' + CA_known_intermediate_file = 'PublicAllIntermediateCertsWithPEMCSV.csv' + + download_to_file(CA_known_intermediate_url, CA_known_intermediate_file) + process(CA_known_intermediate_file) diff --git a/tools/generate-crl-ip-list.py b/tools/generate-crl-ip-list.py deleted file mode 100755 index 6c50a7c..0000000 --- a/tools/generate-crl-ip-list.py +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -from generator import download_to_file, get_version, write_to_file, get_abspath_source_file - - -def process(files, dst): - - warninglist = { - 'type': "string", - 'matching_attributes': ["hostname", "domain", "ip-dst", "ip-src", "url", "domain|ip"], - 'name': "CRL Warninglist", - 'version': get_version(), - 'description': "CRL Warninglist from threatstop (https://github.com/threatstop/crl-ocsp-whitelist/)", - 'list': [] - } - - for file in files: - with open(get_abspath_source_file(file), 'r') as f: - ips = f.readlines() - for ip in ips: - warninglist['list'].append(ip.strip()) - - write_to_file(warninglist, dst) - - -if __name__ == '__main__': - crl_ip_base_url = 'https://raw.githubusercontent.com/threatstop/crl-ocsp-whitelist/master/' - uri_list = ['crl-hostnames.txt', 'crl-ipv4.txt', 'crl-ipv6.txt', - 'ocsp-hostnames.txt', 'ocsp-ipv4.txt', 'ocsp-ipv6.txt'] - crl_ip_dst = 'crl-ip-hostname' - - to_process = list() - - for uri in uri_list: - url = crl_ip_base_url + uri - file = 'ocsp_{}'.format(uri) - download_to_file(url, file) - to_process.append(file) - - process(to_process, crl_ip_dst)