Merge pull request #185 from JakubOnderka/crl

new: [crl] Genreate domains and IPs directly from Mozilla
pull/189/head
Alexandre Dulaunoy 2021-06-13 07:35:42 +02:00 committed by GitHub
commit d9f61e32bf
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 739 additions and 434 deletions

View File

@ -9,7 +9,7 @@ python3 generate-amazon-aws.py
python3 generate-cisco.py
python3 generate-cloudflare.py
python3 generate-covid.py
python3 generate-crl-ip-list.py
python3 generate-crl-ip-domains.py
python3 generate-disposal.py
# TODO: Google page on Wikipedia does not exist anymore
# Suggestion came to use a passivetotal whois search for org:Google LLC

View File

@ -0,0 +1,300 @@
{
"description": "Domains that belongs to CRL or OCSP",
"list": [
"atospki",
"caps.fujixerox.co.jp",
"cdp-ldap.intranet.eon.com",
"cdp-ldap.intranet.uniper.energy",
"cdp.elektronicznypodpis.pl",
"cdp1.disig.sk",
"cdp1.pca.dfn.de",
"cdp1.public-trust.com",
"cdp2.disig.sk",
"cdp2.pca.dfn.de",
"cert.managedpki.com",
"certificates.godaddy.com",
"certificates.starfieldtech.com",
"certigna.ocsp.certigna.fr",
"certigna.ocsp.dhimyotis.com",
"certum.crl.sheca.com",
"ch.siemens.com",
"cl.siemens.com",
"cl.siemens.net",
"commercial.ocsp.identrust.com",
"corppki",
"crl-1.trust.teliasonera.com",
"crl-2.trust.teliasonera.com",
"crl-3.trust.teliasonera.com",
"crl-cpki.telekom.de",
"crl.acs.altech.co.za",
"crl.adacom.com",
"crl.affirmtrust.com",
"crl.anf.es",
"crl.buypass.no",
"crl.ca.pki.africa",
"crl.ca.vodafone.com",
"crl.camerfirma.com",
"crl.certigna.fr",
"crl.certsign.ro",
"crl.certum.pl",
"crl.cfca.com.cn",
"crl.chambersign.org",
"crl.comodo.net",
"crl.comodoca.com",
"crl.d-trust.net",
"crl.dhimyotis.com",
"crl.digicert-cn.com",
"crl.digicert-validation.com",
"crl.digicert.cn",
"crl.e-szigno.hu",
"crl.e-tugra.com",
"crl.eid.belgium.be",
"crl.emsign.com",
"crl.ensuredca.com",
"crl.entrust.net",
"crl.firmaprofesional.com",
"crl.gdca.com.cn",
"crl.global.sheca.com",
"crl.globalsign.com",
"crl.globalsign.net",
"crl.godaddy.com",
"crl.harica.gr",
"crl.identrust.com",
"crl.izenpe.com",
"crl.luxtrust.lu",
"crl.microsoft.com",
"crl.msctrustgate.com",
"crl.netsolssl.com",
"crl.omniroot.com",
"crl.pki.belgium.be",
"crl.pki.goog",
"crl.pkioverheid.nl",
"crl.quovadisglobal.com",
"crl.root-x1.letsencrypt.org",
"crl.rootca1.amazontrust.com",
"crl.rootca2.amazontrust.com",
"crl.rootca3.amazontrust.com",
"crl.rootca4.amazontrust.com",
"crl.rootg2.amazontrust.com",
"crl.sbca.telesec.de",
"crl.securetrust.com",
"crl.sslcom.cn",
"crl.starfieldtech.com",
"crl.swisssign.net",
"crl.symauth.jp",
"crl.trust-provider.com",
"crl.trustcor.ca",
"crl.trustwave.com",
"crl.usertrust.com",
"crl.verisign.co.jp",
"crl.verisign.com",
"crl.verisign.com.au",
"crl.ws.symantec.com",
"crl05.actalis.it",
"crl1.camerfirma.com",
"crl1.e-tugra.com",
"crl1.hongkongpost.gov.hk",
"crl1.netlock.hu",
"crl2.netlock.hu",
"crl3.digicert.com",
"crl3.netlock.hu",
"crl4.digicert.com",
"crls.ssl.com",
"crlv1.harica.gr",
"depo.kamusm.gov.tr",
"directory.d-trust.net",
"directory.s-trust.de",
"directory.swisssign.net",
"domorganisatieservicesocsp-g3.pkioverheid.nl",
"domserver2020ocsp.pkioverheid.nl",
"eca.hinet.net",
"eon-group-ca-2-2013.ocsp.d-trust.net",
"epki.com.tw",
"epscd.catcert.net",
"epscd2.catcert.net",
"evrootocsp.pkioverheid.nl",
"gold-ev-g2.ocsp.swisssign.net",
"grcl2.crl.telesec.de",
"grcl2.ocsp.telesec.de",
"httpcrl.trust.telia.com",
"isrg.trustid.ocsp.identrust.com",
"ldap-cpki.telekom.de",
"ldap.actalis.it",
"ldap.certsign.ro",
"ldap.identrust.com",
"ldap.sbca.telesec.de",
"ldap05.actalis.it",
"ldap2.sheca.com",
"ldapfnmt.cert.fnmt.es",
"mscrl.microsoft.com",
"o.ss2.us",
"ocsp-rca.navercorp.com",
"ocsp.accv.es",
"ocsp.affirmtrust.com",
"ocsp.anf.es",
"ocsp.buypass.com",
"ocsp.ca.pki.africa",
"ocsp.camerfirma.com",
"ocsp.catcert.cat",
"ocsp.certsign.ro",
"ocsp.cfca.com.cn",
"ocsp.comodoca.com",
"ocsp.comodoca2.com",
"ocsp.comodoca3.com",
"ocsp.comodoca4.com",
"ocsp.dcocsp.cn",
"ocsp.digicert-cn.com",
"ocsp.digicert-validation.com",
"ocsp.digicert.cn",
"ocsp.digicert.com",
"ocsp.e-tugra.com",
"ocsp.eca.hinet.net",
"ocsp.eid.belgium.be",
"ocsp.elektronicznypodpis.pl",
"ocsp.emsign.com",
"ocsp.ensuredca.com",
"ocsp.entrust.net",
"ocsp.firmaprofesional.com",
"ocsp.global.sheca.com",
"ocsp.globalsign.com",
"ocsp.globaltrust.eu",
"ocsp.godaddy.com",
"ocsp.harica.gr",
"ocsp.identrust.com",
"ocsp.izenpe.com",
"ocsp.netsolssl.com",
"ocsp.omniroot.com",
"ocsp.pca.dfn.de",
"ocsp.pki-services.siemens.com",
"ocsp.pki.goog",
"ocsp.quovadisglobal.com",
"ocsp.root-x1.letsencrypt.org",
"ocsp.root.cartaodecidadao.pt",
"ocsp.rootca1.amazontrust.com",
"ocsp.rootca2.amazontrust.com",
"ocsp.rootca3.amazontrust.com",
"ocsp.rootca4.amazontrust.com",
"ocsp.rootg2.amazontrust.com",
"ocsp.securetrust.com",
"ocsp.starfieldtech.com",
"ocsp.swisssign.net",
"ocsp.taica.com.tw",
"ocsp.telekom.de",
"ocsp.telesec.de",
"ocsp.trust-provider.com",
"ocsp.trust.telia.com",
"ocsp.trust.teliasonera.com",
"ocsp.trustcor.ca",
"ocsp.trustwave.com",
"ocsp.usertrust.com",
"ocsp.verisign.com",
"ocsp.wisekey.com",
"ocsp0336.telesec.de",
"ocsp04.telesec.de",
"ocsp05.actalis.it",
"ocsp1.hongkongpost.gov.hk",
"ocsp1.netlock.hu",
"ocsp2.gdca.com.cn",
"ocsp2.globalsign.com",
"ocsp2.netlock.hu",
"ocsp3.gdca.com.cn",
"ocsp3.netlock.hu",
"ocsp3.sheca.com",
"ocsp4.gdca.com.cn",
"ocsp5.gdca.com.cn",
"ocsp6.gdca.com.cn",
"ocspape.cert.fnmt.es",
"ocspfnmtrcmca.cert.fnmt.es",
"ocspfnmtssr.cert.fnmt.es",
"ocsps.ssl.com",
"ocspsslkoks1.kamusm.gov.tr",
"oneocsp.microsoft.com",
"onsitecrl.certisign.com.br",
"onsitecrl.niftetrust.com",
"onsitecrl.s-trust.de",
"onsitecrl.trustitalia.it",
"onsitecrl.trustwise.com",
"onsitecrl.verisign.com",
"pecs1.unisys.com",
"pki-crl.atos.net",
"pki-crl.symauth.com",
"pki-ldap.atos.net",
"pki-ocsp.atos.net",
"pki-ocsp.symauth.com",
"pki-ocsp.verisign.com",
"pki.cartaodecidadao.pt",
"pki.intranet.eon.com",
"pki.intranet.uniper.energy",
"pki.telesec.de",
"pki0336.telesec.de",
"pkicdp.uniperapps.com",
"pkildp.unisys.com",
"pkirep.unisys.com",
"platinum-g2.ocsp.swisssign.net",
"portal.actalis.it",
"public.ocsp.identrust.com",
"public.wisekey.com",
"rca.navercorp.com",
"repository.secomtrust.net",
"root-c3-ca2-2009.ocsp.d-trust.net",
"root-c3-ca2-ev-2009.ocsp.d-trust.net",
"root-ca-3-2013.ocsp.d-trust.net",
"rootca.twca.com.tw",
"rootca2009-crl1.e-szigno.hu",
"rootca2009-crl2.e-szigno.hu",
"rootca2009-crl3.e-szigno.hu",
"rootca2009-ocsp1.e-szigno.hu",
"rootca2009-ocsp2.e-szigno.hu",
"rootca2009-ocsp3.e-szigno.hu",
"rootca2017-crl1.e-szigno.hu",
"rootca2017-crl2.e-szigno.hu",
"rootca2017-crl3.e-szigno.hu",
"rootca2017-ocsp1.e-szigno.hu",
"rootca2017-ocsp2.e-szigno.hu",
"rootca2017-ocsp3.e-szigno.hu",
"rootcar2-ocsp.disig.sk",
"rootocsp-g3.pkioverheid.nl",
"rootocsp.twca.com.tw",
"rootocsp2009.e-szigno.hu",
"s.ss2.us",
"s.symcb.com",
"s.symcd.com",
"scrootca1.ocsp.secomtrust.net",
"scrootca2.ocsp.secomtrust.net",
"service.globaltrust.eu",
"servicios.firmaprofesional.com",
"ssl.taica.com.tw",
"sslcom.crl.certum.pl",
"sslcom.ocsp-certum.com",
"ssp-crl-ldap.verisign.com",
"ssp-crl.symauth.com",
"ssp-crl.verisign.com",
"ssp-ocsp.symauth.com",
"ssp-ocsp.verisign.com",
"subca.crl.certum.pl",
"subca.ocsp-certum.com",
"trustidcaas.ocsp.identrust.com",
"uispki.unisys.com",
"uniper-group-ca-2-2015.ocsp.d-trust.net",
"uniper-group-ca-3-2020.ocsp.d-trust.net",
"validation.identrust.com",
"www.accv.es",
"www.anf.es",
"www.cert.fnmt.es",
"www.certigna.fr",
"www.d-trust.net",
"www.dhimyotis.com",
"www.gdca.com.cn",
"www.microsoft.com",
"www2.public-trust.com",
"x1.c.lencr.org"
],
"matching_attributes": [
"hostname",
"domain",
"domain|ip"
],
"name": "CRL and OCSP domains",
"type": "string",
"version": 20210612
}

View File

@ -1,392 +0,0 @@
{
"description": "CRL Warninglist from threatstop (https://github.com/threatstop/crl-ocsp-whitelist/)",
"list": [
"104.16.89.188",
"104.16.90.188",
"104.16.91.188",
"104.16.92.188",
"104.16.93.188",
"104.17.102.175",
"104.17.103.175",
"104.17.104.175",
"104.17.105.175",
"104.17.106.175",
"104.215.29.84",
"104.215.54.174",
"104.41.179.244",
"104.91.166.106",
"104.91.166.112",
"104.91.166.82",
"104.91.166.89",
"104.91.166.96",
"104.91.166.98",
"109.70.240.114",
"113.52.156.18",
"116.92.128.12",
"116.92.128.34",
"119.145.171.206",
"119.145.171.215",
"121.50.63.210",
"121.50.63.211",
"13.114.126.114",
"13.33.164.100",
"13.33.164.105",
"13.33.164.164",
"13.33.164.223",
"13.33.164.236",
"13.33.164.37",
"13.33.164.7",
"13.33.164.93",
"13.78.114.232",
"133.242.48.24",
"133.242.50.38",
"133.242.68.56",
"151.101.46.133",
"153.120.128.154",
"153.127.215.13",
"153.127.216.172",
"153.149.154.120",
"153.149.17.219",
"153.149.96.48",
"153.149.98.42",
"155.207.94.23",
"155.207.94.25",
"172.217.1.46",
"172.217.4.243",
"178.255.83.1",
"18.194.140.191",
"184.73.226.63",
"185.102.40.212",
"185.102.40.23",
"185.33.53.5",
"185.62.162.144",
"185.62.162.145",
"185.69.225.3",
"185.69.225.4",
"192.35.177.117",
"192.35.177.153",
"192.35.177.155",
"193.104.0.178",
"193.104.0.210",
"193.140.71.141",
"193.140.71.35",
"193.27.6.240",
"193.42.222.125",
"194.140.12.241",
"194.140.59.23",
"194.145.83.75",
"194.145.83.79",
"194.30.48.30",
"195.77.23.39",
"195.77.23.49",
"195.80.175.18",
"195.80.175.39",
"195.80.175.7",
"195.95.167.129",
"195.95.167.162",
"195.95.167.163",
"2001:4420:aa01:ff01:210:241:69:194",
"2001:4542:2064:7::1010",
"2001:4542:2064:7::1013",
"2001:559:19:5400::173e:e30b",
"2001:559:19:5400::173e:e319",
"2001:559:19:5400::173e:e361",
"2001:559:19:5400::173e:e36a",
"2001:559:19:5400::173e:e378",
"2001:559:19:5400::173e:e380",
"2001:559:19:5c96::201a",
"2001:559:19:5c98::201a",
"2001:559:19:6483::201a",
"2001:559:19:648f::201a",
"2001:559:19:e000::b854:f46a",
"2001:b031:1306:ff00::1010",
"2001:b031:1306:ff00::1013",
"202.32.255.81",
"202.32.255.82",
"210.151.42.156",
"210.241.69.194",
"210.71.154.56",
"210.74.41.123",
"210.74.41.181",
"212.142.249.49",
"212.175.187.26",
"212.175.187.27",
"212.175.187.59",
"212.31.61.102",
"212.31.61.106",
"213.162.193.244",
"213.162.193.245",
"213.229.84.216",
"213.61.227.196",
"216.58.216.78",
"217.150.144.194",
"217.150.144.200",
"217.150.144.202",
"217.170.186.113",
"217.170.186.115",
"219.127.237.69",
"219.87.64.165",
"219.87.64.186",
"23.215.104.10",
"23.215.104.113",
"23.215.104.16",
"23.215.104.19",
"23.215.104.27",
"23.215.104.35",
"23.215.104.49",
"23.215.104.65",
"23.215.105.96",
"23.34.78.114",
"23.4.43.27",
"23.5.251.27",
"23.54.187.27",
"23.62.227.64",
"23.62.227.72",
"23.62.227.9",
"2600:1407:21:2a1::1b01",
"2600:1407:21:2b3::1b01",
"2600:9000:2044:4800:3:6aa6:6180:21",
"2600:9000:2044:a200:3:6aa6:6180:21",
"2600:9000:2044:ae00:3:6aa6:6180:21",
"2600:9000:2044:bc00:3:6aa6:6180:21",
"2600:9000:2044:e200:3:6aa6:6180:21",
"2600:9000:2044:ec00:3:6aa6:6180:21",
"2600:9000:2044:f800:3:6aa6:6180:21",
"2600:9000:2044:fc00:3:6aa6:6180:21",
"2606:4700::6810:59bc",
"2606:4700::6810:5abc",
"2606:4700::6810:5bbc",
"2606:4700::6810:5cbc",
"2606:4700::6810:5dbc",
"2606:4700::6811:66af",
"2606:4700::6811:67af",
"2606:4700::6811:68af",
"2606:4700::6811:69af",
"2606:4700::6811:6aaf",
"2607:f8b0:4009:80d::200e",
"2607:f8b0:4009:815::2013",
"2607:f8b0:4009:816::200e",
"2620:108:700f::22d4:f675",
"2620:108:700f::22d6:45ab",
"2620:108:700f::3426:765e",
"2a00:17f0:1300:3285::2",
"2a00:17f0:1300:3285::3",
"2a02:1788:2fd::b2ff:5301",
"2a04:4e42:2c::645",
"2a04:4e42:b::645",
"35.163.43.72",
"46.137.168.218",
"46.137.183.10",
"46.29.101.81",
"46.29.101.82",
"46.29.101.83",
"46.29.101.84",
"50.63.243.228",
"50.63.243.229",
"50.63.243.230",
"52.207.77.222",
"52.219.73.78",
"52.222.217.106",
"52.222.217.144",
"52.222.217.59",
"52.222.217.88",
"52.239.142.228",
"54.199.233.192",
"59.106.216.193",
"60.250.3.135",
"60.250.3.156",
"61.114.186.157",
"61.203.134.55",
"62.96.224.138",
"66.225.197.197",
"72.21.91.29",
"80.79.96.210",
"80.79.96.44",
"82.223.54.157",
"86.109.121.18",
"88.87.212.233",
"88.87.212.243",
"91.120.239.74",
"91.121.147.17",
"91.194.146.110",
"91.198.11.52",
"91.198.11.79",
"91.198.11.87",
"91.83.236.157",
"93.92.105.115",
"93.92.105.23",
"aces.ocsp.identrust.com",
"cdn.d-trust-cloudcrl.net",
"cdp.elektronicznypodpis.pl",
"cdp1.disig.sk",
"cdp2.disig.sk",
"commercial.ocsp.identrust.com",
"crl-ssl.certificat2.com",
"crl.affirmtrust.com",
"crl.buypass.no",
"crl.camerfirma.com",
"crl.certsign.ro",
"crl.cfca.com.cn",
"crl.comodoca.com",
"crl.d-trust.net",
"crl.e-tugra.com",
"crl.entrust.net",
"crl.firmaprofesional.com",
"crl.gdca.com.cn",
"crl.globalsign.com",
"crl.godaddy.com",
"crl.igc-g3.certinomis.com",
"crl.infocert.it",
"crl.izenpe.com",
"crl.luxtrust.lu",
"crl.managedpki.com",
"crl.netsolssl.com",
"crl.pki.goog",
"crl.quovadisglobal.com",
"crl.sbca.telesec.de",
"crl.serverpass.telesec.de",
"crl.starfieldtech.com",
"crl.swisssign.net",
"crl.trust-provider.com",
"crl.trustcor.ca",
"crl.trustwave.com",
"crl.usertrust.com",
"crl09.actalis.it",
"crl1.camerfirma.com",
"crl1.e-tugra.com",
"crl1.hongkongpost.gov.hk",
"crl1.netlock.hu",
"crl2.firmaprofesional.com",
"crl2.netlock.hu",
"crl3.digicert.com",
"crl3.netlock.hu",
"crl4.digicert.com",
"crls.ssl.com",
"crlv1.harica.gr",
"depo.kamusm.gov.tr",
"epscd.catcert.net",
"ev.ocsp.quovadisglobal.com",
"ev2.ocsp.secomtrust.net",
"evcrl1.managedpki.com",
"evocsp1.managedpki.com",
"evsslocsp.twca.com.tw",
"fe.symcb.com",
"fe.symcd.com",
"fi.symcb.com",
"fi.symcd.com",
"fj.symcb.com",
"fj.symcd.com",
"g2ocsp.managedpki.com",
"g3ocsp.managedpki.com",
"gca.nat.gov.tw",
"gk.symcb.com",
"gk.symcd.com",
"gm.symcb.com",
"gm.symcd.com",
"gn.symcb.com",
"gn.symcd.com",
"gold-ev-g2.ocsp.swisssign.net",
"igc-g3.certinomis.com",
"jcsitlssignpublicca-ocsp.managedpki.ne.jp",
"ocsp-ssl.certificat2.com",
"ocsp.accv.es",
"ocsp.affirmtrust.com",
"ocsp.buypass.com",
"ocsp.buypass.no",
"ocsp.camerfirma.com",
"ocsp.catcert.cat",
"ocsp.certsign.ro",
"ocsp.cfca.com.cn",
"ocsp.comodoca.com",
"ocsp.digicert.com",
"ocsp.e-tugra.com",
"ocsp.entrust.net",
"ocsp.epki.external.trustcor.ca",
"ocsp.ev.hinet.net",
"ocsp.firmaprofesional.com",
"ocsp.godaddy.com",
"ocsp.harica.gr",
"ocsp.int-x3.letsencrypt.org",
"ocsp.izenpe.com",
"ocsp.netsolssl.com",
"ocsp.ovcf.ca3.infocert.it",
"ocsp.pki.goog",
"ocsp.quovadisglobal.com",
"ocsp.sca0a.amazontrust.com",
"ocsp.sca1a.amazontrust.com",
"ocsp.sca2a.amazontrust.com",
"ocsp.sca3a.amazontrust.com",
"ocsp.sca4a.amazontrust.com",
"ocsp.serverpass.telesec.de",
"ocsp.starfieldtech.com",
"ocsp.trust-provider.com",
"ocsp.trustcor.ca",
"ocsp.trustwave.com",
"ocsp.usertrust.com",
"ocsp.wisekey.com",
"ocsp03.sbca.telesec.de",
"ocsp09.actalis.it",
"ocsp1.hongkongpost.gov.hk",
"ocsp1.netlock.hu",
"ocsp1.trustisfps.com",
"ocsp2.globalsign.com",
"ocsp2.netlock.hu",
"ocsp2.wisekey.com",
"ocsp3.gdca.com.cn",
"ocsp3.netlock.hu",
"ocspap.cert.fnmt.es",
"ocsps.ssl.com",
"ocspssls1.kamusm.gov.tr",
"pki-crl.atos.net",
"pki-ocsp.atos.net",
"public.wisekey.com",
"repo1.secomtrust.net",
"repository.ev.hinet.net",
"rtcrl.managedpki.ne.jp",
"sh.symcb.com",
"sh.symcd.com",
"silver-server-g2.ocsp.swisssign.net",
"sn.symcb.com",
"sn.symcd.com",
"sr.symcb.com",
"sr.symcd.com",
"ss.symcb.com",
"ss.symcd.com",
"ssl-c3-ca1-2009.ocsp.d-trust.net",
"ssl-c3-ca1-ev-2009.ocsp.d-trust.net",
"ssl.ocsp.luxtrust.lu",
"sslca2014-crl1.e-szigno.hu",
"sslca2014-crl2.e-szigno.hu",
"sslca2014-crl3.e-szigno.hu",
"sslca2014-ocsp1.e-szigno.hu",
"sslca2014-ocsp2.e-szigno.hu",
"sslca2014-ocsp3.e-szigno.hu",
"sslserver.twca.com.tw",
"subcar2i2-ocsp.disig.sk",
"sureseries-crl.cybertrust.ne.jp",
"sureseries-ocsp.cybertrust.ne.jp",
"tf.symcb.com",
"tf.symcd.com",
"ti.symcb.com",
"ti.symcd.com",
"tq.symcb.com",
"tq.symcd.com",
"validation.identrust.com",
"www.accv.es",
"www.cert.fnmt.es",
"www.certinomis.com",
"www.certsign.ro",
"www.trustis.com"
],
"matching_attributes": [
"hostname",
"domain",
"ip-dst",
"ip-src",
"url",
"domain|ip"
],
"name": "CRL Warninglist",
"type": "string",
"version": 20210604
}

319
lists/crl-ip/list.json Normal file
View File

@ -0,0 +1,319 @@
{
"description": "IP addresses that belongs to CRL or OCSP",
"list": [
"10.55.52.11",
"100.24.223.135",
"103.140.139.132",
"104.18.20.226",
"104.18.21.226",
"104.89.32.83",
"104.89.37.9",
"107.162.183.49",
"109.197.245.4",
"109.70.240.125",
"109.70.240.128",
"109.70.240.130",
"116.92.128.12",
"116.92.128.37",
"117.25.133.185",
"117.25.156.164",
"120.82.199.11",
"120.82.199.6",
"122.228.74.136",
"122.228.74.138",
"122.228.95.142",
"122.228.95.183",
"125.209.222.101",
"125.209.222.102",
"13.32.11.154",
"13.32.11.157",
"13.32.11.164",
"13.32.11.176",
"13.32.11.185",
"13.32.11.218",
"13.32.11.229",
"13.32.11.230",
"13.32.11.33",
"13.32.11.60",
"13.32.11.63",
"13.32.11.71",
"13.32.2.121",
"13.32.2.32",
"13.32.2.37",
"13.32.2.59",
"13.32.2.62",
"13.32.2.63",
"13.32.2.72",
"13.32.2.73",
"13.32.2.74",
"13.32.2.92",
"13.32.2.94",
"14.143.1.164",
"151.139.128.14",
"152.199.19.160",
"155.207.94.23",
"155.207.94.25",
"172.217.23.227",
"174.138.99.83",
"180.168.84.131",
"180.168.84.137",
"182.76.145.36",
"184.51.10.83",
"185.33.53.5",
"185.62.162.145",
"185.69.225.3",
"192.124.249.22",
"192.124.249.23",
"192.124.249.24",
"192.124.249.31",
"192.124.249.36",
"192.124.249.41",
"192.35.177.153",
"192.35.177.23",
"192.35.177.69",
"193.104.0.116",
"193.104.0.178",
"193.104.0.184",
"193.104.0.210",
"193.140.71.142",
"193.140.71.35",
"193.17.0.203",
"193.17.0.208",
"193.174.13.106",
"193.174.13.86",
"193.27.6.217",
"193.27.6.240",
"193.42.222.125",
"194.138.20.140",
"194.138.21.194",
"194.138.21.32",
"194.140.12.241",
"194.140.59.23",
"194.145.83.75",
"194.145.83.94",
"194.237.208.172",
"194.237.208.174",
"194.252.124.241",
"194.55.113.71",
"194.55.116.61",
"195.77.23.39",
"195.77.23.41",
"195.77.23.49",
"195.80.175.17",
"195.80.175.39",
"195.80.175.7",
"195.95.167.161",
"195.95.167.162",
"195.95.167.163",
"196.43.243.143",
"200.219.128.77",
"2001:2030:0:6::50ef:9449",
"2001:2030:0:6::50ef:c810",
"2001:2030:0:6::50ef:c819",
"2001:2030:0:6::50ef:c81a",
"2001:2030:0:6::50ef:c828",
"2001:2030:0:6::50ef:c831",
"2001:4542:2064:7::1013",
"2001:4542:2064:7::2005",
"2001:4de0:ac19::1:b:1a",
"2001:4de0:ac19::1:b:1b",
"2001:4de0:ac19::1:b:2a",
"2001:4de0:ac19::1:b:2b",
"2001:4de0:ac19::1:b:3a",
"2001:4de0:ac19::1:b:3b",
"2001:638:714:2809:3::1",
"2001:638:714:2809:3::7",
"2001:648:2800:a94:155:207:94:23",
"2001:648:2800:a94:155:207:94:25",
"2001:b031:1306:ff00::1013",
"2001:b031:1306:ff00::2005",
"202.32.181.22",
"202.65.20.176",
"203.26.77.30",
"204.79.197.203",
"210.66.125.97",
"210.71.154.6",
"210.74.41.123",
"210.74.41.181",
"212.174.7.27",
"212.175.187.26",
"212.175.187.27",
"212.210.63.17",
"212.5.219.10",
"212.5.219.17",
"212.5.219.18",
"212.5.219.42",
"212.5.219.58",
"212.5.219.64",
"212.5.219.65",
"212.5.219.72",
"212.5.219.73",
"212.5.219.8",
"212.5.219.9",
"213.162.193.244",
"213.162.193.245",
"213.61.227.196",
"216.168.246.31",
"216.168.246.41",
"217.124.154.30",
"217.124.154.50",
"217.150.144.163",
"217.150.144.200",
"217.150.144.234",
"217.170.186.113",
"217.170.186.115",
"219.80.58.97",
"219.87.64.165",
"23.51.123.27",
"240e:f7:c010:106:3::3fc",
"2600:1f18:232d:c200:280b:13d7:3f1d:c9e6",
"2600:1f18:232d:c201:30ba:778a:fc78:3c4a",
"2600:1f18:232d:c202:28b9:3732:152e:5f29",
"2600:9000:206e:2800:1d:123a:d0c0:93a1",
"2600:9000:206e:4200:1d:123a:d0c0:93a1",
"2600:9000:206e:4e00:3:6aa6:6180:21",
"2600:9000:206e:6c00:3:6aa6:6180:21",
"2600:9000:206e:7e00:1d:123a:d0c0:93a1",
"2600:9000:206e:8600:3:6aa6:6180:21",
"2600:9000:206e:8a00:1d:123a:d0c0:93a1",
"2600:9000:206e:9600:1d:123a:d0c0:93a1",
"2600:9000:206e:a000:3:6aa6:6180:21",
"2600:9000:206e:a200:3:6aa6:6180:21",
"2600:9000:206e:ac00:1d:123a:d0c0:93a1",
"2600:9000:206e:bc00:3:6aa6:6180:21",
"2600:9000:206e:c800:1d:123a:d0c0:93a1",
"2600:9000:206e:d600:1d:123a:d0c0:93a1",
"2600:9000:206e:de00:3:6aa6:6180:21",
"2600:9000:206e:e800:3:6aa6:6180:21",
"2606:4700::6812:14e2",
"2606:4700::6812:15e2",
"2620:108:700f::22d2:a6e7",
"2620:108:700f::22d5:d07f",
"2620:108:700f::2353:356a",
"2620:108:700f::23a5:9612",
"2620:108:700f::23a5:eb9c",
"2620:108:700f::2ceb:b9d0",
"2620:108:700f::3427:5e5a",
"2620:108:700f::3428:b514",
"2620:108:700f::3429:fe62",
"2a00:12a8:1100:e::d405:db12",
"2a00:12a8:1100:e::d405:db2a",
"2a00:12a8:1100:e::d405:db41",
"2a00:12a8:1100:e::d405:db48",
"2a00:1450:4014:80d::2003",
"2a00:17f0:1300:3285::2",
"2a00:17f0:1300:3285::3",
"2a02:26f0:11a::5f65:171b",
"2a02:26f0:11a::5f65:17b8",
"2a02:26f0:11a::5f65:17e0",
"2a02:26f0:11a::5f65:17e9",
"2a02:26f0:11a::5f65:17f0",
"2a02:26f0:1700:1a3::201a",
"2a02:26f0:1700:1aa::201a",
"2a02:26f0:1700:1ab::356e",
"2a02:26f0:1700:1b3::356e",
"2a02:26f0:1700:380::21cc",
"2a02:26f0:1700:389::1b01",
"2a02:26f0:1700:38a::21cc",
"2a02:26f0:1700:38b::1b01",
"34.237.184.165",
"34.250.14.212",
"34.77.53.190",
"46.29.127.179",
"46.29.127.181",
"46.29.127.182",
"47.246.43.168",
"47.246.43.172",
"47.246.43.203",
"47.246.43.209",
"47.73.67.26",
"52.177.240.188",
"52.210.206.107",
"52.219.75.222",
"52.6.97.148",
"54.76.92.234",
"54.77.250.123",
"60.250.3.135",
"61.114.177.151",
"61.114.186.157",
"62.239.7.4",
"62.71.3.136",
"62.96.224.137",
"62.96.224.138",
"62.96.224.156",
"64.18.25.27",
"64.18.25.30",
"64.18.26.163",
"79.133.177.225",
"79.133.177.226",
"79.133.177.227",
"79.133.177.228",
"79.133.177.229",
"79.133.177.230",
"79.133.177.231",
"79.133.177.232",
"80.158.50.254",
"80.158.59.63",
"80.158.61.91",
"80.231.126.181",
"80.231.126.182",
"80.231.126.183",
"80.231.126.184",
"80.231.126.185",
"80.231.126.186",
"80.239.148.73",
"80.239.200.16",
"80.239.200.25",
"80.239.200.26",
"80.239.200.40",
"80.239.200.49",
"80.79.96.44",
"80.79.97.38",
"80.79.98.61",
"82.223.54.157",
"83.137.118.12",
"83.137.118.21",
"83.137.118.28",
"83.137.118.5",
"84.53.161.112",
"84.53.161.114",
"84.53.161.25",
"84.53.161.35",
"84.53.161.80",
"84.53.161.90",
"86.109.121.18",
"90.160.140.202",
"90.160.140.204",
"90.160.140.205",
"90.160.140.230",
"90.160.140.232",
"91.120.239.74",
"91.194.146.110",
"91.194.146.119",
"91.198.11.87",
"91.198.183.20",
"91.199.212.51",
"91.83.236.157",
"93.184.220.29",
"99.86.241.101",
"99.86.241.12",
"99.86.241.50",
"99.86.241.53",
"99.86.245.108",
"99.86.245.175",
"99.86.245.201",
"99.86.245.211",
"99.86.245.53",
"99.86.245.63",
"99.86.245.67",
"99.86.245.92"
],
"matching_attributes": [
"ip-src",
"ip-dst",
"domain|ip"
],
"name": "CRL and OCSP IP addresses",
"type": "cidr",
"version": 20210612
}

View File

@ -2,3 +2,6 @@ beautifulsoup4==4.9.1
pyOpenSSL==19.1.0
python-dateutil==2.8.1
requests==2.24.0
dnspython
pyasn1
pyasn1-modules

116
tools/generate-crl-ip-domains.py Executable file
View File

@ -0,0 +1,116 @@
#!/usr/bin/env python3
import csv
import logging
import multiprocessing.dummy
import urllib.parse
from OpenSSL.crypto import FILETYPE_PEM, load_certificate, X509
from pyasn1.codec.der.decoder import decode as asn1_decoder
from pyasn1_modules.rfc2459 import CRLDistPointsSyntax, AuthorityInfoAccessSyntax
from typing import List, Set
from dns.resolver import Resolver, NoAnswer, NXDOMAIN
from dns.exception import Timeout
from generator import download_to_file, get_version, write_to_file, get_abspath_source_file
def get_domain(url: str) -> str:
return urllib.parse.urlparse(url).hostname
def get_crl_ocsp_domains(cert: X509) -> List[str]:
crl_ocsp_domains = []
for i in range(0, cert.get_extension_count()):
extension = cert.get_extension(i)
short_name = extension.get_short_name()
if short_name == b'crlDistributionPoints':
decoded, _ = asn1_decoder(extension.get_data(), asn1Spec=CRLDistPointsSyntax())
for crl in decoded:
for generalName in crl.getComponentByName('distributionPoint').getComponentByName('fullName'):
crl_url = generalName.getComponentByName('uniformResourceIdentifier')
domain = get_domain(str(crl_url))
if domain:
crl_ocsp_domains.append(domain)
elif short_name == b'authorityInfoAccess':
decoded, _ = asn1_decoder(extension.get_data(), asn1Spec=AuthorityInfoAccessSyntax())
for section in decoded:
if str(section.getComponentByName('accessMethod')) == '1.3.6.1.5.5.7.48.1': # ocsp
ocsp_url = section.getComponentByName('accessLocation').getComponentByName(
'uniformResourceIdentifier')
domain = get_domain(str(ocsp_url))
if domain:
crl_ocsp_domains.append(domain)
return crl_ocsp_domains
def get_ips_from_domain(domain: str) -> Set[str]:
resolver = Resolver()
resolver.timeout = 5
resolver.lifetime = 5
ips = set()
try:
for rdata in resolver.query(domain, 'A'):
ips.add(str(rdata))
except (NoAnswer, NXDOMAIN, Timeout):
pass
try:
for rdata in resolver.query(domain, 'AAAA'):
ips.add(str(rdata))
except (NoAnswer, NXDOMAIN, Timeout):
pass
return ips
def get_ips_from_domains(domains) -> Set[str]:
resolver = Resolver()
resolver.timeout = 5
resolver.lifetime = 5
p = multiprocessing.dummy.Pool(10)
ips = set()
for ips_for_domain in p.map(get_ips_from_domain, domains):
ips.update(ips_for_domain)
return ips
def process(file):
crl_ocsp_domains = set()
with open(get_abspath_source_file(file), 'r') as f_in:
for obj in csv.DictReader(f_in):
try:
pem = obj['PEM Info'].strip("'").replace('\r', '').replace('\n\n', '\n')
cert = load_certificate(FILETYPE_PEM, pem)
crl_ocsp_domains.update(get_crl_ocsp_domains(cert))
except Exception:
logging.exception("Could not process certificate")
warninglist = {
'name': 'CRL and OCSP domains',
'version': get_version(),
'description': 'Domains that belongs to CRL or OCSP',
'list': crl_ocsp_domains,
'matching_attributes': ["hostname", "domain", "domain|ip"],
'type': 'string',
}
write_to_file(warninglist, "crl-hostname")
warninglist = {
'name': 'CRL and OCSP IP addresses',
'version': get_version(),
'description': 'IP addresses that belongs to CRL or OCSP',
'list': get_ips_from_domains(crl_ocsp_domains),
'matching_attributes': ["ip-src", "ip-dst", "domain|ip"],
'type': 'cidr',
}
write_to_file(warninglist, "crl-ip")
if __name__ == '__main__':
CA_known_intermediate_url = 'https://ccadb-public.secure.force.com/mozilla/PublicAllIntermediateCertsWithPEMCSV'
CA_known_intermediate_file = 'PublicAllIntermediateCertsWithPEMCSV.csv'
download_to_file(CA_known_intermediate_url, CA_known_intermediate_file)
process(CA_known_intermediate_file)

View File

@ -1,41 +0,0 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
from generator import download_to_file, get_version, write_to_file, get_abspath_source_file
def process(files, dst):
warninglist = {
'type': "string",
'matching_attributes': ["hostname", "domain", "ip-dst", "ip-src", "url", "domain|ip"],
'name': "CRL Warninglist",
'version': get_version(),
'description': "CRL Warninglist from threatstop (https://github.com/threatstop/crl-ocsp-whitelist/)",
'list': []
}
for file in files:
with open(get_abspath_source_file(file), 'r') as f:
ips = f.readlines()
for ip in ips:
warninglist['list'].append(ip.strip())
write_to_file(warninglist, dst)
if __name__ == '__main__':
crl_ip_base_url = 'https://raw.githubusercontent.com/threatstop/crl-ocsp-whitelist/master/'
uri_list = ['crl-hostnames.txt', 'crl-ipv4.txt', 'crl-ipv6.txt',
'ocsp-hostnames.txt', 'ocsp-ipv4.txt', 'ocsp-ipv6.txt']
crl_ip_dst = 'crl-ip-hostname'
to_process = list()
for uri in uri_list:
url = crl_ip_base_url + uri
file = 'ocsp_{}'.format(uri)
download_to_file(url, file)
to_process.append(file)
process(to_process, crl_ip_dst)