add script to generate warning list from Tranco (https://tranco-list.eu/)

pull/125/head
Trey Darley 2020-01-10 15:27:07 +01:00
parent 8b63c53191
commit 17952df91e
1 changed files with 39 additions and 0 deletions

39
tools/generate_tranco.py Executable file
View File

@ -0,0 +1,39 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import requests
import zipfile
import datetime
import json
tranco_url = 'https://tranco-list.eu/top-1m.csv.zip'
tranco_file = 'top-1m.csv.zip'
user_agent = {'User-agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:46.0) Gecko/20100101 Firefox/46.0'}
r = requests.get(tranco_url, headers=user_agent)
with open(tranco_file, 'wb') as fd:
for chunk in r.iter_content(4096):
fd.write(chunk)
with zipfile.ZipFile(tranco_file, 'r') as tranco_lists:
for name in tranco_lists.namelist():
if name == 'top-1m.csv':
with tranco_lists.open(name) as tranco:
sites = tranco.readlines()
else:
continue
tranco_warninglist = {}
version = int(datetime.date.today().strftime('%Y%m%d'))
tranco_warninglist['description'] = "Event contains one or more entries from the top 1,000,000 most-used sites (Tranco)."
d = datetime.datetime.now()
tranco_warninglist['version'] = version
tranco_warninglist['name'] = "Top 1,000,000 most-used sites from Tranco"
tranco_warninglist['type'] = 'hostname'
tranco_warninglist['list'] = []
tranco_warninglist['matching_attributes'] = ['hostname', 'domain', 'url', 'domain|ip']
for site in sites:
v = site.decode('UTF-8').split(',')[1]
tranco_warninglist['list'].append(v.rstrip())
tranco_warninglist['list'] = sorted(set(tranco_warninglist['list']))
print(json.dumps(tranco_warninglist))