75 lines
2.8 KiB
Python
Executable File
75 lines
2.8 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
# -*- coding: utf-8 -*-
|
|
|
|
import zipfile
|
|
import json
|
|
|
|
from generator import download, download_to_file, get_abspath_list_file, get_version
|
|
|
|
|
|
def process(file, warninglist, dst, limit='1k'):
|
|
|
|
with zipfile.ZipFile(file, 'r') as cisco_lists:
|
|
for name in cisco_lists.namelist():
|
|
if name == "top-1m.csv":
|
|
with cisco_lists.open(name) as cisco_list:
|
|
if limit == '1k':
|
|
top = cisco_list.readlines()[:1000]
|
|
elif limit == '5k':
|
|
top = cisco_list.readlines()[:5000]
|
|
elif limit == '10k':
|
|
top = cisco_list.readlines()[:10000]
|
|
elif limit == '20k':
|
|
top = cisco_list.readlines()[:20000]
|
|
else:
|
|
continue
|
|
|
|
warninglist['version'] = get_version()
|
|
warninglist['type'] = 'hostname'
|
|
warninglist['matching_attributes'] = ['hostname', 'domain', 'url', 'domain|ip']
|
|
warninglist['list'] = []
|
|
|
|
for site in top:
|
|
v = site.decode('UTF-8').split(',')[1]
|
|
warninglist['list'].append(v.strip().replace('\\r\\n',''))
|
|
warninglist['list'] = sorted(set(warninglist['list']))
|
|
|
|
with open(get_abspath_list_file(dst), 'w') as data_file:
|
|
json.dump(warninglist, data_file, indent=2, sort_keys=True)
|
|
data_file.write("\n")
|
|
|
|
|
|
if __name__ == '__main__':
|
|
cisco_url = "http://s3-us-west-1.amazonaws.com/umbrella-static/top-1m.csv.zip"
|
|
cisco_file = "cisco_top-1m.csv.zip"
|
|
|
|
download_to_file(cisco_url, cisco_file)
|
|
|
|
cisco_dst_1k = 'cisco_top1000'
|
|
cisco_1k_warninglist = {
|
|
'name': 'Top 1000 websites from Cisco Umbrella',
|
|
'description': 'Event contains one or more entries from the top 1000 of the most used websites (Cisco Umbrella).'
|
|
}
|
|
process(cisco_file, cisco_1k_warninglist, cisco_dst_1k, limit='1k')
|
|
|
|
cisco_dst_5k = 'cisco_top5k'
|
|
cisco_5k_warninglist = {
|
|
'name': 'Top 5000 websites from Cisco Umbrella',
|
|
'description': 'Event contains one or more entries from the top 5000 of the most used websites (Cisco Umbrella).'
|
|
}
|
|
process(cisco_file, cisco_5k_warninglist, cisco_dst_5k, limit='5k')
|
|
|
|
cisco_dst_10k = 'cisco_top10k'
|
|
cisco_10k_warninglist = {
|
|
'name': 'Top 10 000 websites from Cisco Umbrella',
|
|
'description': 'Event contains one or more entries from the top 10 000 of the most used websites (Cisco Umbrella).'
|
|
}
|
|
process(cisco_file, cisco_10k_warninglist, cisco_dst_10k, limit='10k')
|
|
|
|
cisco_dst_20k = 'cisco_top20k'
|
|
cisco_20k_warninglist = {
|
|
'name': 'Top 20 000 websites from Cisco Umbrella',
|
|
'description': 'Event contains one or more entries from the top 20 000 of the most used websites (Cisco Umbrella).'
|
|
}
|
|
process(cisco_file, cisco_20k_warninglist, cisco_dst_20k, limit='20k')
|