MISP
/
misp-warninglists
mirror of https://github.com/MISP/misp-warninglists
2
0
Fork 0
Code Issues Releases Wiki Activity

add check if downloaded file has changed on server before downloading

pull/154/head
Kevin Holvoet 2020-07-21 09:11:51 +02:00
parent d3e87dc7ae
commit 9dc6ea9ca8
1 changed files with 14 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
tools/generator.py
View File

@ -7,15 +7,22 @@ from inspect import currentframe, getframeinfo
from os import path from os import path
import requests import requests
from dateutil.parser import parse as parsedate
def download_to_file(url, file): def download_to_file(url, file):
user_agent = { user_agent = {
"User-agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:46.0) Gecko/20100101 Firefox/46.0"} "User-agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:46.0) Gecko/20100101 Firefox/46.0"}
r = requests.get(url, headers=user_agent) r = requests.head(url, headers=user_agent)
with open(file, 'wb') as fd: url_datetime = parsedate(r.headers['Last-Modified']).astimezone()
for chunk in r.iter_content(4096): file_datetime = datetime.datetime.fromtimestamp(
fd.write(chunk) path.getmtime(file)).astimezone()
if(url_datetime > file_datetime):
r = requests.get(url, headers=user_agent)
with open(file, 'wb') as fd:
for chunk in r.iter_content(4096):
fd.write(chunk)
def download(url): def download(url):
@ -40,7 +47,9 @@ def unique_sorted_warninglist(warninglist):
warninglist['list'] = sorted(set(warninglist['list'])) warninglist['list'] = sorted(set(warninglist['list']))
return warninglist return warninglist
def write_to_file(warninglist, dst): def write_to_file(warninglist, dst):
with open(get_abspath_list_file(dst), 'w') as data_file: with open(get_abspath_list_file(dst), 'w') as data_file:
json.dump(unique_sorted_warninglist(warninglist), data_file, indent=2, sort_keys=True) json.dump(unique_sorted_warninglist(warninglist),
data_file, indent=2, sort_keys=True)
data_file.write("\n") data_file.write("\n")