diff --git a/tools/generator.py b/tools/generator.py index 2be6fff..34e3010 100644 --- a/tools/generator.py +++ b/tools/generator.py @@ -7,15 +7,22 @@ from inspect import currentframe, getframeinfo from os import path import requests +from dateutil.parser import parse as parsedate def download_to_file(url, file): user_agent = { "User-agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:46.0) Gecko/20100101 Firefox/46.0"} - r = requests.get(url, headers=user_agent) - with open(file, 'wb') as fd: - for chunk in r.iter_content(4096): - fd.write(chunk) + r = requests.head(url, headers=user_agent) + url_datetime = parsedate(r.headers['Last-Modified']).astimezone() + file_datetime = datetime.datetime.fromtimestamp( + path.getmtime(file)).astimezone() + + if(url_datetime > file_datetime): + r = requests.get(url, headers=user_agent) + with open(file, 'wb') as fd: + for chunk in r.iter_content(4096): + fd.write(chunk) def download(url): @@ -40,7 +47,9 @@ def unique_sorted_warninglist(warninglist): warninglist['list'] = sorted(set(warninglist['list'])) return warninglist + def write_to_file(warninglist, dst): with open(get_abspath_list_file(dst), 'w') as data_file: - json.dump(unique_sorted_warninglist(warninglist), data_file, indent=2, sort_keys=True) + json.dump(unique_sorted_warninglist(warninglist), + data_file, indent=2, sort_keys=True) data_file.write("\n")