solved LGTM alerts

pull/154/head
Kevin Holvoet 2020-07-17 12:42:34 +02:00
parent 9ba47c1463
commit cb52a472e6
9 changed files with 39 additions and 33 deletions

View File

@ -3,7 +3,7 @@
import json import json
from generator import download, download_to_file, get_abspath_list_file, get_version from generator import download_to_file, get_abspath_list_file, get_version
def process(file, dst): def process(file, dst):

View File

@ -1,10 +1,10 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
import zipfile
import json import json
import zipfile
from generator import download, download_to_file, get_abspath_list_file, get_version from generator import download_to_file, get_abspath_list_file, get_version
def process(file, warninglist, dst, limit='1k'): def process(file, warninglist, dst, limit='1k'):
@ -26,12 +26,13 @@ def process(file, warninglist, dst, limit='1k'):
warninglist['version'] = get_version() warninglist['version'] = get_version()
warninglist['type'] = 'hostname' warninglist['type'] = 'hostname'
warninglist['matching_attributes'] = ['hostname', 'domain', 'url', 'domain|ip'] warninglist['matching_attributes'] = [
'hostname', 'domain', 'url', 'domain|ip']
warninglist['list'] = [] warninglist['list'] = []
for site in top: for site in top:
v = site.decode('UTF-8').split(',')[1] v = site.decode('UTF-8').split(',')[1]
warninglist['list'].append(v.strip().replace('\\r\\n','')) warninglist['list'].append(v.strip().replace('\\r\\n', ''))
warninglist['list'] = sorted(set(warninglist['list'])) warninglist['list'] = sorted(set(warninglist['list']))
with open(get_abspath_list_file(dst), 'w') as data_file: with open(get_abspath_list_file(dst), 'w') as data_file:

View File

@ -2,7 +2,7 @@
import json import json
from generator import download, download_to_file, get_abspath_list_file, get_version from generator import download_to_file, get_abspath_list_file, get_version
def process(files, dst): def process(files, dst):
@ -12,7 +12,7 @@ def process(files, dst):
warninglist['description'] = "List of known Cloudflare IP ranges (https://www.cloudflare.com/ips/)" warninglist['description'] = "List of known Cloudflare IP ranges (https://www.cloudflare.com/ips/)"
warninglist['type'] = "cidr" warninglist['type'] = "cidr"
warninglist['list'] = [] warninglist['list'] = []
warninglist['matching_attributes'] = ["ip-dst","ip-src","domain|ip"] warninglist['matching_attributes'] = ["ip-dst", "ip-src", "domain|ip"]
for file in files: for file in files:
with open(file, 'r') as f: with open(file, 'r') as f:
@ -28,7 +28,7 @@ def process(files, dst):
if __name__ == '__main__': if __name__ == '__main__':
cf_base_url = "https://www.cloudflare.com/" cf_base_url = "https://www.cloudflare.com/"
uri_list = ['ips-v4','ips-v6'] uri_list = ['ips-v4', 'ips-v6']
cf_dst = 'cloudflare' cf_dst = 'cloudflare'
to_process = list() to_process = list()

View File

@ -3,7 +3,7 @@
import json import json
from generator import download, download_to_file, get_abspath_list_file, get_version from generator import download, get_abspath_list_file, get_version
def process(url, warninglist, dst): def process(url, warninglist, dst):
@ -35,4 +35,5 @@ if __name__ == '__main__':
} }
process(covid_krassi_url, covid_krassi_warninglist, covid_krassi_dst) process(covid_krassi_url, covid_krassi_warninglist, covid_krassi_dst)
process(covid_cyber_threat_coalition_url, covid_cyber_threat_coalition_warninglist, covid_cyber_threat_coalition_dst) process(covid_cyber_threat_coalition_url,
covid_cyber_threat_coalition_warninglist, covid_cyber_threat_coalition_dst)

View File

@ -2,10 +2,11 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
import json import json
import requests
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from generator import download, download_to_file, get_abspath_list_file, get_version from generator import (download, download_to_file, get_abspath_list_file,
get_version)
def get_json_url(page): def get_json_url(page):

View File

@ -4,7 +4,7 @@
import json import json
import zipfile import zipfile
from generator import download, download_to_file, get_abspath_list_file, get_version from generator import download_to_file, get_abspath_list_file, get_version
def process(file, dst): def process(file, dst):

View File

@ -2,7 +2,8 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
import json import json
from generator import download, download_to_file, get_abspath_list_file, get_version
from generator import download_to_file, get_abspath_list_file, get_version
def process(file, dst): def process(file, dst):

View File

@ -7,7 +7,7 @@ import json
from OpenSSL.crypto import FILETYPE_PEM, load_certificate from OpenSSL.crypto import FILETYPE_PEM, load_certificate
from generator import download, download_to_file, get_abspath_list_file, get_version from generator import download_to_file, get_abspath_list_file, get_version
def gethash(cert, digest): def gethash(cert, digest):
@ -31,7 +31,7 @@ def process(file, dst, type):
warninglist = {} warninglist = {}
warninglist['name'] = 'Fingerprint of {type}'.format(type=type) warninglist['name'] = 'Fingerprint of {type}'.format(type=type)
warninglist['version'] = int(datetime.date.today().strftime('%Y%m%d')) warninglist['version'] = get_version()
warninglist['description'] = "Fingerprint of {type} taken from Mozilla's lists at https://wiki.mozilla.org/CA".format( warninglist['description'] = "Fingerprint of {type} taken from Mozilla's lists at https://wiki.mozilla.org/CA".format(
type=type) type=type)
warninglist['list'] = sorted(hashes) warninglist['list'] = sorted(hashes)

View File

@ -4,7 +4,7 @@
import json import json
import zipfile import zipfile
from generator import download, download_to_file, get_abspath_list_file, get_version from generator import download_to_file, get_abspath_list_file, get_version
def process(file, warninglist, dst, first_10k=False): def process(file, warninglist, dst, first_10k=False):
@ -22,7 +22,8 @@ def process(file, warninglist, dst, first_10k=False):
warninglist['type'] = 'hostname' warninglist['type'] = 'hostname'
warninglist['version'] = get_version() warninglist['version'] = get_version()
warninglist['matching_attributes'] = ['hostname', 'domain', 'url', 'domain|ip'] warninglist['matching_attributes'] = [
'hostname', 'domain', 'url', 'domain|ip']
for site in sites: for site in sites:
v = site.decode('UTF-8').split(',')[1] v = site.decode('UTF-8').split(',')[1]
@ -54,4 +55,5 @@ if __name__ == '__main__':
'description': "Event contains one or more entries from the top 10K most-used sites (https://tranco-list.eu/).", 'description': "Event contains one or more entries from the top 10K most-used sites (https://tranco-list.eu/).",
'name': "Top 10K most-used sites from Tranco" 'name': "Top 10K most-used sites from Tranco"
} }
process(tranco_file, tranco_10k_warninglist, tranco_10k_dst, first_10k=True) process(tranco_file, tranco_10k_warninglist,
tranco_10k_dst, first_10k=True)