new: [second-level-tlds] Add generator and update to latest version

pull/184/head
Jakub Onderka 2021-06-10 10:36:26 +02:00
parent 66a92ab133
commit 104bf6883f
4 changed files with 3216 additions and 481 deletions

View File

@ -29,6 +29,7 @@ python3 generate_tranco.py
python3 generate-university-domain-list.py python3 generate-university-domain-list.py
python3 generate-vpn.py python3 generate-vpn.py
python3 generate-wikimedia.py python3 generate-wikimedia.py
python3 genetate-second-level-tlds.py
popd popd
./jq_all_the_things.sh ./jq_all_the_things.sh

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,27 @@
#!/usr/bin/env python3
from generator import download, get_version, write_to_file
if __name__ == '__main__':
source_url = 'https://publicsuffix.org/list/public_suffix_list.dat'
destination_folder = 'second-level-tlds'
data = download(source_url).text
lines = data.split("\n")
# Filter out comments
domains = [line.strip() for line in lines if len(line) != 0 and not line.startswith('//')]
# Convert IDN domain to xn-- format
domains = [domain.encode('idna').decode('utf-8') for domain in domains]
# Filter out invalid domains
domains = [domain.lstrip('*.') for domain in domains if not domain.startswith('!')]
warninglist = {
'name': 'Second level TLDs as known by Mozilla Foundation',
'description': 'Event contains one or more second level TLDs as attribute with an IDS flag set.',
'matching_attributes': ['hostname', 'domain', 'domain|ip'],
'type': 'string',
'version': get_version(),
'list': domains,
}
write_to_file(warninglist, destination_folder)

View File

@ -132,7 +132,7 @@ def write_to_file(warninglist, dst):
get_abspath_list_file(dst))) get_abspath_list_file(dst)))
except Exception as exc: except Exception as exc:
logging.error( logging.error(
'{} General exception occured: {}.'.format(caller, str(exc))) '{} General exception occurred: {}.'.format(caller, str(exc)))
def main(): def main():