From 4766c457c07ba5d275cd4ede18f693fe79483d83 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Fri, 22 May 2020 19:23:08 +0200 Subject: [PATCH] Validate values in CI --- tools/validate_values.py | 78 ++++++++++++++++++++++++++++++++++++++++ validate_all.sh | 4 +++ 2 files changed, 82 insertions(+) create mode 100644 tools/validate_values.py diff --git a/tools/validate_values.py b/tools/validate_values.py new file mode 100644 index 0000000..b7cce97 --- /dev/null +++ b/tools/validate_values.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +import re +import json +import sys +from ipaddress import ip_network +from pathlib import Path +from typing import List, Iterator, Optional, Tuple + +HOSTNAME_RE = re.compile("(?!-)[A-Z\d-]{1,63}(? 255: + raise ValueError("Hostname {0!r} is too long (maximum is 255 chars)".format(hostname)) + + if hostname[-1] == ".": + hostname = hostname[:-1] # strip exactly one dot from the right, if present + + if hostname[0] == ".": + hostname = hostname[1:] # strip exactly one dot from the left, if present + + if not all(HOSTNAME_RE.match(x) for x in hostname.split(".")): + raise ValueError("Hostname {0!r} contains invalid characters".format(hostname)) + + +def is_valid_regexp(regexp: str): + try: + re.compile(regexp) + except re.error: + raise ValueError("Regexp {0!r} is not valid".format(regexp)) + + +def validate(values: List[str], func) -> Iterator[Tuple[str, ValueError]]: + for value in values: + try: + func(value) + except ValueError as e: + yield value, e + + +def validate_file(p: Path) -> Iterator[InvalidListValue]: + invalid_values = [] + with p.open() as f: + warninglist = json.load(f, encoding="utf-8") + if warninglist["type"] == "cidr": + invalid_values = validate(warninglist["list"], lambda value: ip_network(value, strict=True)) + elif warninglist["type"] == "regexp": + invalid_values = validate(warninglist["list"], lambda value: is_valid_regexp(value)) + # Disabled, because current lists contains invalid domains + # elif warninglist["type"] == "hostname": + # invalid_values = validate_hostnames(warninglist["list"]) + + for (value, exception) in invalid_values: + yield InvalidListValue(p, value, str(exception)) + + +if __name__ == "__main__": + invalid_values = [] + for p in Path('../lists/').glob('*/*.json'): + invalid_values.extend(validate_file(p)) + + if invalid_values: + for invalid_value in invalid_values: + print(invalid_value, file=sys.stderr) + sys.exit(1) diff --git a/validate_all.sh b/validate_all.sh index a43b849..a21bcc4 100755 --- a/validate_all.sh +++ b/validate_all.sh @@ -42,3 +42,7 @@ do echo '' done +pushd tools +python3 validate_values.py +popd +