diff --git a/bin/dbinsert.py b/bin/dbinsert.py index 16f4d04..c39b8c3 100755 --- a/bin/dbinsert.py +++ b/bin/dbinsert.py @@ -74,6 +74,7 @@ class DBInsertManager(AbstractManager): data = sanitized_data[i] if not data: self.logger.warning(f'No data for UUID {uuid}. This should not happen, but lets move on.') + done.append(uuid) continue routing_info = responses['responses'][i]['response'] # our queries are on one single date, not a range # Data gathered from IPASN History: @@ -81,6 +82,7 @@ class DBInsertManager(AbstractManager): # * AS number if 'error' in routing_info: self.logger.warning(f"Unable to find routing information for {data['ip']} - {data['datetime']}: {routing_info['error']}") + done.append(uuid) continue # Single date query, getting from the object datetime_routing = list(routing_info.keys())[0] @@ -91,9 +93,11 @@ class DBInsertManager(AbstractManager): continue if 'asn' in entry and entry['asn'] in [None, '0']: self.logger.warning(f"Unable to find the AS number associated to {data['ip']} - {data['datetime']} (got {entry['asn']}).") + done.append(uuid) continue if 'prefix' in entry and entry['prefix'] in [None, '0.0.0.0/0', '::/0']: self.logger.warning(f"Unable to find the prefix associated to {data['ip']} - {data['datetime']} (got {entry['prefix']}).") + done.append(uuid) continue # Format: |sources -> set([, ...]) diff --git a/poetry.lock b/poetry.lock index b0a6dd8..baf4f59 100644 --- a/poetry.lock +++ b/poetry.lock @@ -818,8 +818,8 @@ testing = ["flake8 (<5)", "func-timeout", "jaraco.functools", "jaraco.itertools" [metadata] lock-version = "1.1" -python-versions = "^3.8" -content-hash = "a7b14e4e425b55589575ff0b0b2e8ec99f3d32ef32c8c3ac90dcf7ec2afb9fee" +python-versions = "^3.8.1" +content-hash = "00557ce6a5aa68b4251296d5d45716a49467cecf5cde1a0cab3c00e2f1d721ff" [metadata.files] aiohttp = [ diff --git a/pyproject.toml b/pyproject.toml index 8a4bd78..aecf058 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,7 +24,7 @@ ssfetcher = "bin.ssfetcher:main" [tool.poetry.dependencies] -python = "^3.8" +python = "^3.8.1" redis = {version = "^4.4.0", extras = ["hiredis"]} flask-restx = "^0.5.1" gunicorn = "^20.1.0" diff --git a/tools/clear_prepare_db.py b/tools/clear_prepare_db.py new file mode 100644 index 0000000..f0fd4dc --- /dev/null +++ b/tools/clear_prepare_db.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python3 + +import uuid + +from redis import Redis +from bgpranking.default import get_socket_path + +redis_sanitized = Redis(unix_socket_path=get_socket_path('prepare'), db=0, decode_responses=True) + +while name := redis_sanitized.scan_iter(_type='HASH', count=10): + try: + uuid.uuid(name) + except Exception: + pass + if not redis_sanitized.sismember('to_insert'): + print(name)