diff --git a/config/modules.json.sample b/config/modules.json.sample index 98c034df..81aeec0d 100644 --- a/config/modules.json.sample +++ b/config/modules.json.sample @@ -74,6 +74,11 @@ "allow_auto_trigger": false, "default_first_seen_in_days": 5 }, + "CIRCLPDNS": { + "user": null, + "password": null, + "allow_auto_trigger": false + }, "_notes": { "apikey": "null disables the module. Pass a string otherwise.", "autosubmit": "Automatically submits the URL to the 3rd party service.", diff --git a/lookyloo/lookyloo.py b/lookyloo/lookyloo.py index f09c04f2..d647c8dc 100644 --- a/lookyloo/lookyloo.py +++ b/lookyloo/lookyloo.py @@ -54,7 +54,7 @@ from .helpers import (get_captures_dir, get_email_template, from .indexing import Indexing from .modules import (MISPs, PhishingInitiative, UniversalWhois, UrlScan, VirusTotal, Phishtank, Hashlookup, - RiskIQ, RiskIQError, Pandora, URLhaus) + RiskIQ, RiskIQError, Pandora, URLhaus, CIRCLPDNS) if TYPE_CHECKING: from playwright.async_api import Cookie @@ -120,6 +120,7 @@ class Lookyloo(): pass # ## Done with MISP(s) + self.pi = PhishingInitiative(config_name='PhishingInitiative') self.vt = VirusTotal(config_name='VirusTotal') self.uwhois = UniversalWhois(config_name='UniversalWhois') @@ -129,6 +130,7 @@ class Lookyloo(): self.riskiq = RiskIQ(config_name='RiskIQ') self.pandora = Pandora(config_name='Pandora') self.urlhaus = URLhaus(config_name='URLhaus') + self.circl_pdns = CIRCLPDNS(config_name='CIRCLPDNS') self.monitoring_enabled = False if monitoring_config := get_config('generic', 'monitoring'): @@ -423,7 +425,7 @@ class Lookyloo(): if not cache: self.logger.warning(f'Unable to get the modules responses unless the capture {capture_uuid} is cached') return {} - to_return: Dict[str, Any] = {} + to_return: Dict[str, Any] = defaultdict(dict) if self.riskiq.available: try: self.riskiq.capture_default_trigger(cache) @@ -435,6 +437,15 @@ class Lookyloo(): to_return['riskiq'] = self.riskiq.get_passivedns(hostname) except RiskIQError as e: self.logger.warning(e.response.content) + if self.circl_pdns.available: + self.circl_pdns.capture_default_trigger(cache) + if hasattr(cache, 'redirects') and cache.redirects: + hostname = urlparse(cache.redirects[-1]).hostname + else: + hostname = urlparse(cache.url).hostname + if hostname: + to_return['circl_pdns'][hostname] = self.circl_pdns.get_passivedns(hostname) + return to_return def hide_capture(self, capture_uuid: str, /) -> None: diff --git a/lookyloo/modules/__init__.py b/lookyloo/modules/__init__.py index f5c2b973..5a2c378e 100644 --- a/lookyloo/modules/__init__.py +++ b/lookyloo/modules/__init__.py @@ -13,3 +13,4 @@ from .hashlookup import HashlookupModule as Hashlookup # noqa from .riskiq import RiskIQ, RiskIQError # noqa from .urlhaus import URLhaus # noqa from .cloudflare import Cloudflare # noqa +from .circlpdns import CIRCLPDNS # noqa diff --git a/lookyloo/modules/circlpdns.py b/lookyloo/modules/circlpdns.py new file mode 100644 index 00000000..ed55ac62 --- /dev/null +++ b/lookyloo/modules/circlpdns.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 + +import json + +from datetime import date +from typing import Dict, List, Optional, TYPE_CHECKING +from urllib.parse import urlparse + +from pypdns import PyPDNS, PDNSRecord + +from ..default import ConfigError, get_homedir +from ..helpers import get_cache_directory + +if TYPE_CHECKING: + from ..capturecache import CaptureCache + +from .abstractmodule import AbstractModule + + +class CIRCLPDNS(AbstractModule): + + def module_init(self) -> bool: + if not (self.config.get('user') and self.config.get('password')): + self.logger.info('Missing credentials.') + return False + + self.pypdns = PyPDNS(basic_auth=(self.config['user'], self.config['password'])) + + self.allow_auto_trigger = bool(self.config.get('allow_auto_trigger', False)) + + self.storage_dir_pypdns = get_homedir() / 'circl_pypdns' + self.storage_dir_pypdns.mkdir(parents=True, exist_ok=True) + return True + + def get_passivedns(self, query: str) -> Optional[List[PDNSRecord]]: + # The query can be IP or Hostname. For now, we only do it on domains. + url_storage_dir = get_cache_directory(self.storage_dir_pypdns, query, 'pdns') + if not url_storage_dir.exists(): + return None + cached_entries = sorted(url_storage_dir.glob('*'), reverse=True) + if not cached_entries: + return None + + with cached_entries[0].open() as f: + return [PDNSRecord(record) for record in json.load(f)] + + def capture_default_trigger(self, cache: 'CaptureCache', /, *, force: bool=False, auto_trigger: bool=False) -> Dict: + '''Run the module on all the nodes up to the final redirect''' + if not self.available: + return {'error': 'Module not available'} + if auto_trigger and not self.allow_auto_trigger: + return {'error': 'Auto trigger not allowed on module'} + if cache.url.startswith('file'): + return {'error': 'CIRCL Passive DNS does not support files.'} + + if cache.redirects: + hostname = urlparse(cache.redirects[-1]).hostname + else: + hostname = urlparse(cache.url).hostname + + if not hostname: + return {'error': 'No hostname found.'} + + self.pdns_lookup(hostname, force) + return {'success': 'Module triggered'} + + def pdns_lookup(self, hostname: str, force: bool=False) -> None: + '''Lookup an hostname on CIRCL Passive DNS + Note: force means re-fetch the entry even if we already did it today + ''' + if not self.available: + raise ConfigError('CIRCL Passive DNS not available, probably no API key') + + url_storage_dir = get_cache_directory(self.storage_dir_pypdns, hostname, 'pdns') + url_storage_dir.mkdir(parents=True, exist_ok=True) + pypdns_file = url_storage_dir / date.today().isoformat() + + if not force and pypdns_file.exists(): + return + + pdns_info = [entry for entry in self.pypdns.iter_query(hostname)] + if not pdns_info: + try: + url_storage_dir.rmdir() + except OSError: + # Not empty. + pass + return + pdns_info_store = [entry.raw for entry in sorted(pdns_info, key=lambda k: k.time_last_datetime, reverse=True)] + with pypdns_file.open('w') as _f: + json.dump(pdns_info_store, _f) diff --git a/poetry.lock b/poetry.lock index 4f72c038..b040bb13 100644 --- a/poetry.lock +++ b/poetry.lock @@ -895,13 +895,13 @@ files = [ [[package]] name = "fsspec" -version = "2023.12.1" +version = "2023.12.2" description = "File-system specification" optional = false python-versions = ">=3.8" files = [ - {file = "fsspec-2023.12.1-py3-none-any.whl", hash = "sha256:6271f1d3075a378bfe432f6f42bf7e1d2a6ba74f78dd9b512385474c579146a0"}, - {file = "fsspec-2023.12.1.tar.gz", hash = "sha256:c4da01a35ac65c853f833e43f67802c25213f560820d54ddf248f92eddd5e990"}, + {file = "fsspec-2023.12.2-py3-none-any.whl", hash = "sha256:d800d87f72189a745fa3d6b033b9dc4a34ad069f60ca60b943a63599f5501960"}, + {file = "fsspec-2023.12.2.tar.gz", hash = "sha256:8548d39e8810b59c38014934f6b31e57f40c1b20f911f4cc2b85389c7e9bf0cb"}, ] [package.extras] @@ -2083,13 +2083,13 @@ recaptcha = ["SpeechRecognition (>=3.10.1,<4.0.0)", "pydub (>=0.25.1,<0.26.0)", [[package]] name = "prompt-toolkit" -version = "3.0.41" +version = "3.0.42" description = "Library for building powerful interactive command lines in Python" optional = false python-versions = ">=3.7.0" files = [ - {file = "prompt_toolkit-3.0.41-py3-none-any.whl", hash = "sha256:f36fe301fafb7470e86aaf90f036eef600a3210be4decf461a5b1ca8403d3cb2"}, - {file = "prompt_toolkit-3.0.41.tar.gz", hash = "sha256:941367d97fc815548822aa26c2a269fdc4eb21e9ec05fc5d447cf09bad5d75f0"}, + {file = "prompt_toolkit-3.0.42-py3-none-any.whl", hash = "sha256:3b50b5fc50660dc8e39dfe464b170959ad82ff185ffa53bfd3be02222e7156a1"}, + {file = "prompt_toolkit-3.0.42.tar.gz", hash = "sha256:bfbf7d6ea9744e4ec94c9a69539e8106c77a2a607d728ded87c9182a4aec39be"}, ] [package.dependencies] @@ -2830,19 +2830,19 @@ files = [ [[package]] name = "s3fs" -version = "2023.12.1" +version = "2023.12.2" description = "Convenient Filesystem interface over S3" optional = false python-versions = ">= 3.8" files = [ - {file = "s3fs-2023.12.1-py3-none-any.whl", hash = "sha256:ed0b7df8cc20a2b5cefe607b1cf4e860d37c5ca4ac2d68f55464805d75d18710"}, - {file = "s3fs-2023.12.1.tar.gz", hash = "sha256:63e429bb6b5e814568cacd3f2a8551fc35493e8c418ddfcb44e6f86aa8696ccd"}, + {file = "s3fs-2023.12.2-py3-none-any.whl", hash = "sha256:0d5a99039665f30b2dbee5495de3b299a022d51b3195a9440f5df47c2621b777"}, + {file = "s3fs-2023.12.2.tar.gz", hash = "sha256:b5ec07062481bbb45cb061b31984c7188d106e292c27033039e024e4ba5740dc"}, ] [package.dependencies] aiobotocore = ">=2.5.4,<3.0.0" aiohttp = "<4.0.0a0 || >4.0.0a0,<4.0.0a1 || >4.0.0a1" -fsspec = "2023.12.1" +fsspec = "2023.12.2" [package.extras] awscli = ["aiobotocore[awscli] (>=2.5.4,<3.0.0)"] @@ -3488,4 +3488,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = ">=3.8.1,<3.12" -content-hash = "d0419f51f92d82e4099ad6ccf66f931eafc7499f4e30c9b085167d8ede8fb43c" +content-hash = "3f92ebac858e5a46242bea9b8b4539a913c5a397a8a54db1fce87c020a647c3c" diff --git a/pyproject.toml b/pyproject.toml index e126971c..0d5bad19 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -74,7 +74,7 @@ chardet = "^5.2.0" pysecuritytxt = "^1.2.0" pylookyloomonitoring = "^1.1.3" pytz = {"version" = "^2023.3.post1", python = "<3.9"} -s3fs = "^2023.12.1" +s3fs = "^2023.12.2" urllib3 = [ {version = "<2", python = "<3.10"}, {version = "^2.0.7", python = ">=3.10"} diff --git a/website/web/__init__.py b/website/web/__init__.py index fe13bec1..f1b3225e 100644 --- a/website/web/__init__.py +++ b/website/web/__init__.py @@ -306,7 +306,8 @@ def historical_lookups(tree_uuid: str): force = True if (request.args.get('force') and request.args.get('force') == 'True') else False data = lookyloo.get_historical_lookups(tree_uuid, force) return render_template('historical_lookups.html', tree_uuid=tree_uuid, - riskiq=data.get('riskiq')) + riskiq=data.get('riskiq'), + circl_pdns=data.get('circl_pdns')) @app.route('/tree//categories_capture/', defaults={'query': ''}) diff --git a/website/web/templates/historical_lookups.html b/website/web/templates/historical_lookups.html index 4ca04136..283983de 100644 --- a/website/web/templates/historical_lookups.html +++ b/website/web/templates/historical_lookups.html @@ -26,6 +26,39 @@ {% endfor %} + {% endif%} +{% if circl_pdns %} +
+
+

CIRCL Passve DNS

+ {% for query, responses in circl_pdns.items() %} +
+

{{query}}

+ + + + + + + + + + + {%for response in responses %} + + + + + + + + {% endfor %} +
First seenLast seenR DataRR NameRR Type
{{response.time_first_datetime}}{{response.time_last_datetime}}{{response.rdata}}{{response.rrname}}{{response.rrtype}}
+
+ {%endfor%} +
+ +{% endif%}