new: CIRCL Passive DNS module

config/modules.json.sample

@@ -74,6 +74,11 @@
     "allow_auto_trigger": false,
     "default_first_seen_in_days": 5
   },
+  "CIRCLPDNS": {
+    "user": null,
+    "password": null,
+    "allow_auto_trigger": false
+  },
   "_notes": {
     "apikey": "null disables the module. Pass a string otherwise.",
     "autosubmit": "Automatically submits the URL to the 3rd party service.",

lookyloo/lookyloo.py

@@ -54,7 +54,7 @@ from .helpers import (get_captures_dir, get_email_template,
 from .indexing import Indexing
 from .modules import (MISPs, PhishingInitiative, UniversalWhois,
                       UrlScan, VirusTotal, Phishtank, Hashlookup,
-                      RiskIQ, RiskIQError, Pandora, URLhaus)
+                      RiskIQ, RiskIQError, Pandora, URLhaus, CIRCLPDNS)
 
 if TYPE_CHECKING:
     from playwright.async_api import Cookie
@@ -120,6 +120,7 @@ class Lookyloo():
                 pass
 
         # ## Done with MISP(s)
+
         self.pi = PhishingInitiative(config_name='PhishingInitiative')
         self.vt = VirusTotal(config_name='VirusTotal')
         self.uwhois = UniversalWhois(config_name='UniversalWhois')
@@ -129,6 +130,7 @@ class Lookyloo():
         self.riskiq = RiskIQ(config_name='RiskIQ')
         self.pandora = Pandora(config_name='Pandora')
         self.urlhaus = URLhaus(config_name='URLhaus')
+        self.circl_pdns = CIRCLPDNS(config_name='CIRCLPDNS')
 
         self.monitoring_enabled = False
         if monitoring_config := get_config('generic', 'monitoring'):
@@ -423,7 +425,7 @@ class Lookyloo():
         if not cache:
             self.logger.warning(f'Unable to get the modules responses unless the capture {capture_uuid} is cached')
             return {}
-        to_return: Dict[str, Any] = {}
+        to_return: Dict[str, Any] = defaultdict(dict)
         if self.riskiq.available:
             try:
                 self.riskiq.capture_default_trigger(cache)
@@ -435,6 +437,15 @@ class Lookyloo():
                     to_return['riskiq'] = self.riskiq.get_passivedns(hostname)
             except RiskIQError as e:
                 self.logger.warning(e.response.content)
+        if self.circl_pdns.available:
+            self.circl_pdns.capture_default_trigger(cache)
+            if hasattr(cache, 'redirects') and cache.redirects:
+                hostname = urlparse(cache.redirects[-1]).hostname
+            else:
+                hostname = urlparse(cache.url).hostname
+            if hostname:
+                to_return['circl_pdns'][hostname] = self.circl_pdns.get_passivedns(hostname)
+
         return to_return
 
     def hide_capture(self, capture_uuid: str, /) -> None:

lookyloo/modules/__init__.py

@@ -13,3 +13,4 @@ from .hashlookup import HashlookupModule as Hashlookup  # noqa
 from .riskiq import RiskIQ, RiskIQError  # noqa
 from .urlhaus import URLhaus  # noqa
 from .cloudflare import Cloudflare  # noqa
+from .circlpdns import CIRCLPDNS  # noqa

lookyloo/modules/circlpdns.py

@@ -0,0 +1,91 @@
+#!/usr/bin/env python3
+
+import json
+
+from datetime import date
+from typing import Dict, List, Optional, TYPE_CHECKING
+from urllib.parse import urlparse
+
+from pypdns import PyPDNS, PDNSRecord
+
+from ..default import ConfigError, get_homedir
+from ..helpers import get_cache_directory
+
+if TYPE_CHECKING:
+    from ..capturecache import CaptureCache
+
+from .abstractmodule import AbstractModule
+
+
+class CIRCLPDNS(AbstractModule):
+
+    def module_init(self) -> bool:
+        if not (self.config.get('user') and self.config.get('password')):
+            self.logger.info('Missing credentials.')
+            return False
+
+        self.pypdns = PyPDNS(basic_auth=(self.config['user'], self.config['password']))
+
+        self.allow_auto_trigger = bool(self.config.get('allow_auto_trigger', False))
+
+        self.storage_dir_pypdns = get_homedir() / 'circl_pypdns'
+        self.storage_dir_pypdns.mkdir(parents=True, exist_ok=True)
+        return True
+
+    def get_passivedns(self, query: str) -> Optional[List[PDNSRecord]]:
+        # The query can be IP or Hostname. For now, we only do it on domains.
+        url_storage_dir = get_cache_directory(self.storage_dir_pypdns, query, 'pdns')
+        if not url_storage_dir.exists():
+            return None
+        cached_entries = sorted(url_storage_dir.glob('*'), reverse=True)
+        if not cached_entries:
+            return None
+
+        with cached_entries[0].open() as f:
+            return [PDNSRecord(record) for record in json.load(f)]
+
+    def capture_default_trigger(self, cache: 'CaptureCache', /, *, force: bool=False, auto_trigger: bool=False) -> Dict:
+        '''Run the module on all the nodes up to the final redirect'''
+        if not self.available:
+            return {'error': 'Module not available'}
+        if auto_trigger and not self.allow_auto_trigger:
+            return {'error': 'Auto trigger not allowed on module'}
+        if cache.url.startswith('file'):
+            return {'error': 'CIRCL Passive DNS does not support files.'}
+
+        if cache.redirects:
+            hostname = urlparse(cache.redirects[-1]).hostname
+        else:
+            hostname = urlparse(cache.url).hostname
+
+        if not hostname:
+            return {'error': 'No hostname found.'}
+
+        self.pdns_lookup(hostname, force)
+        return {'success': 'Module triggered'}
+
+    def pdns_lookup(self, hostname: str, force: bool=False) -> None:
+        '''Lookup an hostname on CIRCL Passive DNS
+        Note: force means re-fetch the entry even if we already did it today
+        '''
+        if not self.available:
+            raise ConfigError('CIRCL Passive DNS not available, probably no API key')
+
+        url_storage_dir = get_cache_directory(self.storage_dir_pypdns, hostname, 'pdns')
+        url_storage_dir.mkdir(parents=True, exist_ok=True)
+        pypdns_file = url_storage_dir / date.today().isoformat()
+
+        if not force and pypdns_file.exists():
+            return
+
+        pdns_info = [entry for entry in self.pypdns.iter_query(hostname)]
+        if not pdns_info:
+            try:
+                url_storage_dir.rmdir()
+            except OSError:
+                # Not empty.
+                pass
+            return
+        pdns_info_store = [entry.raw for entry in sorted(pdns_info, key=lambda k: k.time_last_datetime, reverse=True)]
+        with pypdns_file.open('w') as _f:
+            json.dump(pdns_info_store, _f)

poetry.lock

@@ -895,13 +895,13 @@ files = [
 
 [[package]]
 name = "fsspec"
-version = "2023.12.1"
+version = "2023.12.2"
 description = "File-system specification"
 optional = false
 python-versions = ">=3.8"
 files = [
-    {file = "fsspec-2023.12.1-py3-none-any.whl", hash = "sha256:6271f1d3075a378bfe432f6f42bf7e1d2a6ba74f78dd9b512385474c579146a0"},
+    {file = "fsspec-2023.12.2-py3-none-any.whl", hash = "sha256:d800d87f72189a745fa3d6b033b9dc4a34ad069f60ca60b943a63599f5501960"},
-    {file = "fsspec-2023.12.1.tar.gz", hash = "sha256:c4da01a35ac65c853f833e43f67802c25213f560820d54ddf248f92eddd5e990"},
+    {file = "fsspec-2023.12.2.tar.gz", hash = "sha256:8548d39e8810b59c38014934f6b31e57f40c1b20f911f4cc2b85389c7e9bf0cb"},
 ]
 
 [package.extras]
@@ -2083,13 +2083,13 @@ recaptcha = ["SpeechRecognition (>=3.10.1,<4.0.0)", "pydub (>=0.25.1,<0.26.0)",
 
 [[package]]
 name = "prompt-toolkit"
-version = "3.0.41"
+version = "3.0.42"
 description = "Library for building powerful interactive command lines in Python"
 optional = false
 python-versions = ">=3.7.0"
 files = [
-    {file = "prompt_toolkit-3.0.41-py3-none-any.whl", hash = "sha256:f36fe301fafb7470e86aaf90f036eef600a3210be4decf461a5b1ca8403d3cb2"},
+    {file = "prompt_toolkit-3.0.42-py3-none-any.whl", hash = "sha256:3b50b5fc50660dc8e39dfe464b170959ad82ff185ffa53bfd3be02222e7156a1"},
-    {file = "prompt_toolkit-3.0.41.tar.gz", hash = "sha256:941367d97fc815548822aa26c2a269fdc4eb21e9ec05fc5d447cf09bad5d75f0"},
+    {file = "prompt_toolkit-3.0.42.tar.gz", hash = "sha256:bfbf7d6ea9744e4ec94c9a69539e8106c77a2a607d728ded87c9182a4aec39be"},
 ]
 
 [package.dependencies]
@@ -2830,19 +2830,19 @@ files = [
 
 [[package]]
 name = "s3fs"
-version = "2023.12.1"
+version = "2023.12.2"
 description = "Convenient Filesystem interface over S3"
 optional = false
 python-versions = ">= 3.8"
 files = [
-    {file = "s3fs-2023.12.1-py3-none-any.whl", hash = "sha256:ed0b7df8cc20a2b5cefe607b1cf4e860d37c5ca4ac2d68f55464805d75d18710"},
+    {file = "s3fs-2023.12.2-py3-none-any.whl", hash = "sha256:0d5a99039665f30b2dbee5495de3b299a022d51b3195a9440f5df47c2621b777"},
-    {file = "s3fs-2023.12.1.tar.gz", hash = "sha256:63e429bb6b5e814568cacd3f2a8551fc35493e8c418ddfcb44e6f86aa8696ccd"},
+    {file = "s3fs-2023.12.2.tar.gz", hash = "sha256:b5ec07062481bbb45cb061b31984c7188d106e292c27033039e024e4ba5740dc"},
 ]
 
 [package.dependencies]
 aiobotocore = ">=2.5.4,<3.0.0"
 aiohttp = "<4.0.0a0 || >4.0.0a0,<4.0.0a1 || >4.0.0a1"
-fsspec = "2023.12.1"
+fsspec = "2023.12.2"
 
 [package.extras]
 awscli = ["aiobotocore[awscli] (>=2.5.4,<3.0.0)"]
@@ -3488,4 +3488,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p
 [metadata]
 lock-version = "2.0"
 python-versions = ">=3.8.1,<3.12"
-content-hash = "d0419f51f92d82e4099ad6ccf66f931eafc7499f4e30c9b085167d8ede8fb43c"
+content-hash = "3f92ebac858e5a46242bea9b8b4539a913c5a397a8a54db1fce87c020a647c3c"

pyproject.toml

@@ -74,7 +74,7 @@ chardet = "^5.2.0"
 pysecuritytxt = "^1.2.0"
 pylookyloomonitoring = "^1.1.3"
 pytz = {"version" = "^2023.3.post1", python = "<3.9"}
-s3fs = "^2023.12.1"
+s3fs = "^2023.12.2"
 urllib3 = [
     {version = "<2", python = "<3.10"},
     {version = "^2.0.7", python = ">=3.10"}

website/web/__init__.py

@@ -306,7 +306,8 @@ def historical_lookups(tree_uuid: str):
     force = True if (request.args.get('force') and request.args.get('force') == 'True') else False
     data = lookyloo.get_historical_lookups(tree_uuid, force)
     return render_template('historical_lookups.html', tree_uuid=tree_uuid,
-                           riskiq=data.get('riskiq'))
+                           riskiq=data.get('riskiq'),
+                           circl_pdns=data.get('circl_pdns'))
 
 
 @app.route('/tree/<string:tree_uuid>/categories_capture/', defaults={'query': ''})

website/web/templates/historical_lookups.html

@@ -26,6 +26,39 @@
       </tr>
       {% endfor %}
   </table>
+  </div>
   </center>
 {% endif%}
+{% if circl_pdns %}
+  <hr>
+  <center>
+    <h1 class="display-4">CIRCL Passve DNS</h1>
+  {% for query, responses in circl_pdns.items() %}
+  <div>
+    <h3>{{query}}</h3>
+    <table class="table">
+      <thead>
+        <tr>
+          <th class="col-sm-2" scope="col">First seen</th>
+          <th class="col-sm-2" scope="col">Last seen</th>
+          <th scope="col">R Data</th>
+          <th scope="col">RR Name</th>
+          <th class="col-sm-1" scope="col">RR Type</th>
+      </thead>
+      <tbody>
+        {%for response in responses %}
+        <tr>
+          <td>{{response.time_first_datetime}}</td>
+          <td>{{response.time_last_datetime}}</td>
+          <td>{{response.rdata}}</td>
+          <td>{{response.rrname}}</td>
+          <td>{{response.rrtype}}</td>
+        </tr>
+        {% endfor %}
+    </table>
+  </div>
+  {%endfor%}
+  </center>
+
+{% endif%}
 </div>