chg: Remove RiskIQ, killed by Microsoft.

config/modules.json.sample

@@ -80,13 +80,6 @@
     "allow_auto_trigger": true,
     "admin_only": false
   },
-  "RiskIQ": {
-    "user": null,
-    "apikey": null,
-    "allow_auto_trigger": false,
-    "default_first_seen_in_days": 5,
-    "admin_only": true
-  },
   "CIRCLPDNS": {
     "user": null,
     "password": null,
@@ -109,7 +102,6 @@
     "Hashlookup": "Module to query Hashlookup (https://github.com/adulau/hashlookup-server). URL set to none means querying the public instance.",
     "FOX": "Submission only interface by and for CCCS",
     "Pandora": "Submission only interface for https://github.com/pandora-analysis/",
-    "RiskIQ": "Module to query RiskIQ (https://community.riskiq.com/)",
     "CIRCLPDNS": "Module to query CIRCL Passive DNS (https://www.circl.lu/services/passive-dns/)"
   }
 }

lookyloo/lookyloo.py

@@ -65,7 +65,7 @@ from .helpers import (get_captures_dir, get_email_template,
                       )
 from .modules import (MISPs, PhishingInitiative, UniversalWhois,
                       UrlScan, VirusTotal, Phishtank, Hashlookup,
-                      RiskIQ, RiskIQError, Pandora, URLhaus, CIRCLPDNS)
+                      Pandora, URLhaus, CIRCLPDNS)
 
 if TYPE_CHECKING:
     from playwright.async_api import Cookie
@@ -131,7 +131,6 @@ class Lookyloo():
         self.urlscan = UrlScan(config_name='UrlScan')
         self.phishtank = Phishtank(config_name='Phishtank')
         self.hashlookup = Hashlookup(config_name='Hashlookup')
-        self.riskiq = RiskIQ(config_name='RiskIQ')
         self.pandora = Pandora()
         self.urlhaus = URLhaus(config_name='URLhaus')
         self.circl_pdns = CIRCLPDNS(config_name='CIRCLPDNS')
@@ -449,18 +448,6 @@ class Lookyloo():
             self.logger.warning(f'Unable to get the modules responses unless the capture {capture_uuid} is cached')
             return {}
         to_return: dict[str, Any] = defaultdict(dict)
-        if self.riskiq.available:
-            try:
-                self.riskiq.capture_default_trigger(cache, force=force, auto_trigger=auto_trigger, as_admin=as_admin)
-                if hasattr(cache, 'redirects') and cache.redirects:
-                    hostname = urlparse(cache.redirects[-1]).hostname
-                else:
-                    hostname = urlparse(cache.url).hostname
-                if hostname:
-                    if _riskiq_entries := self.riskiq.get_passivedns(hostname):
-                        to_return['riskiq'] = _riskiq_entries
-            except RiskIQError as e:
-                self.logger.warning(e.response.content)
         if self.circl_pdns.available:
             self.circl_pdns.capture_default_trigger(cache, force=force, auto_trigger=auto_trigger, as_admin=as_admin)
             if hasattr(cache, 'redirects') and cache.redirects:

lookyloo/modules/__init__.py

@@ -10,7 +10,6 @@ from .vt import VirusTotal  # noqa
 from .pandora import Pandora  # noqa
 from .phishtank import Phishtank  # noqa
 from .hashlookup import HashlookupModule as Hashlookup  # noqa
-from .riskiq import RiskIQ, RiskIQError  # noqa
 from .urlhaus import URLhaus  # noqa
 from .cloudflare import Cloudflare  # noqa
 from .circlpdns import CIRCLPDNS  # noqa
@@ -27,8 +26,6 @@ __all__ = [
     'Pandora',
     'Phishtank',
     'Hashlookup',
-    'RiskIQ',
-    'RiskIQError',
     'URLhaus',
     'Cloudflare',
     'CIRCLPDNS'

lookyloo/modules/riskiq.py

@@ -1,122 +0,0 @@
-#!/usr/bin/env python3
-
-from __future__ import annotations
-
-import json
-
-from datetime import date, datetime, timedelta
-from typing import Any, TYPE_CHECKING
-from urllib.parse import urlparse
-
-from passivetotal import AccountClient, DnsRequest, WhoisRequest  # type: ignore[import-untyped]
-from requests import Response
-
-from ..default import ConfigError, get_homedir
-from ..exceptions import ModuleError
-from ..helpers import get_cache_directory
-
-if TYPE_CHECKING:
-    from ..capturecache import CaptureCache
-
-from .abstractmodule import AbstractModule
-
-
-class RiskIQError(ModuleError):
-
-    def __init__(self, response: Response):
-        self.response = response
-
-
-class RiskIQ(AbstractModule):
-
-    def module_init(self) -> bool:
-        if not (self.config.get('user') and self.config.get('apikey')):
-            self.logger.info('Missing credentials.')
-            return False
-
-        try:
-            # Check if account is working
-            test_client = AccountClient(username=self.config.get('user'),
-                                        api_key=self.config.get('apikey'), exception_class=RiskIQError)
-            details = test_client.get_account_details()
-            self.client_dns = DnsRequest(username=self.config.get('user'),
-                                         api_key=self.config.get('apikey'), exception_class=RiskIQError)
-            self.client_whois = WhoisRequest(username=self.config.get('user'),
-                                             api_key=self.config.get('apikey'), exception_class=RiskIQError)
-        except RiskIQError as e:
-            details = e.response.json()
-            if 'message' in details:
-                self.logger.warning(f'RiskIQ not available: {details["message"]}')
-            else:
-                self.logger.warning(f'RiskIQ not available: {details}')
-            return False
-
-        self.default_first_seen = self.config.get('default_first_seen_in_days', 5)
-
-        self.storage_dir_riskiq = get_homedir() / 'riskiq'
-        self.storage_dir_riskiq.mkdir(parents=True, exist_ok=True)
-        return True
-
-    def get_passivedns(self, query: str) -> dict[str, Any] | None:
-        # The query can be IP or Hostname. For now, we only do it on domains.
-        url_storage_dir = get_cache_directory(self.storage_dir_riskiq, query, 'pdns')
-        if not url_storage_dir.exists():
-            return None
-        cached_entries = sorted(url_storage_dir.glob('*'), reverse=True)
-        if not cached_entries:
-            return None
-
-        with cached_entries[0].open() as f:
-            return json.load(f)
-
-    def capture_default_trigger(self, cache: CaptureCache, /, *, force: bool,
-                                auto_trigger: bool, as_admin: bool) -> dict[str, str]:
-        '''Run the module on all the nodes up to the final redirect'''
-
-        if error := super().capture_default_trigger(cache, force=force, auto_trigger=auto_trigger, as_admin=as_admin):
-            return error
-
-        if cache.url.startswith('file'):
-            return {'error': 'RiskIQ does not support files.'}
-
-        if cache.redirects:
-            hostname = urlparse(cache.redirects[-1]).hostname
-        else:
-            hostname = urlparse(cache.url).hostname
-
-        if not hostname:
-            return {'error': 'No hostname found.'}
-
-        self.__pdns_lookup(hostname, force)
-        return {'success': 'Module triggered'}
-
-    def __pdns_lookup(self, hostname: str, force: bool=False, first_seen: date | datetime | None=None) -> None:
-        '''Lookup an hostname on RiskIQ Passive DNS
-        Note: force means re-fetch the entry RiskIQ even if we already did it today
-        '''
-        if not self.available:
-            raise ConfigError('RiskIQ not available, probably no API key')
-
-        if first_seen is None:
-            first_seen = date.today() - timedelta(days=self.default_first_seen)
-        if isinstance(first_seen, datetime):
-            first_seen = first_seen.date()
-
-        url_storage_dir = get_cache_directory(self.storage_dir_riskiq, hostname, 'pdns')
-        url_storage_dir.mkdir(parents=True, exist_ok=True)
-        riskiq_file = url_storage_dir / date.today().isoformat()
-
-        if not force and riskiq_file.exists():
-            return
-
-        pdns_info = self.client_dns.get_passive_dns(query=hostname, start=first_seen.isoformat())
-        if not pdns_info or not pdns_info.get('results'):
-            try:
-                url_storage_dir.rmdir()
-            except OSError:
-                # Not empty.
-                pass
-            return
-        pdns_info['results'] = sorted(pdns_info['results'], key=lambda k: k['lastSeen'], reverse=True)
-        with riskiq_file.open('w') as _f:
-            json.dump(pdns_info, _f)

website/web/__init__.py

@@ -659,7 +659,6 @@ def historical_lookups(tree_uuid: str) -> str | WerkzeugResponse | Response:
     auto_trigger = True if (request.args.get('auto_trigger') and request.args.get('auto_trigger') == 'True') else False
     data = lookyloo.get_historical_lookups(tree_uuid, force=force, auto_trigger=auto_trigger, as_admin=flask_login.current_user.is_authenticated)
     return render_template('historical_lookups.html', tree_uuid=tree_uuid,
-                           riskiq=data.get('riskiq'),
                            circl_pdns=data.get('circl_pdns'))
 
 

website/web/templates/historical_lookups.html

@@ -1,7 +1,7 @@
 {% from "macros.html" import shorten_string %}
 
 <div>
-{% if not circl_pdns and not riskiq %}
+{% if not circl_pdns %}
   No historical data available
 {%else%}
   {% if circl_pdns %}
@@ -50,43 +50,5 @@
     {%endfor%}
     </center>
   {% endif%}
-  {% if riskiq %}
-    <hr>
-    <script type="text/javascript" nonce="{{ csp_nonce() }}">
-      new DataTable('#RiskIQ_pdns_table', {
-        order: [[ 1, "desc" ]],
-        autoWidth: false,
-        columnDefs: [{ width: '15%', targets: 0 },
-                     { width: '15%', targets: 1 },
-                     { width: '10%', targets: 2 },
-                     { width: '60%', targets: 3 }]
-      });
-    </script>
-    <center>
-      <h1 class="display-4">RiskIQ</h1>
-    <div>
-    <h3>{{riskiq['queryValue']}}</h3>
-    <h4>{{riskiq['firstSeen']}} - {{ riskiq['lastSeen']}}</h4>
-    <table id="RiskIQ_pdns_table" class="table table-striped" style="width:100%">
-      <thead>
-        <tr>
-          <th class="col-sm-2" scope="col">First seen</th>
-          <th class="col-sm-2" scope="col">Last seen</th>
-          <th class="col-sm-1" scope="col">Type</th>
-          <th class="col-sm-2" scope="col">Resolve</th>
-      </thead>
-      <tbody>
-        {% for entry in riskiq['results'] %}
-        <tr>
-          <td>{{entry['firstSeen']}}</td>
-          <td>{{entry['lastSeen']}}</td>
-          <td>{{entry['recordType']}}</td>
-          <td class="text-break">{{entry['resolve']}}</td>
-        </tr>
-        {% endfor %}
-    </table>
-    </div>
-    </center>
-  {% endif%}
 {% endif%}
 </div>