chg: Improve RiskIQ module

config/modules.json.sample

@@ -54,7 +54,8 @@
   "RiskIQ": {
     "user": null,
     "apikey": null,
-    "allow_auto_trigger": false
+    "allow_auto_trigger": false,
+    "default_first_seen_in_days": 5,
   },
   "_notes": {
     "apikey": "null disables the module. Pass a string otherwise.",

lookyloo/exceptions.py

@@ -17,3 +17,7 @@ class MissingCaptureDirectory(LookylooException):
 
 class TreeNeedsRebuild(LookylooException):
     pass
+
+
+class ModuleError(LookylooException):
+    pass

lookyloo/lookyloo.py

@@ -35,7 +35,7 @@ from .helpers import (CaptureStatus, get_captures_dir, get_email_template,
 from .indexing import Indexing
 from .modules import (MISP, PhishingInitiative, UniversalWhois,
                       UrlScan, VirusTotal, Phishtank, Hashlookup,
-                      RiskIQ)
+                      RiskIQ, RiskIQError)
 
 
 class Lookyloo():
@@ -295,11 +295,14 @@ class Lookyloo():
             ct = self.get_crawled_tree(capture_uuid)
         except LookylooException:
             self.logger.warning(f'Unable to get the modules responses unless the tree ({capture_uuid}) is cached.')
-            return None
+            return {}
         to_return: Dict[str, Any] = {}
         if self.riskiq.available:
-            self.riskiq.capture_default_trigger(ct)
+            try:
-            to_return['riskiq'] = self.riskiq.get_passivedns(ct.root_hartree.rendered_node.hostname)
+                self.riskiq.capture_default_trigger(ct)
+                to_return['riskiq'] = self.riskiq.get_passivedns(ct.root_hartree.rendered_node.hostname)
+            except RiskIQError as e:
+                self.logger.warning(e.response.content)
         return to_return
 
     def hide_capture(self, capture_uuid: str, /) -> None:

lookyloo/modules/__init__.py

@@ -9,4 +9,4 @@ from .uwhois import UniversalWhois  # noqa
 from .vt import VirusTotal  # noqa
 from .phishtank import Phishtank  # noqa
 from .hashlookup import HashlookupModule as Hashlookup  # noqa
-from .riskiq import RiskIQ  # noqa
+from .riskiq import RiskIQ, RiskIQError  # noqa

lookyloo/modules/riskiq.py

@@ -1,17 +1,26 @@
 #!/usr/bin/env python3
 
 import json
+import logging
 
-from datetime import date
+from datetime import date, datetime, timedelta
-from typing import Any, Dict
+from typing import Any, Dict, Optional, Union
 
 from har2tree import CrawledTree
-from passivetotal import AccountClient, DnsRequest, WhoisRequest
+from passivetotal import AccountClient, DnsRequest, WhoisRequest  # type: ignore
+from requests import Response
 
-from ..default import ConfigError, get_homedir
+from ..default import ConfigError, get_homedir, get_config
+from ..exceptions import ModuleError
 from ..helpers import get_cache_directory
 
 
+class RiskIQError(ModuleError):
+
+    def __init__(self, response: Response):
+        self.response = response
+
+
 class RiskIQ():
 
     def __init__(self, config: Dict[str, Any]):
@@ -19,27 +28,37 @@ class RiskIQ():
             self.available = False
             return
 
+        self.logger = logging.getLogger(f'{self.__class__.__name__}')
+        self.logger.setLevel(get_config('generic', 'loglevel'))
+
         self.available = True
         self.allow_auto_trigger = False
-        test_client = AccountClient(username=config.get('user'), api_key=config.get('apikey'))
+        test_client = AccountClient(username=config.get('user'), api_key=config.get('apikey'), exception_class=RiskIQError)
 
-        # Check account is working
+        try:
-        details = test_client.get_account_details()
+            # Check account is working
-        if 'message' in details and details['message'] == 'invalid credentials':
+            details = test_client.get_account_details()
-            self.available = False
+        except RiskIQError as e:
-            raise ConfigError('RiskIQ not available, invalid credentials')
+            details = e.response.json()
-            return
+            if 'message' in details:
+                self.available = False
+                self.logger.warning(f'RiskIQ not available, {details["message"]}')
+                return
+            else:
+                raise e
 
-        self.client_dns = DnsRequest(username=config.get('user'), api_key=config.get('apikey'))
+        self.client_dns = DnsRequest(username=config.get('user'), api_key=config.get('apikey'), exception_class=RiskIQError)
-        self.client_whois = WhoisRequest(username=config.get('user'), api_key=config.get('apikey'))
+        self.client_whois = WhoisRequest(username=config.get('user'), api_key=config.get('apikey'), exception_class=RiskIQError)
 
         if config.get('allow_auto_trigger'):
             self.allow_auto_trigger = True
 
+        self.default_first_seen = config.get('default_first_seen_in_days', 5)
+
         self.storage_dir_riskiq = get_homedir() / 'riskiq'
         self.storage_dir_riskiq.mkdir(parents=True, exist_ok=True)
 
-    def get_passivedns(self, query: str) -> Dict[str, Any]:
+    def get_passivedns(self, query: str) -> Optional[Dict[str, Any]]:
         # The query can be IP or Hostname. For now, we only do it on domains.
         url_storage_dir = get_cache_directory(self.storage_dir_riskiq, query, 'pdns')
         if not url_storage_dir.exists():
@@ -61,13 +80,18 @@ class RiskIQ():
         self.pdns_lookup(crawled_tree.root_hartree.rendered_node.hostname, force)
         return {'success': 'Module triggered'}
 
-    def pdns_lookup(self, hostname: str, force: bool=False) -> None:
+    def pdns_lookup(self, hostname: str, force: bool=False, first_seen: Optional[Union[date, datetime]]=None) -> None:
         '''Lookup an hostname on RiskIQ Passive DNS
         Note: force means re-fetch the entry RiskIQ even if we already did it today
         '''
         if not self.available:
             raise ConfigError('RiskIQ not available, probably no API key')
 
+        if first_seen is None:
+            first_seen = date.today() - timedelta(days=self.default_first_seen)
+        if isinstance(first_seen, datetime):
+            first_seen = first_seen.date()
+
         url_storage_dir = get_cache_directory(self.storage_dir_riskiq, hostname, 'pdns')
         url_storage_dir.mkdir(parents=True, exist_ok=True)
         riskiq_file = url_storage_dir / date.today().isoformat()
@@ -75,8 +99,9 @@ class RiskIQ():
         if not force and riskiq_file.exists():
             return
 
-        pdns_info = self.client_dns.get_passive_dns(query=hostname)
+        pdns_info = self.client_dns.get_passive_dns(query=hostname, start=first_seen.isoformat())
         if not pdns_info:
             return
+        pdns_info['results'] = sorted(pdns_info['results'], key=lambda k: k['lastSeen'], reverse=True)
         with riskiq_file.open('w') as _f:
             json.dump(pdns_info, _f)

website/web/__init__.py

@@ -256,7 +256,7 @@ def historical_lookups(tree_uuid: str):
     force = True if (request.args.get('force') and request.args.get('force') == 'True') else False
     data = lookyloo.get_historical_lookups(tree_uuid, force)
     return render_template('historical_lookups.html', tree_uuid=tree_uuid,
-                           riskiq=data['riskiq'])
+                           riskiq=data.get('riskiq'))
 
 
 @app.route('/tree/<string:tree_uuid>/categories_capture/', defaults={'query': ''})

website/web/templates/historical_lookups.html

@@ -11,10 +11,10 @@
   <table class="table">
     <thead>
       <tr>
-        <th scope="col">First seen</th>
+        <th class="col-sm-2" scope="col">First seen</th>
-        <th scope="col">Last seen</th>
+        <th class="col-sm-2" scope="col">Last seen</th>
         <th scope="col">Resolve</th>
-        <th scope="col">Type</th>
+        <th class="col-sm-1" scope="col">Type</th>
     </thead>
     <tbody>
       {% for entry in riskiq['results'] %}