From ce8eeda9ebf95f9475524bcafd3277a1e3f04410 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 18 Jul 2022 13:08:26 +0200 Subject: [PATCH] chg: Improve RiskIQ module --- config/modules.json.sample | 3 +- lookyloo/exceptions.py | 4 ++ lookyloo/lookyloo.py | 11 ++-- lookyloo/modules/__init__.py | 2 +- lookyloo/modules/riskiq.py | 57 +++++++++++++------ website/web/__init__.py | 2 +- website/web/templates/historical_lookups.html | 6 +- 7 files changed, 59 insertions(+), 26 deletions(-) diff --git a/config/modules.json.sample b/config/modules.json.sample index c4a12cf..339f3bd 100644 --- a/config/modules.json.sample +++ b/config/modules.json.sample @@ -54,7 +54,8 @@ "RiskIQ": { "user": null, "apikey": null, - "allow_auto_trigger": false + "allow_auto_trigger": false, + "default_first_seen_in_days": 5, }, "_notes": { "apikey": "null disables the module. Pass a string otherwise.", diff --git a/lookyloo/exceptions.py b/lookyloo/exceptions.py index 23f2475..0981d1b 100644 --- a/lookyloo/exceptions.py +++ b/lookyloo/exceptions.py @@ -17,3 +17,7 @@ class MissingCaptureDirectory(LookylooException): class TreeNeedsRebuild(LookylooException): pass + + +class ModuleError(LookylooException): + pass diff --git a/lookyloo/lookyloo.py b/lookyloo/lookyloo.py index ddaea4c..76ee33b 100644 --- a/lookyloo/lookyloo.py +++ b/lookyloo/lookyloo.py @@ -35,7 +35,7 @@ from .helpers import (CaptureStatus, get_captures_dir, get_email_template, from .indexing import Indexing from .modules import (MISP, PhishingInitiative, UniversalWhois, UrlScan, VirusTotal, Phishtank, Hashlookup, - RiskIQ) + RiskIQ, RiskIQError) class Lookyloo(): @@ -295,11 +295,14 @@ class Lookyloo(): ct = self.get_crawled_tree(capture_uuid) except LookylooException: self.logger.warning(f'Unable to get the modules responses unless the tree ({capture_uuid}) is cached.') - return None + return {} to_return: Dict[str, Any] = {} if self.riskiq.available: - self.riskiq.capture_default_trigger(ct) - to_return['riskiq'] = self.riskiq.get_passivedns(ct.root_hartree.rendered_node.hostname) + try: + self.riskiq.capture_default_trigger(ct) + to_return['riskiq'] = self.riskiq.get_passivedns(ct.root_hartree.rendered_node.hostname) + except RiskIQError as e: + self.logger.warning(e.response.content) return to_return def hide_capture(self, capture_uuid: str, /) -> None: diff --git a/lookyloo/modules/__init__.py b/lookyloo/modules/__init__.py index 3d3bfe8..fef59b2 100644 --- a/lookyloo/modules/__init__.py +++ b/lookyloo/modules/__init__.py @@ -9,4 +9,4 @@ from .uwhois import UniversalWhois # noqa from .vt import VirusTotal # noqa from .phishtank import Phishtank # noqa from .hashlookup import HashlookupModule as Hashlookup # noqa -from .riskiq import RiskIQ # noqa +from .riskiq import RiskIQ, RiskIQError # noqa diff --git a/lookyloo/modules/riskiq.py b/lookyloo/modules/riskiq.py index 973c0a3..1e13970 100644 --- a/lookyloo/modules/riskiq.py +++ b/lookyloo/modules/riskiq.py @@ -1,17 +1,26 @@ #!/usr/bin/env python3 import json +import logging -from datetime import date -from typing import Any, Dict +from datetime import date, datetime, timedelta +from typing import Any, Dict, Optional, Union from har2tree import CrawledTree -from passivetotal import AccountClient, DnsRequest, WhoisRequest +from passivetotal import AccountClient, DnsRequest, WhoisRequest # type: ignore +from requests import Response -from ..default import ConfigError, get_homedir +from ..default import ConfigError, get_homedir, get_config +from ..exceptions import ModuleError from ..helpers import get_cache_directory +class RiskIQError(ModuleError): + + def __init__(self, response: Response): + self.response = response + + class RiskIQ(): def __init__(self, config: Dict[str, Any]): @@ -19,27 +28,37 @@ class RiskIQ(): self.available = False return + self.logger = logging.getLogger(f'{self.__class__.__name__}') + self.logger.setLevel(get_config('generic', 'loglevel')) + self.available = True self.allow_auto_trigger = False - test_client = AccountClient(username=config.get('user'), api_key=config.get('apikey')) + test_client = AccountClient(username=config.get('user'), api_key=config.get('apikey'), exception_class=RiskIQError) - # Check account is working - details = test_client.get_account_details() - if 'message' in details and details['message'] == 'invalid credentials': - self.available = False - raise ConfigError('RiskIQ not available, invalid credentials') - return + try: + # Check account is working + details = test_client.get_account_details() + except RiskIQError as e: + details = e.response.json() + if 'message' in details: + self.available = False + self.logger.warning(f'RiskIQ not available, {details["message"]}') + return + else: + raise e - self.client_dns = DnsRequest(username=config.get('user'), api_key=config.get('apikey')) - self.client_whois = WhoisRequest(username=config.get('user'), api_key=config.get('apikey')) + self.client_dns = DnsRequest(username=config.get('user'), api_key=config.get('apikey'), exception_class=RiskIQError) + self.client_whois = WhoisRequest(username=config.get('user'), api_key=config.get('apikey'), exception_class=RiskIQError) if config.get('allow_auto_trigger'): self.allow_auto_trigger = True + self.default_first_seen = config.get('default_first_seen_in_days', 5) + self.storage_dir_riskiq = get_homedir() / 'riskiq' self.storage_dir_riskiq.mkdir(parents=True, exist_ok=True) - def get_passivedns(self, query: str) -> Dict[str, Any]: + def get_passivedns(self, query: str) -> Optional[Dict[str, Any]]: # The query can be IP or Hostname. For now, we only do it on domains. url_storage_dir = get_cache_directory(self.storage_dir_riskiq, query, 'pdns') if not url_storage_dir.exists(): @@ -61,13 +80,18 @@ class RiskIQ(): self.pdns_lookup(crawled_tree.root_hartree.rendered_node.hostname, force) return {'success': 'Module triggered'} - def pdns_lookup(self, hostname: str, force: bool=False) -> None: + def pdns_lookup(self, hostname: str, force: bool=False, first_seen: Optional[Union[date, datetime]]=None) -> None: '''Lookup an hostname on RiskIQ Passive DNS Note: force means re-fetch the entry RiskIQ even if we already did it today ''' if not self.available: raise ConfigError('RiskIQ not available, probably no API key') + if first_seen is None: + first_seen = date.today() - timedelta(days=self.default_first_seen) + if isinstance(first_seen, datetime): + first_seen = first_seen.date() + url_storage_dir = get_cache_directory(self.storage_dir_riskiq, hostname, 'pdns') url_storage_dir.mkdir(parents=True, exist_ok=True) riskiq_file = url_storage_dir / date.today().isoformat() @@ -75,8 +99,9 @@ class RiskIQ(): if not force and riskiq_file.exists(): return - pdns_info = self.client_dns.get_passive_dns(query=hostname) + pdns_info = self.client_dns.get_passive_dns(query=hostname, start=first_seen.isoformat()) if not pdns_info: return + pdns_info['results'] = sorted(pdns_info['results'], key=lambda k: k['lastSeen'], reverse=True) with riskiq_file.open('w') as _f: json.dump(pdns_info, _f) diff --git a/website/web/__init__.py b/website/web/__init__.py index 823790e..73ce11d 100644 --- a/website/web/__init__.py +++ b/website/web/__init__.py @@ -256,7 +256,7 @@ def historical_lookups(tree_uuid: str): force = True if (request.args.get('force') and request.args.get('force') == 'True') else False data = lookyloo.get_historical_lookups(tree_uuid, force) return render_template('historical_lookups.html', tree_uuid=tree_uuid, - riskiq=data['riskiq']) + riskiq=data.get('riskiq')) @app.route('/tree//categories_capture/', defaults={'query': ''}) diff --git a/website/web/templates/historical_lookups.html b/website/web/templates/historical_lookups.html index 569dc89..4ca0413 100644 --- a/website/web/templates/historical_lookups.html +++ b/website/web/templates/historical_lookups.html @@ -11,10 +11,10 @@ - - + + - + {% for entry in riskiq['results'] %}
First seenLast seenFirst seenLast seen ResolveTypeType