chg: Improve RiskIQ module

pull/463/head
Raphaël Vinot 2022-07-18 13:08:26 +02:00
parent 8b43fbf154
commit ce8eeda9eb
7 changed files with 59 additions and 26 deletions

View File

@ -54,7 +54,8 @@
"RiskIQ": { "RiskIQ": {
"user": null, "user": null,
"apikey": null, "apikey": null,
"allow_auto_trigger": false "allow_auto_trigger": false,
"default_first_seen_in_days": 5,
}, },
"_notes": { "_notes": {
"apikey": "null disables the module. Pass a string otherwise.", "apikey": "null disables the module. Pass a string otherwise.",

View File

@ -17,3 +17,7 @@ class MissingCaptureDirectory(LookylooException):
class TreeNeedsRebuild(LookylooException): class TreeNeedsRebuild(LookylooException):
pass pass
class ModuleError(LookylooException):
pass

View File

@ -35,7 +35,7 @@ from .helpers import (CaptureStatus, get_captures_dir, get_email_template,
from .indexing import Indexing from .indexing import Indexing
from .modules import (MISP, PhishingInitiative, UniversalWhois, from .modules import (MISP, PhishingInitiative, UniversalWhois,
UrlScan, VirusTotal, Phishtank, Hashlookup, UrlScan, VirusTotal, Phishtank, Hashlookup,
RiskIQ) RiskIQ, RiskIQError)
class Lookyloo(): class Lookyloo():
@ -295,11 +295,14 @@ class Lookyloo():
ct = self.get_crawled_tree(capture_uuid) ct = self.get_crawled_tree(capture_uuid)
except LookylooException: except LookylooException:
self.logger.warning(f'Unable to get the modules responses unless the tree ({capture_uuid}) is cached.') self.logger.warning(f'Unable to get the modules responses unless the tree ({capture_uuid}) is cached.')
return None return {}
to_return: Dict[str, Any] = {} to_return: Dict[str, Any] = {}
if self.riskiq.available: if self.riskiq.available:
self.riskiq.capture_default_trigger(ct) try:
to_return['riskiq'] = self.riskiq.get_passivedns(ct.root_hartree.rendered_node.hostname) self.riskiq.capture_default_trigger(ct)
to_return['riskiq'] = self.riskiq.get_passivedns(ct.root_hartree.rendered_node.hostname)
except RiskIQError as e:
self.logger.warning(e.response.content)
return to_return return to_return
def hide_capture(self, capture_uuid: str, /) -> None: def hide_capture(self, capture_uuid: str, /) -> None:

View File

@ -9,4 +9,4 @@ from .uwhois import UniversalWhois # noqa
from .vt import VirusTotal # noqa from .vt import VirusTotal # noqa
from .phishtank import Phishtank # noqa from .phishtank import Phishtank # noqa
from .hashlookup import HashlookupModule as Hashlookup # noqa from .hashlookup import HashlookupModule as Hashlookup # noqa
from .riskiq import RiskIQ # noqa from .riskiq import RiskIQ, RiskIQError # noqa

View File

@ -1,17 +1,26 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
import json import json
import logging
from datetime import date from datetime import date, datetime, timedelta
from typing import Any, Dict from typing import Any, Dict, Optional, Union
from har2tree import CrawledTree from har2tree import CrawledTree
from passivetotal import AccountClient, DnsRequest, WhoisRequest from passivetotal import AccountClient, DnsRequest, WhoisRequest # type: ignore
from requests import Response
from ..default import ConfigError, get_homedir from ..default import ConfigError, get_homedir, get_config
from ..exceptions import ModuleError
from ..helpers import get_cache_directory from ..helpers import get_cache_directory
class RiskIQError(ModuleError):
def __init__(self, response: Response):
self.response = response
class RiskIQ(): class RiskIQ():
def __init__(self, config: Dict[str, Any]): def __init__(self, config: Dict[str, Any]):
@ -19,27 +28,37 @@ class RiskIQ():
self.available = False self.available = False
return return
self.logger = logging.getLogger(f'{self.__class__.__name__}')
self.logger.setLevel(get_config('generic', 'loglevel'))
self.available = True self.available = True
self.allow_auto_trigger = False self.allow_auto_trigger = False
test_client = AccountClient(username=config.get('user'), api_key=config.get('apikey')) test_client = AccountClient(username=config.get('user'), api_key=config.get('apikey'), exception_class=RiskIQError)
# Check account is working try:
details = test_client.get_account_details() # Check account is working
if 'message' in details and details['message'] == 'invalid credentials': details = test_client.get_account_details()
self.available = False except RiskIQError as e:
raise ConfigError('RiskIQ not available, invalid credentials') details = e.response.json()
return if 'message' in details:
self.available = False
self.logger.warning(f'RiskIQ not available, {details["message"]}')
return
else:
raise e
self.client_dns = DnsRequest(username=config.get('user'), api_key=config.get('apikey')) self.client_dns = DnsRequest(username=config.get('user'), api_key=config.get('apikey'), exception_class=RiskIQError)
self.client_whois = WhoisRequest(username=config.get('user'), api_key=config.get('apikey')) self.client_whois = WhoisRequest(username=config.get('user'), api_key=config.get('apikey'), exception_class=RiskIQError)
if config.get('allow_auto_trigger'): if config.get('allow_auto_trigger'):
self.allow_auto_trigger = True self.allow_auto_trigger = True
self.default_first_seen = config.get('default_first_seen_in_days', 5)
self.storage_dir_riskiq = get_homedir() / 'riskiq' self.storage_dir_riskiq = get_homedir() / 'riskiq'
self.storage_dir_riskiq.mkdir(parents=True, exist_ok=True) self.storage_dir_riskiq.mkdir(parents=True, exist_ok=True)
def get_passivedns(self, query: str) -> Dict[str, Any]: def get_passivedns(self, query: str) -> Optional[Dict[str, Any]]:
# The query can be IP or Hostname. For now, we only do it on domains. # The query can be IP or Hostname. For now, we only do it on domains.
url_storage_dir = get_cache_directory(self.storage_dir_riskiq, query, 'pdns') url_storage_dir = get_cache_directory(self.storage_dir_riskiq, query, 'pdns')
if not url_storage_dir.exists(): if not url_storage_dir.exists():
@ -61,13 +80,18 @@ class RiskIQ():
self.pdns_lookup(crawled_tree.root_hartree.rendered_node.hostname, force) self.pdns_lookup(crawled_tree.root_hartree.rendered_node.hostname, force)
return {'success': 'Module triggered'} return {'success': 'Module triggered'}
def pdns_lookup(self, hostname: str, force: bool=False) -> None: def pdns_lookup(self, hostname: str, force: bool=False, first_seen: Optional[Union[date, datetime]]=None) -> None:
'''Lookup an hostname on RiskIQ Passive DNS '''Lookup an hostname on RiskIQ Passive DNS
Note: force means re-fetch the entry RiskIQ even if we already did it today Note: force means re-fetch the entry RiskIQ even if we already did it today
''' '''
if not self.available: if not self.available:
raise ConfigError('RiskIQ not available, probably no API key') raise ConfigError('RiskIQ not available, probably no API key')
if first_seen is None:
first_seen = date.today() - timedelta(days=self.default_first_seen)
if isinstance(first_seen, datetime):
first_seen = first_seen.date()
url_storage_dir = get_cache_directory(self.storage_dir_riskiq, hostname, 'pdns') url_storage_dir = get_cache_directory(self.storage_dir_riskiq, hostname, 'pdns')
url_storage_dir.mkdir(parents=True, exist_ok=True) url_storage_dir.mkdir(parents=True, exist_ok=True)
riskiq_file = url_storage_dir / date.today().isoformat() riskiq_file = url_storage_dir / date.today().isoformat()
@ -75,8 +99,9 @@ class RiskIQ():
if not force and riskiq_file.exists(): if not force and riskiq_file.exists():
return return
pdns_info = self.client_dns.get_passive_dns(query=hostname) pdns_info = self.client_dns.get_passive_dns(query=hostname, start=first_seen.isoformat())
if not pdns_info: if not pdns_info:
return return
pdns_info['results'] = sorted(pdns_info['results'], key=lambda k: k['lastSeen'], reverse=True)
with riskiq_file.open('w') as _f: with riskiq_file.open('w') as _f:
json.dump(pdns_info, _f) json.dump(pdns_info, _f)

View File

@ -256,7 +256,7 @@ def historical_lookups(tree_uuid: str):
force = True if (request.args.get('force') and request.args.get('force') == 'True') else False force = True if (request.args.get('force') and request.args.get('force') == 'True') else False
data = lookyloo.get_historical_lookups(tree_uuid, force) data = lookyloo.get_historical_lookups(tree_uuid, force)
return render_template('historical_lookups.html', tree_uuid=tree_uuid, return render_template('historical_lookups.html', tree_uuid=tree_uuid,
riskiq=data['riskiq']) riskiq=data.get('riskiq'))
@app.route('/tree/<string:tree_uuid>/categories_capture/', defaults={'query': ''}) @app.route('/tree/<string:tree_uuid>/categories_capture/', defaults={'query': ''})

View File

@ -11,10 +11,10 @@
<table class="table"> <table class="table">
<thead> <thead>
<tr> <tr>
<th scope="col">First seen</th> <th class="col-sm-2" scope="col">First seen</th>
<th scope="col">Last seen</th> <th class="col-sm-2" scope="col">Last seen</th>
<th scope="col">Resolve</th> <th scope="col">Resolve</th>
<th scope="col">Type</th> <th class="col-sm-1" scope="col">Type</th>
</thead> </thead>
<tbody> <tbody>
{% for entry in riskiq['results'] %} {% for entry in riskiq['results'] %}