mirror of https://github.com/CIRCL/lookyloo
chg: Improve RiskIQ module
parent
8b43fbf154
commit
ce8eeda9eb
|
@ -54,7 +54,8 @@
|
||||||
"RiskIQ": {
|
"RiskIQ": {
|
||||||
"user": null,
|
"user": null,
|
||||||
"apikey": null,
|
"apikey": null,
|
||||||
"allow_auto_trigger": false
|
"allow_auto_trigger": false,
|
||||||
|
"default_first_seen_in_days": 5,
|
||||||
},
|
},
|
||||||
"_notes": {
|
"_notes": {
|
||||||
"apikey": "null disables the module. Pass a string otherwise.",
|
"apikey": "null disables the module. Pass a string otherwise.",
|
||||||
|
|
|
@ -17,3 +17,7 @@ class MissingCaptureDirectory(LookylooException):
|
||||||
|
|
||||||
class TreeNeedsRebuild(LookylooException):
|
class TreeNeedsRebuild(LookylooException):
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
class ModuleError(LookylooException):
|
||||||
|
pass
|
||||||
|
|
|
@ -35,7 +35,7 @@ from .helpers import (CaptureStatus, get_captures_dir, get_email_template,
|
||||||
from .indexing import Indexing
|
from .indexing import Indexing
|
||||||
from .modules import (MISP, PhishingInitiative, UniversalWhois,
|
from .modules import (MISP, PhishingInitiative, UniversalWhois,
|
||||||
UrlScan, VirusTotal, Phishtank, Hashlookup,
|
UrlScan, VirusTotal, Phishtank, Hashlookup,
|
||||||
RiskIQ)
|
RiskIQ, RiskIQError)
|
||||||
|
|
||||||
|
|
||||||
class Lookyloo():
|
class Lookyloo():
|
||||||
|
@ -295,11 +295,14 @@ class Lookyloo():
|
||||||
ct = self.get_crawled_tree(capture_uuid)
|
ct = self.get_crawled_tree(capture_uuid)
|
||||||
except LookylooException:
|
except LookylooException:
|
||||||
self.logger.warning(f'Unable to get the modules responses unless the tree ({capture_uuid}) is cached.')
|
self.logger.warning(f'Unable to get the modules responses unless the tree ({capture_uuid}) is cached.')
|
||||||
return None
|
return {}
|
||||||
to_return: Dict[str, Any] = {}
|
to_return: Dict[str, Any] = {}
|
||||||
if self.riskiq.available:
|
if self.riskiq.available:
|
||||||
|
try:
|
||||||
self.riskiq.capture_default_trigger(ct)
|
self.riskiq.capture_default_trigger(ct)
|
||||||
to_return['riskiq'] = self.riskiq.get_passivedns(ct.root_hartree.rendered_node.hostname)
|
to_return['riskiq'] = self.riskiq.get_passivedns(ct.root_hartree.rendered_node.hostname)
|
||||||
|
except RiskIQError as e:
|
||||||
|
self.logger.warning(e.response.content)
|
||||||
return to_return
|
return to_return
|
||||||
|
|
||||||
def hide_capture(self, capture_uuid: str, /) -> None:
|
def hide_capture(self, capture_uuid: str, /) -> None:
|
||||||
|
|
|
@ -9,4 +9,4 @@ from .uwhois import UniversalWhois # noqa
|
||||||
from .vt import VirusTotal # noqa
|
from .vt import VirusTotal # noqa
|
||||||
from .phishtank import Phishtank # noqa
|
from .phishtank import Phishtank # noqa
|
||||||
from .hashlookup import HashlookupModule as Hashlookup # noqa
|
from .hashlookup import HashlookupModule as Hashlookup # noqa
|
||||||
from .riskiq import RiskIQ # noqa
|
from .riskiq import RiskIQ, RiskIQError # noqa
|
||||||
|
|
|
@ -1,17 +1,26 @@
|
||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
import json
|
import json
|
||||||
|
import logging
|
||||||
|
|
||||||
from datetime import date
|
from datetime import date, datetime, timedelta
|
||||||
from typing import Any, Dict
|
from typing import Any, Dict, Optional, Union
|
||||||
|
|
||||||
from har2tree import CrawledTree
|
from har2tree import CrawledTree
|
||||||
from passivetotal import AccountClient, DnsRequest, WhoisRequest
|
from passivetotal import AccountClient, DnsRequest, WhoisRequest # type: ignore
|
||||||
|
from requests import Response
|
||||||
|
|
||||||
from ..default import ConfigError, get_homedir
|
from ..default import ConfigError, get_homedir, get_config
|
||||||
|
from ..exceptions import ModuleError
|
||||||
from ..helpers import get_cache_directory
|
from ..helpers import get_cache_directory
|
||||||
|
|
||||||
|
|
||||||
|
class RiskIQError(ModuleError):
|
||||||
|
|
||||||
|
def __init__(self, response: Response):
|
||||||
|
self.response = response
|
||||||
|
|
||||||
|
|
||||||
class RiskIQ():
|
class RiskIQ():
|
||||||
|
|
||||||
def __init__(self, config: Dict[str, Any]):
|
def __init__(self, config: Dict[str, Any]):
|
||||||
|
@ -19,27 +28,37 @@ class RiskIQ():
|
||||||
self.available = False
|
self.available = False
|
||||||
return
|
return
|
||||||
|
|
||||||
|
self.logger = logging.getLogger(f'{self.__class__.__name__}')
|
||||||
|
self.logger.setLevel(get_config('generic', 'loglevel'))
|
||||||
|
|
||||||
self.available = True
|
self.available = True
|
||||||
self.allow_auto_trigger = False
|
self.allow_auto_trigger = False
|
||||||
test_client = AccountClient(username=config.get('user'), api_key=config.get('apikey'))
|
test_client = AccountClient(username=config.get('user'), api_key=config.get('apikey'), exception_class=RiskIQError)
|
||||||
|
|
||||||
|
try:
|
||||||
# Check account is working
|
# Check account is working
|
||||||
details = test_client.get_account_details()
|
details = test_client.get_account_details()
|
||||||
if 'message' in details and details['message'] == 'invalid credentials':
|
except RiskIQError as e:
|
||||||
|
details = e.response.json()
|
||||||
|
if 'message' in details:
|
||||||
self.available = False
|
self.available = False
|
||||||
raise ConfigError('RiskIQ not available, invalid credentials')
|
self.logger.warning(f'RiskIQ not available, {details["message"]}')
|
||||||
return
|
return
|
||||||
|
else:
|
||||||
|
raise e
|
||||||
|
|
||||||
self.client_dns = DnsRequest(username=config.get('user'), api_key=config.get('apikey'))
|
self.client_dns = DnsRequest(username=config.get('user'), api_key=config.get('apikey'), exception_class=RiskIQError)
|
||||||
self.client_whois = WhoisRequest(username=config.get('user'), api_key=config.get('apikey'))
|
self.client_whois = WhoisRequest(username=config.get('user'), api_key=config.get('apikey'), exception_class=RiskIQError)
|
||||||
|
|
||||||
if config.get('allow_auto_trigger'):
|
if config.get('allow_auto_trigger'):
|
||||||
self.allow_auto_trigger = True
|
self.allow_auto_trigger = True
|
||||||
|
|
||||||
|
self.default_first_seen = config.get('default_first_seen_in_days', 5)
|
||||||
|
|
||||||
self.storage_dir_riskiq = get_homedir() / 'riskiq'
|
self.storage_dir_riskiq = get_homedir() / 'riskiq'
|
||||||
self.storage_dir_riskiq.mkdir(parents=True, exist_ok=True)
|
self.storage_dir_riskiq.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
def get_passivedns(self, query: str) -> Dict[str, Any]:
|
def get_passivedns(self, query: str) -> Optional[Dict[str, Any]]:
|
||||||
# The query can be IP or Hostname. For now, we only do it on domains.
|
# The query can be IP or Hostname. For now, we only do it on domains.
|
||||||
url_storage_dir = get_cache_directory(self.storage_dir_riskiq, query, 'pdns')
|
url_storage_dir = get_cache_directory(self.storage_dir_riskiq, query, 'pdns')
|
||||||
if not url_storage_dir.exists():
|
if not url_storage_dir.exists():
|
||||||
|
@ -61,13 +80,18 @@ class RiskIQ():
|
||||||
self.pdns_lookup(crawled_tree.root_hartree.rendered_node.hostname, force)
|
self.pdns_lookup(crawled_tree.root_hartree.rendered_node.hostname, force)
|
||||||
return {'success': 'Module triggered'}
|
return {'success': 'Module triggered'}
|
||||||
|
|
||||||
def pdns_lookup(self, hostname: str, force: bool=False) -> None:
|
def pdns_lookup(self, hostname: str, force: bool=False, first_seen: Optional[Union[date, datetime]]=None) -> None:
|
||||||
'''Lookup an hostname on RiskIQ Passive DNS
|
'''Lookup an hostname on RiskIQ Passive DNS
|
||||||
Note: force means re-fetch the entry RiskIQ even if we already did it today
|
Note: force means re-fetch the entry RiskIQ even if we already did it today
|
||||||
'''
|
'''
|
||||||
if not self.available:
|
if not self.available:
|
||||||
raise ConfigError('RiskIQ not available, probably no API key')
|
raise ConfigError('RiskIQ not available, probably no API key')
|
||||||
|
|
||||||
|
if first_seen is None:
|
||||||
|
first_seen = date.today() - timedelta(days=self.default_first_seen)
|
||||||
|
if isinstance(first_seen, datetime):
|
||||||
|
first_seen = first_seen.date()
|
||||||
|
|
||||||
url_storage_dir = get_cache_directory(self.storage_dir_riskiq, hostname, 'pdns')
|
url_storage_dir = get_cache_directory(self.storage_dir_riskiq, hostname, 'pdns')
|
||||||
url_storage_dir.mkdir(parents=True, exist_ok=True)
|
url_storage_dir.mkdir(parents=True, exist_ok=True)
|
||||||
riskiq_file = url_storage_dir / date.today().isoformat()
|
riskiq_file = url_storage_dir / date.today().isoformat()
|
||||||
|
@ -75,8 +99,9 @@ class RiskIQ():
|
||||||
if not force and riskiq_file.exists():
|
if not force and riskiq_file.exists():
|
||||||
return
|
return
|
||||||
|
|
||||||
pdns_info = self.client_dns.get_passive_dns(query=hostname)
|
pdns_info = self.client_dns.get_passive_dns(query=hostname, start=first_seen.isoformat())
|
||||||
if not pdns_info:
|
if not pdns_info:
|
||||||
return
|
return
|
||||||
|
pdns_info['results'] = sorted(pdns_info['results'], key=lambda k: k['lastSeen'], reverse=True)
|
||||||
with riskiq_file.open('w') as _f:
|
with riskiq_file.open('w') as _f:
|
||||||
json.dump(pdns_info, _f)
|
json.dump(pdns_info, _f)
|
||||||
|
|
|
@ -256,7 +256,7 @@ def historical_lookups(tree_uuid: str):
|
||||||
force = True if (request.args.get('force') and request.args.get('force') == 'True') else False
|
force = True if (request.args.get('force') and request.args.get('force') == 'True') else False
|
||||||
data = lookyloo.get_historical_lookups(tree_uuid, force)
|
data = lookyloo.get_historical_lookups(tree_uuid, force)
|
||||||
return render_template('historical_lookups.html', tree_uuid=tree_uuid,
|
return render_template('historical_lookups.html', tree_uuid=tree_uuid,
|
||||||
riskiq=data['riskiq'])
|
riskiq=data.get('riskiq'))
|
||||||
|
|
||||||
|
|
||||||
@app.route('/tree/<string:tree_uuid>/categories_capture/', defaults={'query': ''})
|
@app.route('/tree/<string:tree_uuid>/categories_capture/', defaults={'query': ''})
|
||||||
|
|
|
@ -11,10 +11,10 @@
|
||||||
<table class="table">
|
<table class="table">
|
||||||
<thead>
|
<thead>
|
||||||
<tr>
|
<tr>
|
||||||
<th scope="col">First seen</th>
|
<th class="col-sm-2" scope="col">First seen</th>
|
||||||
<th scope="col">Last seen</th>
|
<th class="col-sm-2" scope="col">Last seen</th>
|
||||||
<th scope="col">Resolve</th>
|
<th scope="col">Resolve</th>
|
||||||
<th scope="col">Type</th>
|
<th class="col-sm-1" scope="col">Type</th>
|
||||||
</thead>
|
</thead>
|
||||||
<tbody>
|
<tbody>
|
||||||
{% for entry in riskiq['results'] %}
|
{% for entry in riskiq['results'] %}
|
||||||
|
|
Loading…
Reference in New Issue