mirror of https://github.com/CIRCL/lookyloo
chg: Remove RiskIQ, killed by Microsoft.
Raphaël Vinot
parent
830878e2bc
commit
b6399b0a95
|
|
@ -80,13 +80,6 @@
|
|||
"allow_auto_trigger": true,
|
||||
"admin_only": false
|
||||
},
|
||||
"RiskIQ": {
|
||||
"user": null,
|
||||
"apikey": null,
|
||||
"allow_auto_trigger": false,
|
||||
"default_first_seen_in_days": 5,
|
||||
"admin_only": true
|
||||
},
|
||||
"CIRCLPDNS": {
|
||||
"user": null,
|
||||
"password": null,
|
||||
|
|
@ -109,7 +102,6 @@
|
|||
"Hashlookup": "Module to query Hashlookup (https://github.com/adulau/hashlookup-server). URL set to none means querying the public instance.",
|
||||
"FOX": "Submission only interface by and for CCCS",
|
||||
"Pandora": "Submission only interface for https://github.com/pandora-analysis/",
|
||||
"RiskIQ": "Module to query RiskIQ (https://community.riskiq.com/)",
|
||||
"CIRCLPDNS": "Module to query CIRCL Passive DNS (https://www.circl.lu/services/passive-dns/)"
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -65,7 +65,7 @@ from .helpers import (get_captures_dir, get_email_template,
|
|||
)
|
||||
from .modules import (MISPs, PhishingInitiative, UniversalWhois,
|
||||
UrlScan, VirusTotal, Phishtank, Hashlookup,
|
||||
RiskIQ, RiskIQError, Pandora, URLhaus, CIRCLPDNS)
|
||||
Pandora, URLhaus, CIRCLPDNS)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from playwright.async_api import Cookie
|
||||
|
|
@ -131,7 +131,6 @@ class Lookyloo():
|
|||
self.urlscan = UrlScan(config_name='UrlScan')
|
||||
self.phishtank = Phishtank(config_name='Phishtank')
|
||||
self.hashlookup = Hashlookup(config_name='Hashlookup')
|
||||
self.riskiq = RiskIQ(config_name='RiskIQ')
|
||||
self.pandora = Pandora()
|
||||
self.urlhaus = URLhaus(config_name='URLhaus')
|
||||
self.circl_pdns = CIRCLPDNS(config_name='CIRCLPDNS')
|
||||
|
|
@ -449,18 +448,6 @@ class Lookyloo():
|
|||
self.logger.warning(f'Unable to get the modules responses unless the capture {capture_uuid} is cached')
|
||||
return {}
|
||||
to_return: dict[str, Any] = defaultdict(dict)
|
||||
if self.riskiq.available:
|
||||
try:
|
||||
self.riskiq.capture_default_trigger(cache, force=force, auto_trigger=auto_trigger, as_admin=as_admin)
|
||||
if hasattr(cache, 'redirects') and cache.redirects:
|
||||
hostname = urlparse(cache.redirects[-1]).hostname
|
||||
else:
|
||||
hostname = urlparse(cache.url).hostname
|
||||
if hostname:
|
||||
if _riskiq_entries := self.riskiq.get_passivedns(hostname):
|
||||
to_return['riskiq'] = _riskiq_entries
|
||||
except RiskIQError as e:
|
||||
self.logger.warning(e.response.content)
|
||||
if self.circl_pdns.available:
|
||||
self.circl_pdns.capture_default_trigger(cache, force=force, auto_trigger=auto_trigger, as_admin=as_admin)
|
||||
if hasattr(cache, 'redirects') and cache.redirects:
|
||||
|
|
|
|||
|
|
@ -10,7 +10,6 @@ from .vt import VirusTotal # noqa
|
|||
from .pandora import Pandora # noqa
|
||||
from .phishtank import Phishtank # noqa
|
||||
from .hashlookup import HashlookupModule as Hashlookup # noqa
|
||||
from .riskiq import RiskIQ, RiskIQError # noqa
|
||||
from .urlhaus import URLhaus # noqa
|
||||
from .cloudflare import Cloudflare # noqa
|
||||
from .circlpdns import CIRCLPDNS # noqa
|
||||
|
|
@ -27,8 +26,6 @@ __all__ = [
|
|||
'Pandora',
|
||||
'Phishtank',
|
||||
'Hashlookup',
|
||||
'RiskIQ',
|
||||
'RiskIQError',
|
||||
'URLhaus',
|
||||
'Cloudflare',
|
||||
'CIRCLPDNS'
|
||||
|
|
|
|||
|
|
@ -1,122 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
from datetime import date, datetime, timedelta
|
||||
from typing import Any, TYPE_CHECKING
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from passivetotal import AccountClient, DnsRequest, WhoisRequest # type: ignore[import-untyped]
|
||||
from requests import Response
|
||||
|
||||
from ..default import ConfigError, get_homedir
|
||||
from ..exceptions import ModuleError
|
||||
from ..helpers import get_cache_directory
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ..capturecache import CaptureCache
|
||||
|
||||
from .abstractmodule import AbstractModule
|
||||
|
||||
|
||||
class RiskIQError(ModuleError):
|
||||
|
||||
def __init__(self, response: Response):
|
||||
self.response = response
|
||||
|
||||
|
||||
class RiskIQ(AbstractModule):
|
||||
|
||||
def module_init(self) -> bool:
|
||||
if not (self.config.get('user') and self.config.get('apikey')):
|
||||
self.logger.info('Missing credentials.')
|
||||
return False
|
||||
|
||||
try:
|
||||
# Check if account is working
|
||||
test_client = AccountClient(username=self.config.get('user'),
|
||||
api_key=self.config.get('apikey'), exception_class=RiskIQError)
|
||||
details = test_client.get_account_details()
|
||||
self.client_dns = DnsRequest(username=self.config.get('user'),
|
||||
api_key=self.config.get('apikey'), exception_class=RiskIQError)
|
||||
self.client_whois = WhoisRequest(username=self.config.get('user'),
|
||||
api_key=self.config.get('apikey'), exception_class=RiskIQError)
|
||||
except RiskIQError as e:
|
||||
details = e.response.json()
|
||||
if 'message' in details:
|
||||
self.logger.warning(f'RiskIQ not available: {details["message"]}')
|
||||
else:
|
||||
self.logger.warning(f'RiskIQ not available: {details}')
|
||||
return False
|
||||
|
||||
self.default_first_seen = self.config.get('default_first_seen_in_days', 5)
|
||||
|
||||
self.storage_dir_riskiq = get_homedir() / 'riskiq'
|
||||
self.storage_dir_riskiq.mkdir(parents=True, exist_ok=True)
|
||||
return True
|
||||
|
||||
def get_passivedns(self, query: str) -> dict[str, Any] | None:
|
||||
# The query can be IP or Hostname. For now, we only do it on domains.
|
||||
url_storage_dir = get_cache_directory(self.storage_dir_riskiq, query, 'pdns')
|
||||
if not url_storage_dir.exists():
|
||||
return None
|
||||
cached_entries = sorted(url_storage_dir.glob('*'), reverse=True)
|
||||
if not cached_entries:
|
||||
return None
|
||||
|
||||
with cached_entries[0].open() as f:
|
||||
return json.load(f)
|
||||
|
||||
def capture_default_trigger(self, cache: CaptureCache, /, *, force: bool,
|
||||
auto_trigger: bool, as_admin: bool) -> dict[str, str]:
|
||||
'''Run the module on all the nodes up to the final redirect'''
|
||||
|
||||
if error := super().capture_default_trigger(cache, force=force, auto_trigger=auto_trigger, as_admin=as_admin):
|
||||
return error
|
||||
|
||||
if cache.url.startswith('file'):
|
||||
return {'error': 'RiskIQ does not support files.'}
|
||||
|
||||
if cache.redirects:
|
||||
hostname = urlparse(cache.redirects[-1]).hostname
|
||||
else:
|
||||
hostname = urlparse(cache.url).hostname
|
||||
|
||||
if not hostname:
|
||||
return {'error': 'No hostname found.'}
|
||||
|
||||
self.__pdns_lookup(hostname, force)
|
||||
return {'success': 'Module triggered'}
|
||||
|
||||
def __pdns_lookup(self, hostname: str, force: bool=False, first_seen: date | datetime | None=None) -> None:
|
||||
'''Lookup an hostname on RiskIQ Passive DNS
|
||||
Note: force means re-fetch the entry RiskIQ even if we already did it today
|
||||
'''
|
||||
if not self.available:
|
||||
raise ConfigError('RiskIQ not available, probably no API key')
|
||||
|
||||
if first_seen is None:
|
||||
first_seen = date.today() - timedelta(days=self.default_first_seen)
|
||||
if isinstance(first_seen, datetime):
|
||||
first_seen = first_seen.date()
|
||||
|
||||
url_storage_dir = get_cache_directory(self.storage_dir_riskiq, hostname, 'pdns')
|
||||
url_storage_dir.mkdir(parents=True, exist_ok=True)
|
||||
riskiq_file = url_storage_dir / date.today().isoformat()
|
||||
|
||||
if not force and riskiq_file.exists():
|
||||
return
|
||||
|
||||
pdns_info = self.client_dns.get_passive_dns(query=hostname, start=first_seen.isoformat())
|
||||
if not pdns_info or not pdns_info.get('results'):
|
||||
try:
|
||||
url_storage_dir.rmdir()
|
||||
except OSError:
|
||||
# Not empty.
|
||||
pass
|
||||
return
|
||||
pdns_info['results'] = sorted(pdns_info['results'], key=lambda k: k['lastSeen'], reverse=True)
|
||||
with riskiq_file.open('w') as _f:
|
||||
json.dump(pdns_info, _f)
|
||||
|
|
@ -659,7 +659,6 @@ def historical_lookups(tree_uuid: str) -> str | WerkzeugResponse | Response:
|
|||
auto_trigger = True if (request.args.get('auto_trigger') and request.args.get('auto_trigger') == 'True') else False
|
||||
data = lookyloo.get_historical_lookups(tree_uuid, force=force, auto_trigger=auto_trigger, as_admin=flask_login.current_user.is_authenticated)
|
||||
return render_template('historical_lookups.html', tree_uuid=tree_uuid,
|
||||
riskiq=data.get('riskiq'),
|
||||
circl_pdns=data.get('circl_pdns'))
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
{% from "macros.html" import shorten_string %}
|
||||
|
||||
<div>
|
||||
{% if not circl_pdns and not riskiq %}
|
||||
{% if not circl_pdns %}
|
||||
No historical data available
|
||||
{%else%}
|
||||
{% if circl_pdns %}
|
||||
|
|
@ -50,43 +50,5 @@
|
|||
{%endfor%}
|
||||
</center>
|
||||
{% endif%}
|
||||
{% if riskiq %}
|
||||
<hr>
|
||||
<script type="text/javascript" nonce="{{ csp_nonce() }}">
|
||||
new DataTable('#RiskIQ_pdns_table', {
|
||||
order: [[ 1, "desc" ]],
|
||||
autoWidth: false,
|
||||
columnDefs: [{ width: '15%', targets: 0 },
|
||||
{ width: '15%', targets: 1 },
|
||||
{ width: '10%', targets: 2 },
|
||||
{ width: '60%', targets: 3 }]
|
||||
});
|
||||
</script>
|
||||
<center>
|
||||
<h1 class="display-4">RiskIQ</h1>
|
||||
<div>
|
||||
<h3>{{riskiq['queryValue']}}</h3>
|
||||
<h4>{{riskiq['firstSeen']}} - {{ riskiq['lastSeen']}}</h4>
|
||||
<table id="RiskIQ_pdns_table" class="table table-striped" style="width:100%">
|
||||
<thead>
|
||||
<tr>
|
||||
<th class="col-sm-2" scope="col">First seen</th>
|
||||
<th class="col-sm-2" scope="col">Last seen</th>
|
||||
<th class="col-sm-1" scope="col">Type</th>
|
||||
<th class="col-sm-2" scope="col">Resolve</th>
|
||||
</thead>
|
||||
<tbody>
|
||||
{% for entry in riskiq['results'] %}
|
||||
<tr>
|
||||
<td>{{entry['firstSeen']}}</td>
|
||||
<td>{{entry['lastSeen']}}</td>
|
||||
<td>{{entry['recordType']}}</td>
|
||||
<td class="text-break">{{entry['resolve']}}</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</table>
|
||||
</div>
|
||||
</center>
|
||||
{% endif%}
|
||||
{% endif%}
|
||||
</div>
|
||||
|
|
|
|||
Loading…
Reference in New Issue