CIRCL
/
lookyloo
mirror of https://github.com/CIRCL/lookyloo
2
0
Fork 0
Code Issues Releases Wiki Activity

new: CIRCL Passive DNS module

pull/847/head
Raphaël Vinot 2023-12-12 16:19:01 +01:00
parent 1645a89354
commit 4668298943
8 changed files with 157 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
config/modules.json.sample
View File

@ -74,6 +74,11 @@
"allow_auto_trigger": false,
"default_first_seen_in_days": 5
},
"CIRCLPDNS": {
"user": null,
"password": null,
"allow_auto_trigger": false
},
"_notes": {
"apikey": "null disables the module. Pass a string otherwise.",
"autosubmit": "Automatically submits the URL to the 3rd party service.",

15
lookyloo/lookyloo.py
View File

@ -54,7 +54,7 @@ from .helpers import (get_captures_dir, get_email_template,
from .indexing import Indexing
from .modules import (MISPs, PhishingInitiative, UniversalWhois,
UrlScan, VirusTotal, Phishtank, Hashlookup,
RiskIQ, RiskIQError, Pandora, URLhaus)
RiskIQ, RiskIQError, Pandora, URLhaus, CIRCLPDNS)
if TYPE_CHECKING:
from playwright.async_api import Cookie
@ -120,6 +120,7 @@ class Lookyloo():
pass
# ## Done with MISP(s)
self.pi = PhishingInitiative(config_name='PhishingInitiative')
self.vt = VirusTotal(config_name='VirusTotal')
self.uwhois = UniversalWhois(config_name='UniversalWhois')
@ -129,6 +130,7 @@ class Lookyloo():
self.riskiq = RiskIQ(config_name='RiskIQ')
self.pandora = Pandora(config_name='Pandora')
self.urlhaus = URLhaus(config_name='URLhaus')
self.circl_pdns = CIRCLPDNS(config_name='CIRCLPDNS')
self.monitoring_enabled = False
if monitoring_config := get_config('generic', 'monitoring'):
@ -423,7 +425,7 @@ class Lookyloo():
if not cache:
self.logger.warning(f'Unable to get the modules responses unless the capture {capture_uuid} is cached')
return {}
to_return: Dict[str, Any] = {}
to_return: Dict[str, Any] = defaultdict(dict)
if self.riskiq.available:
try:
self.riskiq.capture_default_trigger(cache)
@ -435,6 +437,15 @@ class Lookyloo():
to_return['riskiq'] = self.riskiq.get_passivedns(hostname)
except RiskIQError as e:
self.logger.warning(e.response.content)
if self.circl_pdns.available:
self.circl_pdns.capture_default_trigger(cache)
if hasattr(cache, 'redirects') and cache.redirects:
hostname = urlparse(cache.redirects[-1]).hostname
else:
hostname = urlparse(cache.url).hostname
if hostname:
to_return['circl_pdns'][hostname] = self.circl_pdns.get_passivedns(hostname)
return to_return
def hide_capture(self, capture_uuid: str, /) -> None:

1
lookyloo/modules/__init__.py
View File

@ -13,3 +13,4 @@ from .hashlookup import HashlookupModule as Hashlookup # noqa
from .riskiq import RiskIQ, RiskIQError # noqa
from .urlhaus import URLhaus # noqa
from .cloudflare import Cloudflare # noqa
from .circlpdns import CIRCLPDNS # noqa

91
lookyloo/modules/circlpdns.py Normal file
View File

@ -0,0 +1,91 @@
#!/usr/bin/env python3
import json
from datetime import date
from typing import Dict, List, Optional, TYPE_CHECKING
from urllib.parse import urlparse
from pypdns import PyPDNS, PDNSRecord
from ..default import ConfigError, get_homedir
from ..helpers import get_cache_directory
if TYPE_CHECKING:
from ..capturecache import CaptureCache
from .abstractmodule import AbstractModule
class CIRCLPDNS(AbstractModule):
def module_init(self) -> bool:
if not (self.config.get('user') and self.config.get('password')):
self.logger.info('Missing credentials.')
return False
self.pypdns = PyPDNS(basic_auth=(self.config['user'], self.config['password']))
self.allow_auto_trigger = bool(self.config.get('allow_auto_trigger', False))
self.storage_dir_pypdns = get_homedir() / 'circl_pypdns'
self.storage_dir_pypdns.mkdir(parents=True, exist_ok=True)
return True
def get_passivedns(self, query: str) -> Optional[List[PDNSRecord]]:
# The query can be IP or Hostname. For now, we only do it on domains.
url_storage_dir = get_cache_directory(self.storage_dir_pypdns, query, 'pdns')
if not url_storage_dir.exists():
return None
cached_entries = sorted(url_storage_dir.glob('*'), reverse=True)
if not cached_entries:
return None
with cached_entries[0].open() as f:
return [PDNSRecord(record) for record in json.load(f)]
def capture_default_trigger(self, cache: 'CaptureCache', /, *, force: bool=False, auto_trigger: bool=False) -> Dict:
'''Run the module on all the nodes up to the final redirect'''
if not self.available:
return {'error': 'Module not available'}
if auto_trigger and not self.allow_auto_trigger:
return {'error': 'Auto trigger not allowed on module'}
if cache.url.startswith('file'):
return {'error': 'CIRCL Passive DNS does not support files.'}
if cache.redirects:
hostname = urlparse(cache.redirects[-1]).hostname
else:
hostname = urlparse(cache.url).hostname
if not hostname:
return {'error': 'No hostname found.'}
self.pdns_lookup(hostname, force)
return {'success': 'Module triggered'}
def pdns_lookup(self, hostname: str, force: bool=False) -> None:
'''Lookup an hostname on CIRCL Passive DNS
Note: force means re-fetch the entry even if we already did it today
'''
if not self.available:
raise ConfigError('CIRCL Passive DNS not available, probably no API key')
url_storage_dir = get_cache_directory(self.storage_dir_pypdns, hostname, 'pdns')
url_storage_dir.mkdir(parents=True, exist_ok=True)
pypdns_file = url_storage_dir / date.today().isoformat()
if not force and pypdns_file.exists():
return
pdns_info = [entry for entry in self.pypdns.iter_query(hostname)]
if not pdns_info:
try:
url_storage_dir.rmdir()
except OSError:
# Not empty.
pass
return
pdns_info_store = [entry.raw for entry in sorted(pdns_info, key=lambda k: k.time_last_datetime, reverse=True)]
with pypdns_file.open('w') as _f:
json.dump(pdns_info_store, _f)

22
poetry.lock generated
View File

@ -895,13 +895,13 @@ files = [
[[package]]
name = "fsspec"
version = "2023.12.1"
version = "2023.12.2"
description = "File-system specification"
optional = false
python-versions = ">=3.8"
files = [
{file = "fsspec-2023.12.1-py3-none-any.whl", hash = "sha256:6271f1d3075a378bfe432f6f42bf7e1d2a6ba74f78dd9b512385474c579146a0"},
{file = "fsspec-2023.12.1.tar.gz", hash = "sha256:c4da01a35ac65c853f833e43f67802c25213f560820d54ddf248f92eddd5e990"},
{file = "fsspec-2023.12.2-py3-none-any.whl", hash = "sha256:d800d87f72189a745fa3d6b033b9dc4a34ad069f60ca60b943a63599f5501960"},
{file = "fsspec-2023.12.2.tar.gz", hash = "sha256:8548d39e8810b59c38014934f6b31e57f40c1b20f911f4cc2b85389c7e9bf0cb"},
]
[package.extras]
@ -2083,13 +2083,13 @@ recaptcha = ["SpeechRecognition (>=3.10.1,<4.0.0)", "pydub (>=0.25.1,<0.26.0)",
[[package]]
name = "prompt-toolkit"
version = "3.0.41"
version = "3.0.42"
description = "Library for building powerful interactive command lines in Python"
optional = false
python-versions = ">=3.7.0"
files = [
{file = "prompt_toolkit-3.0.41-py3-none-any.whl", hash = "sha256:f36fe301fafb7470e86aaf90f036eef600a3210be4decf461a5b1ca8403d3cb2"},
{file = "prompt_toolkit-3.0.41.tar.gz", hash = "sha256:941367d97fc815548822aa26c2a269fdc4eb21e9ec05fc5d447cf09bad5d75f0"},
{file = "prompt_toolkit-3.0.42-py3-none-any.whl", hash = "sha256:3b50b5fc50660dc8e39dfe464b170959ad82ff185ffa53bfd3be02222e7156a1"},
{file = "prompt_toolkit-3.0.42.tar.gz", hash = "sha256:bfbf7d6ea9744e4ec94c9a69539e8106c77a2a607d728ded87c9182a4aec39be"},
]
[package.dependencies]
@ -2830,19 +2830,19 @@ files = [
[[package]]
name = "s3fs"
version = "2023.12.1"
version = "2023.12.2"
description = "Convenient Filesystem interface over S3"
optional = false
python-versions = ">= 3.8"
files = [
{file = "s3fs-2023.12.1-py3-none-any.whl", hash = "sha256:ed0b7df8cc20a2b5cefe607b1cf4e860d37c5ca4ac2d68f55464805d75d18710"},
{file = "s3fs-2023.12.1.tar.gz", hash = "sha256:63e429bb6b5e814568cacd3f2a8551fc35493e8c418ddfcb44e6f86aa8696ccd"},
{file = "s3fs-2023.12.2-py3-none-any.whl", hash = "sha256:0d5a99039665f30b2dbee5495de3b299a022d51b3195a9440f5df47c2621b777"},
{file = "s3fs-2023.12.2.tar.gz", hash = "sha256:b5ec07062481bbb45cb061b31984c7188d106e292c27033039e024e4ba5740dc"},
]
[package.dependencies]
aiobotocore = ">=2.5.4,<3.0.0"
aiohttp = "<4.0.0a0 || >4.0.0a0,<4.0.0a1 || >4.0.0a1"
fsspec = "2023.12.1"
fsspec = "2023.12.2"
[package.extras]
awscli = ["aiobotocore[awscli] (>=2.5.4,<3.0.0)"]
@ -3488,4 +3488,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p
[metadata]
lock-version = "2.0"
python-versions = ">=3.8.1,<3.12"
content-hash = "d0419f51f92d82e4099ad6ccf66f931eafc7499f4e30c9b085167d8ede8fb43c"
content-hash = "3f92ebac858e5a46242bea9b8b4539a913c5a397a8a54db1fce87c020a647c3c"

2
pyproject.toml
View File

@ -74,7 +74,7 @@ chardet = "^5.2.0"
pysecuritytxt = "^1.2.0"
pylookyloomonitoring = "^1.1.3"
pytz = {"version" = "^2023.3.post1", python = "<3.9"}
s3fs = "^2023.12.1"
s3fs = "^2023.12.2"
urllib3 = [
{version = "<2", python = "<3.10"},
{version = "^2.0.7", python = ">=3.10"}

3
website/web/__init__.py
View File

@ -306,7 +306,8 @@ def historical_lookups(tree_uuid: str):
force = True if (request.args.get('force') and request.args.get('force') == 'True') else False
data = lookyloo.get_historical_lookups(tree_uuid, force)
return render_template('historical_lookups.html', tree_uuid=tree_uuid,
riskiq=data.get('riskiq'))
riskiq=data.get('riskiq'),
circl_pdns=data.get('circl_pdns'))
@app.route('/tree/<string:tree_uuid>/categories_capture/', defaults={'query': ''})

33
website/web/templates/historical_lookups.html
View File

@ -26,6 +26,39 @@
</tr>
{% endfor %}
</table>
</div>
</center>
{% endif%}
{% if circl_pdns %}
<hr>
<center>
<h1 class="display-4">CIRCL Passve DNS</h1>
{% for query, responses in circl_pdns.items() %}
<div>
<h3>{{query}}</h3>
<table class="table">
<thead>
<tr>
<th class="col-sm-2" scope="col">First seen</th>
<th class="col-sm-2" scope="col">Last seen</th>
<th scope="col">R Data</th>
<th scope="col">RR Name</th>
<th class="col-sm-1" scope="col">RR Type</th>
</thead>
<tbody>
{%for response in responses %}
<tr>
<td>{{response.time_first_datetime}}</td>
<td>{{response.time_last_datetime}}</td>
<td>{{response.rdata}}</td>
<td>{{response.rrname}}</td>
<td>{{response.rrtype}}</td>
</tr>
{% endfor %}
</table>
</div>
{%endfor%}
</center>
{% endif%}
</div>