CIRCL
/
lookyloo
mirror of https://github.com/CIRCL/lookyloo
2
0
Fork 0
Code Issues Releases Wiki Activity

new: Basic support for monitoring

monitoring
Raphaël Vinot 2023-02-23 18:37:40 +01:00
parent 370850285d
commit 5e72e1104e
7 changed files with 118 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
config/generic.json.sample
View File

@ -26,6 +26,10 @@
"enable": false,
"url": ""
},
"monitoring": {
"enable": false,
"url": "https://127.0.0.1:5200"
},
"tor_proxy": {
"server": "socks5://127.0.0.1:9050"
},
@ -74,6 +78,7 @@
"auto_trigger_modules": "Automatically trigger the modules when the tree is loaded and when the capture is cached",
"enable_mail_notification": "Allow users to notify a pre-configured email address about a specific capture",
"remote_lacus": "By default, lookyloo will do the capture locally. Enabling this feature means you have a dedicated Lacus instance somewhere",
"monitoring": "Enable connection to a remote monitoring instance",
"tor_proxy": "URL to connect to a SOCKS 5 proxy for tor",
"email": "Configuration for sending email notifications.",
"priority": "Define the priority of a new capture. A capture from the web interface has priority over a capture from the API, same for authenticated user vs. anonymous.",

12
lookyloo/lookyloo.py
View File

@ -34,6 +34,7 @@ from pylacus import (PyLacus,
# CaptureSettings as CaptureSettingsPy)
from pymisp import MISPAttribute, MISPEvent, MISPObject
from pysecuritytxt import PySecurityTXT, SecurityTXTNotAvailable
from pylookyloomonitoring import PyLookylooMonitoring
from redis import ConnectionPool, Redis
from redis.connection import UnixDomainSocketConnection
@ -44,7 +45,8 @@ from .exceptions import (MissingCaptureDirectory,
MissingUUID, TreeNeedsRebuild, NoValidHarFile)
from .helpers import (get_captures_dir, get_email_template,
get_resources_hashes, get_taxonomies,
uniq_domains, ParsedUserAgent, load_cookies, UserAgents)
uniq_domains, ParsedUserAgent, load_cookies, UserAgents,
get_useragent_for_requests)
from .indexing import Indexing
from .modules import (MISP, PhishingInitiative, UniversalWhois,
UrlScan, VirusTotal, Phishtank, Hashlookup,
@ -112,6 +114,14 @@ class Lookyloo():
if not self.urlhaus.available:
self.logger.warning('Unable to setup the URLhaus module')
self.monitoring_enabled = False
if monitoring_config := get_config('generic', 'monitoring'):
print(monitoring_config)
if monitoring_config['enable']:
self.monitoring = PyLookylooMonitoring(monitoring_config['url'], get_useragent_for_requests())
if self.monitoring.is_up:
self.monitoring_enabled = True
self.logger.info('Initializing context...')
self.context = Context()
self.logger.info('Context initialized.')

24
poetry.lock generated
View File

@ -2185,6 +2185,28 @@ requests = ">=2.28.2,<3.0.0"
[package.extras]
docs = ["Sphinx (>=6.1.3,<7.0.0)"]
[[package]]
name = "pylookyloomonitoring"
version = "0.1.0"
description = "Python API to connect to lookyloo monitoring"
category = "main"
optional = false
python-versions = "^3.8"
files = []
develop = false
[package.dependencies]
requests = "^2.28.2"
[package.extras]
docs = ["Sphinx (>=6.1.3,<7.0.0)"]
[package.source]
type = "git"
url = "https://github.com/Lookyloo/PyLookylooMonitoring.git"
reference = "HEAD"
resolved_reference = "99edb224dce7126c0ed8aabb4628e04d93e1580a"
[[package]]
name = "pymisp"
version = "2.4.168"
@ -3159,4 +3181,4 @@ testing = ["flake8 (<5)", "func-timeout", "jaraco.functools", "jaraco.itertools"
[metadata]
lock-version = "2.0"
python-versions = ">=3.8,<3.12"
content-hash = "cb5328d141e3bafc70de74d7a715b842271da5c19c0fb4e9996907f3e5751959"
content-hash = "37b91a1a0b9b1409a2206b80d029d5c5684850c56f839bb24b09d14b0f639c6f"

1
pyproject.toml
View File

@ -72,6 +72,7 @@ publicsuffixlist = "^0.9.3"
pyfaup = "^1.2"
chardet = "^5.1.0"
pysecuritytxt = "^1.0.1"
pylookyloomonitoring = {git = "https://github.com/Lookyloo/PyLookylooMonitoring.git"}
[tool.poetry.group.dev.dependencies]
mypy = "^1.0.1"

18
website/web/__init__.py
View File

@ -621,6 +621,23 @@ def cache_tree(tree_uuid: str):
return redirect(url_for('index'))
@app.route('/tree/<string:tree_uuid>/monitor', methods=['POST', 'GET'])
def monitor(tree_uuid: str):
if not lookyloo.monitoring_enabled:
return redirect(url_for('tree', tree_uuid=tree_uuid))
if request.form.get('name') or not request.form.get('confirm'):
# got a bot.
logging.info(f'{src_request_ip(request)} is a bot - {request.headers.get("User-Agent")}.')
return redirect('https://www.youtube.com/watch?v=iwGFalTRHDA')
collection: str = request.form['collection'] if request.form.get('collection') else ''
frequency: str = request.form['frequency'] if request.form.get('frequency') else 'daily'
cache = lookyloo.capture_cache(tree_uuid)
monitoring_uuid = lookyloo.monitoring.monitor({'url': cache.url}, frequency=frequency, collection=collection)
flash(f"Sent to monitoring: {monitoring_uuid}", 'success')
return redirect(url_for('tree', tree_uuid=tree_uuid))
@app.route('/tree/<string:tree_uuid>/send_mail', methods=['POST', 'GET'])
def send_mail(tree_uuid: str):
if not enable_mail_notification:
@ -688,6 +705,7 @@ def tree(tree_uuid: str, node_uuid: Optional[str]=None):
screenshot_thumbnail=b64_thumbnail, page_title=cache.title,
screenshot_size=screenshot_size,
meta=meta, enable_mail_notification=enable_mail_notification,
enable_monitoring=lookyloo.monitoring_enabled,
enable_context_by_users=enable_context_by_users,
enable_categorization=enable_categorization,
enable_bookmark=enable_bookmark,

11
website/web/genericapi.py
View File

@ -14,6 +14,7 @@ from lacuscore import CaptureStatus as CaptureStatusCore
from pylacus import CaptureStatus as CaptureStatusPy
from lookyloo.lookyloo import Lookyloo
from lookyloo.comparator import Comparator
from lookyloo.exceptions import MissingUUID
from .helpers import build_users_table, load_user_from_request, src_request_ip
@ -455,7 +456,15 @@ class CompareCaptures(Resource):
@api.doc(body=compare_captures_fields)
def post(self):
parameters: Dict = request.get_json(force=True)
result = comparator.compare_captures(parameters.get('capture_left'), parameters.get('capture_right'))
left_uuid = parameters.get('capture_left')
right_uuid = parameters.get('capture_right')
try:
result = comparator.compare_captures(left_uuid, right_uuid)
except MissingUUID as e:
# UUID non-existent, or capture still ongoing.
status_left = lookyloo.get_capture_status(left_uuid)
status_right = lookyloo.get_capture_status(right_uuid)
return {'error': e, 'details': {left_uuid: status_left, right_uuid: status_right}}
return result

50
website/web/templates/tree.html
View File

@ -191,6 +191,12 @@
<li class="list-group-item">
<a href="{{ url_for('index') }}" role="button">Homepage</a>
</li>
{% if enable_monitoring %}
<hr/>
<li class="list-group-item">
<a href="#monitoringModal" data-bs-toggle="modal" data-bs-target="#monitoringModal" role="button" class="btn btn-outline-info">Monitor</br>capture</a>
</li>
{% endif %}
{% if enable_mail_notification %}
<hr/>
<li class="list-group-item">
@ -681,6 +687,50 @@
</div>
{% endif %}
{% if enable_monitoring %}
<div class="modal fade" id="monitoringModal" tabindex="-1" role="dialog">
<div class="modal-dialog modal-xl" role="document">
<form role="form" action="{{ tree_uuid }}/monitor" method=post enctype=multipart/form-data>
<div class="modal-content">
<div class="modal-header">
<h5 class="modal-title" id="monitorModalLabel">Monitor capture</h5>
<button type="button" class="btn btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
</div>
<div class="modal-body">
<p>The capture will be submitted to the monitoring interface.</p>
<div class="mb-3">
<label for=frequency" class="form-label">Frequency</label>
<select name="frequency" id="frequency" class="form-select" aria-label="Select a frequency for the monitoring">
<option value="hourly" selected>Hourly</option>
<option value="daily">Daily</option>
</select>
<!-- boat fields -->
<label class="boatymcboat form-label" for="name">Your Name</label>
<input class="boatymcboat" autocomplete="off" type="text" id="name" name="name"
placeholder="Your fav boat name here">
</div>
<div class="mb-3">
<label for="collection" class="form-label">Collection</label>
<input type="text" class="form-control" name="collection" id="collection" placeholder="Name of the collection">
</div>
<div class="mb-3 form-check">
<input class="form-check-input" type="checkbox" name="confirm" onchange="document.getElementById('btn-notification').disabled = !this.checked;"></input>
<label for="force_push" class="form-check-label">{{ confirm_message }}</label>
</div>
</div>
<div class="modal-footer">
<button type="submit" class="btn btn-success" id="btn-notification" disabled=true>Send to monitoring</button>
<button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Close</button>
</div>
</div>
</form>
</div>
</div>
{% endif %}
<div class="modal fade" id="urlsInPageModal" tabindex="-1" role="dialog">
<div class="modal-dialog modal-xl" role="document">
{% if enable_mail_notification %}
<div class="modal fade" id="emailModal" tabindex="-1" role="dialog">
<div class="modal-dialog modal-xl" role="document">