diff --git a/config/generic.json.sample b/config/generic.json.sample index 05d0c20..47a3ff7 100644 --- a/config/generic.json.sample +++ b/config/generic.json.sample @@ -26,6 +26,10 @@ "enable": false, "url": "" }, + "monitoring": { + "enable": false, + "url": "https://127.0.0.1:5200" + }, "tor_proxy": { "server": "socks5://127.0.0.1:9050" }, @@ -74,6 +78,7 @@ "auto_trigger_modules": "Automatically trigger the modules when the tree is loaded and when the capture is cached", "enable_mail_notification": "Allow users to notify a pre-configured email address about a specific capture", "remote_lacus": "By default, lookyloo will do the capture locally. Enabling this feature means you have a dedicated Lacus instance somewhere", + "monitoring": "Enable connection to a remote monitoring instance", "tor_proxy": "URL to connect to a SOCKS 5 proxy for tor", "email": "Configuration for sending email notifications.", "priority": "Define the priority of a new capture. A capture from the web interface has priority over a capture from the API, same for authenticated user vs. anonymous.", diff --git a/lookyloo/lookyloo.py b/lookyloo/lookyloo.py index 577f8c9..4154c9e 100644 --- a/lookyloo/lookyloo.py +++ b/lookyloo/lookyloo.py @@ -34,6 +34,7 @@ from pylacus import (PyLacus, # CaptureSettings as CaptureSettingsPy) from pymisp import MISPAttribute, MISPEvent, MISPObject from pysecuritytxt import PySecurityTXT, SecurityTXTNotAvailable +from pylookyloomonitoring import PyLookylooMonitoring from redis import ConnectionPool, Redis from redis.connection import UnixDomainSocketConnection @@ -44,7 +45,8 @@ from .exceptions import (MissingCaptureDirectory, MissingUUID, TreeNeedsRebuild, NoValidHarFile) from .helpers import (get_captures_dir, get_email_template, get_resources_hashes, get_taxonomies, - uniq_domains, ParsedUserAgent, load_cookies, UserAgents) + uniq_domains, ParsedUserAgent, load_cookies, UserAgents, + get_useragent_for_requests) from .indexing import Indexing from .modules import (MISP, PhishingInitiative, UniversalWhois, UrlScan, VirusTotal, Phishtank, Hashlookup, @@ -112,6 +114,14 @@ class Lookyloo(): if not self.urlhaus.available: self.logger.warning('Unable to setup the URLhaus module') + self.monitoring_enabled = False + if monitoring_config := get_config('generic', 'monitoring'): + print(monitoring_config) + if monitoring_config['enable']: + self.monitoring = PyLookylooMonitoring(monitoring_config['url'], get_useragent_for_requests()) + if self.monitoring.is_up: + self.monitoring_enabled = True + self.logger.info('Initializing context...') self.context = Context() self.logger.info('Context initialized.') diff --git a/poetry.lock b/poetry.lock index 892a8c2..8a284fb 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2185,6 +2185,28 @@ requests = ">=2.28.2,<3.0.0" [package.extras] docs = ["Sphinx (>=6.1.3,<7.0.0)"] +[[package]] +name = "pylookyloomonitoring" +version = "0.1.0" +description = "Python API to connect to lookyloo monitoring" +category = "main" +optional = false +python-versions = "^3.8" +files = [] +develop = false + +[package.dependencies] +requests = "^2.28.2" + +[package.extras] +docs = ["Sphinx (>=6.1.3,<7.0.0)"] + +[package.source] +type = "git" +url = "https://github.com/Lookyloo/PyLookylooMonitoring.git" +reference = "HEAD" +resolved_reference = "99edb224dce7126c0ed8aabb4628e04d93e1580a" + [[package]] name = "pymisp" version = "2.4.168" @@ -3159,4 +3181,4 @@ testing = ["flake8 (<5)", "func-timeout", "jaraco.functools", "jaraco.itertools" [metadata] lock-version = "2.0" python-versions = ">=3.8,<3.12" -content-hash = "cb5328d141e3bafc70de74d7a715b842271da5c19c0fb4e9996907f3e5751959" +content-hash = "37b91a1a0b9b1409a2206b80d029d5c5684850c56f839bb24b09d14b0f639c6f" diff --git a/pyproject.toml b/pyproject.toml index 0325858..590b835 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -72,6 +72,7 @@ publicsuffixlist = "^0.9.3" pyfaup = "^1.2" chardet = "^5.1.0" pysecuritytxt = "^1.0.1" +pylookyloomonitoring = {git = "https://github.com/Lookyloo/PyLookylooMonitoring.git"} [tool.poetry.group.dev.dependencies] mypy = "^1.0.1" diff --git a/website/web/__init__.py b/website/web/__init__.py index dfdaaf2..e8e2431 100644 --- a/website/web/__init__.py +++ b/website/web/__init__.py @@ -621,6 +621,23 @@ def cache_tree(tree_uuid: str): return redirect(url_for('index')) +@app.route('/tree//monitor', methods=['POST', 'GET']) +def monitor(tree_uuid: str): + if not lookyloo.monitoring_enabled: + return redirect(url_for('tree', tree_uuid=tree_uuid)) + if request.form.get('name') or not request.form.get('confirm'): + # got a bot. + logging.info(f'{src_request_ip(request)} is a bot - {request.headers.get("User-Agent")}.') + return redirect('https://www.youtube.com/watch?v=iwGFalTRHDA') + + collection: str = request.form['collection'] if request.form.get('collection') else '' + frequency: str = request.form['frequency'] if request.form.get('frequency') else 'daily' + cache = lookyloo.capture_cache(tree_uuid) + monitoring_uuid = lookyloo.monitoring.monitor({'url': cache.url}, frequency=frequency, collection=collection) + flash(f"Sent to monitoring: {monitoring_uuid}", 'success') + return redirect(url_for('tree', tree_uuid=tree_uuid)) + + @app.route('/tree//send_mail', methods=['POST', 'GET']) def send_mail(tree_uuid: str): if not enable_mail_notification: @@ -688,6 +705,7 @@ def tree(tree_uuid: str, node_uuid: Optional[str]=None): screenshot_thumbnail=b64_thumbnail, page_title=cache.title, screenshot_size=screenshot_size, meta=meta, enable_mail_notification=enable_mail_notification, + enable_monitoring=lookyloo.monitoring_enabled, enable_context_by_users=enable_context_by_users, enable_categorization=enable_categorization, enable_bookmark=enable_bookmark, diff --git a/website/web/genericapi.py b/website/web/genericapi.py index 0dca70f..9da294b 100644 --- a/website/web/genericapi.py +++ b/website/web/genericapi.py @@ -14,6 +14,7 @@ from lacuscore import CaptureStatus as CaptureStatusCore from pylacus import CaptureStatus as CaptureStatusPy from lookyloo.lookyloo import Lookyloo from lookyloo.comparator import Comparator +from lookyloo.exceptions import MissingUUID from .helpers import build_users_table, load_user_from_request, src_request_ip @@ -455,7 +456,15 @@ class CompareCaptures(Resource): @api.doc(body=compare_captures_fields) def post(self): parameters: Dict = request.get_json(force=True) - result = comparator.compare_captures(parameters.get('capture_left'), parameters.get('capture_right')) + left_uuid = parameters.get('capture_left') + right_uuid = parameters.get('capture_right') + try: + result = comparator.compare_captures(left_uuid, right_uuid) + except MissingUUID as e: + # UUID non-existent, or capture still ongoing. + status_left = lookyloo.get_capture_status(left_uuid) + status_right = lookyloo.get_capture_status(right_uuid) + return {'error': e, 'details': {left_uuid: status_left, right_uuid: status_right}} return result diff --git a/website/web/templates/tree.html b/website/web/templates/tree.html index ae5767c..be23a8a 100644 --- a/website/web/templates/tree.html +++ b/website/web/templates/tree.html @@ -191,6 +191,12 @@
  • Homepage
    • + {% if enable_monitoring %} +
    +
  • + Monitor
    capture     +
    • + {% endif %} {% if enable_mail_notification %}
  • @@ -681,6 +687,50 @@ {% endif %} +{% if enable_monitoring %} + +{% endif %} + +