From cafa9c150229d2311045743f64cd49f0c5766a78 Mon Sep 17 00:00:00 2001 From: Adrian Maraj Date: Tue, 30 Apr 2024 11:54:24 +0200 Subject: [PATCH 01/11] new simple capture --- website/web/__init__.py | 24 ++++ website/web/templates/index.html | 5 + website/web/templates/simple_capture.html | 127 ++++++++++++++++++++++ 3 files changed, 156 insertions(+) create mode 100644 website/web/templates/simple_capture.html diff --git a/website/web/__init__.py b/website/web/__init__.py index 0bd8158..59d4a28 100644 --- a/website/web/__init__.py +++ b/website/web/__init__.py @@ -1632,6 +1632,30 @@ def capture_web() -> str | Response | WerkzeugResponse: # render template return _prepare_capture_template(user_ua=request.headers.get('User-Agent')) +@app.route('/simple_capture', methods=['GET', 'POST']) +def simple_capture() -> str | Response | WerkzeugResponse: + if flask_login.current_user.is_authenticated: + user = flask_login.current_user.get_id() + else: + user = src_request_ip(request) + + if request.method == 'POST': + if not (request.form.get('url') or request.form.get('urls')): + flash('Invalid submission: please submit at least a URL.', 'error') + return render_template('simple_capture.html') + capture_query: CaptureSettings = {} + capture_query['url'] = request.form['url'] + + perma_uuid = lookyloo.enqueue_capture(capture_query, source='web', user=user, + authenticated=flask_login.current_user.is_authenticated) + if perma_uuid: + flash('Recording is in progress and is reported automatically.', 'success') + time.sleep(2) + return redirect(url_for('simple_capture')) + + # render template + return render_template('simple_capture.html') + @app.route('/cookies/', methods=['GET']) def cookies_name_detail(cookie_name: str) -> str: diff --git a/website/web/templates/index.html b/website/web/templates/index.html index a9ee30f..9bf88f0 100644 --- a/website/web/templates/index.html +++ b/website/web/templates/index.html @@ -83,6 +83,11 @@ $(document).ready(function () { + {% if current_user.is_authenticated %} + + + + {% endif %}

{{ render_messages(container=True, dismissible=True) }} diff --git a/website/web/templates/simple_capture.html b/website/web/templates/simple_capture.html new file mode 100644 index 0000000..3ac7c60 --- /dev/null +++ b/website/web/templates/simple_capture.html @@ -0,0 +1,127 @@ +{% extends "main.html" %} +{% from 'bootstrap5/utils.html' import render_messages %} +{% block title %}Capture{% endblock %} + +{% block card %} + + + + + + +{% endblock %} + +{% block content %} +
+
+ + Lookyloo + +
+ {{ render_messages(container=True, dismissible=True) }} +
+
+
+
+ + +
+ + +
+
+
+ + + +
+
+ +
+
+ + + +
+ + {% if default_public %} + By default, the capture is private. If you do not want that, untick the box at the top of the form. + {% else %} + By default, the capture is private (not visible on the index page). If you want it to be public tick the box at the top of the form. + {% endif %} + +
+
+ +
+
+
+{% endblock %} + +{% block scripts %} + {{ super() }} + + + + + +{% endblock %} From ae0c9e3449b4fbc85cfdd30602409ae781315f23 Mon Sep 17 00:00:00 2001 From: Adrian Maraj Date: Tue, 30 Apr 2024 15:32:14 +0200 Subject: [PATCH 02/11] new: simplified version to automatically report url --- bin/async_capture.py | 2 +- config/users/adrian.json.sample | 3 ++ lookyloo/default/helpers.py | 4 +++ lookyloo/lookyloo.py | 4 +-- poetry.lock | 6 ++-- website/web/__init__.py | 39 ++++++++++++++++------- website/web/templates/simple_capture.html | 4 +-- 7 files changed, 42 insertions(+), 20 deletions(-) create mode 100644 config/users/adrian.json.sample diff --git a/bin/async_capture.py b/bin/async_capture.py index d471784..f7c9513 100755 --- a/bin/async_capture.py +++ b/bin/async_capture.py @@ -116,7 +116,7 @@ class AsyncCapture(AbstractManager): if send_report: self.lookyloo.send_mail(uuid, email=settings.get('email', ''), - comment=settings.get('comment')) + comment=settings.get('comment'), email_prio= settings["email"] if settings["email"] else None) lazy_cleanup = self.lookyloo.redis.pipeline() if queue and self.lookyloo.redis.zscore('queues', queue): diff --git a/config/users/adrian.json.sample b/config/users/adrian.json.sample new file mode 100644 index 0000000..1ea327d --- /dev/null +++ b/config/users/adrian.json.sample @@ -0,0 +1,3 @@ +{ + "email" : "analyst@test.de" +} \ No newline at end of file diff --git a/lookyloo/default/helpers.py b/lookyloo/default/helpers.py index 4631a8f..b28446d 100644 --- a/lookyloo/default/helpers.py +++ b/lookyloo/default/helpers.py @@ -57,6 +57,10 @@ def load_configs(path_to_config_files: str | Path | None=None) -> None: for path in config_path.glob('*.json'): with path.open() as _c: configs[path.stem] = json.load(_c) + user_path = config_path / 'users' + for path in user_path.glob('*.json'): + with path.open() as _c: + configs[path.stem] = json.load(_c) @lru_cache(64) diff --git a/lookyloo/lookyloo.py b/lookyloo/lookyloo.py index af66733..f46343b 100644 --- a/lookyloo/lookyloo.py +++ b/lookyloo/lookyloo.py @@ -864,7 +864,7 @@ class Lookyloo(): return f"Malicious capture according to {len(modules)} module(s): {', '.join(modules)}" - def send_mail(self, capture_uuid: str, /, email: str='', comment: str | None=None) -> bool | dict[str, Any]: + def send_mail(self, capture_uuid: str, /, email: str='', comment: str | None=None, email_prio: str | None = None) -> bool | dict[str, Any]: '''Send an email notification regarding a specific capture''' if not get_config('generic', 'enable_mail_notification'): return {"error": "Unable to send mail: mail notification disabled"} @@ -913,7 +913,7 @@ class Lookyloo(): msg['From'] = email_config['from'] if email: msg['Reply-To'] = email - msg['To'] = email_config['to'] + msg['To'] = email_config['to'] if not email_prio else email_prio msg['Subject'] = email_config['subject'] body = get_email_template() body = body.format( diff --git a/poetry.lock b/poetry.lock index c278b04..5b802bf 100644 --- a/poetry.lock +++ b/poetry.lock @@ -737,13 +737,13 @@ tests = ["asttokens (>=2.1.0)", "coverage", "coverage-enable-subprocess", "ipyth [[package]] name = "filelock" -version = "3.13.4" +version = "3.14.0" description = "A platform independent file lock." optional = false python-versions = ">=3.8" files = [ - {file = "filelock-3.13.4-py3-none-any.whl", hash = "sha256:404e5e9253aa60ad457cae1be07c0f0ca90a63931200a47d9b6a6af84fd7b45f"}, - {file = "filelock-3.13.4.tar.gz", hash = "sha256:d13f466618bfde72bd2c18255e269f72542c6e70e7bac83a0232d6b1cc5c8cf4"}, + {file = "filelock-3.14.0-py3-none-any.whl", hash = "sha256:43339835842f110ca7ae60f1e1c160714c5a6afd15a2873419ab185334975c0f"}, + {file = "filelock-3.14.0.tar.gz", hash = "sha256:6ea72da3be9b8c82afd3edcf99f2fffbb5076335a5ae4d03248bb5b6c3eae78a"}, ] [package.extras] diff --git a/website/web/__init__.py b/website/web/__init__.py index 59d4a28..da86226 100644 --- a/website/web/__init__.py +++ b/website/web/__init__.py @@ -41,7 +41,7 @@ from werkzeug.wrappers.response import Response as WerkzeugResponse from lookyloo import Lookyloo, CaptureSettings, Indexing from lookyloo.capturecache import CaptureCache -from lookyloo.default import get_config +from lookyloo.default import get_config, get_homedir from lookyloo.exceptions import MissingUUID, NoValidHarFile from lookyloo.helpers import get_taxonomies, UserAgents, load_cookies @@ -1632,7 +1632,7 @@ def capture_web() -> str | Response | WerkzeugResponse: # render template return _prepare_capture_template(user_ua=request.headers.get('User-Agent')) -@app.route('/simple_capture', methods=['GET', 'POST']) +@app.route('/simple_capture', methods=['GET','POST']) def simple_capture() -> str | Response | WerkzeugResponse: if flask_login.current_user.is_authenticated: user = flask_login.current_user.get_id() @@ -1644,15 +1644,32 @@ def simple_capture() -> str | Response | WerkzeugResponse: flash('Invalid submission: please submit at least a URL.', 'error') return render_template('simple_capture.html') capture_query: CaptureSettings = {} - capture_query['url'] = request.form['url'] - - perma_uuid = lookyloo.enqueue_capture(capture_query, source='web', user=user, - authenticated=flask_login.current_user.is_authenticated) - if perma_uuid: - flash('Recording is in progress and is reported automatically.', 'success') - time.sleep(2) - return redirect(url_for('simple_capture')) - + if request.form.get('auto_report'): + path = get_homedir() /'config'/ 'users' / (user + ".json") + if os.path.isfile(path): + email = get_config(user, 'email') + capture_query['auto_report'] = {"email": email} + else: + capture_query['auto_report'] = True + if request.form.get('url'): + capture_query['url'] = request.form['url'] + perma_uuid = lookyloo.enqueue_capture(capture_query, source='web', user=user, + authenticated=flask_login.current_user.is_authenticated) + time.sleep(2) + if perma_uuid: + flash('Recording is in progress and is reported automatically.', 'success') + return redirect(url_for('simple_capture')) + elif request.form.get('urls'): + for url in request.form['urls'].strip().split('\n'): + if not url: + continue + query = capture_query.copy() + query['url'] = url + new_capture_uuid = lookyloo.enqueue_capture(query, source='web', user=user, + authenticated=flask_login.current_user.is_authenticated) + if new_capture_uuid: + flash('Recording is in progress and is reported automatically.', 'success') + return redirect(url_for('simple_capture')) # render template return render_template('simple_capture.html') diff --git a/website/web/templates/simple_capture.html b/website/web/templates/simple_capture.html index 3ac7c60..dbb6d09 100644 --- a/website/web/templates/simple_capture.html +++ b/website/web/templates/simple_capture.html @@ -122,6 +122,4 @@ } }) - - -{% endblock %} +{% endblock %} \ No newline at end of file From f856a72cf5c988bf3524f9c078e560fd31d40a33 Mon Sep 17 00:00:00 2001 From: Adrian Maraj Date: Tue, 30 Apr 2024 15:44:19 +0200 Subject: [PATCH 03/11] chg:capture are private --- website/web/__init__.py | 1 + website/web/templates/simple_capture.html | 3 --- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/website/web/__init__.py b/website/web/__init__.py index da86226..35a46b8 100644 --- a/website/web/__init__.py +++ b/website/web/__init__.py @@ -1644,6 +1644,7 @@ def simple_capture() -> str | Response | WerkzeugResponse: flash('Invalid submission: please submit at least a URL.', 'error') return render_template('simple_capture.html') capture_query: CaptureSettings = {} + capture_query['listing'] = False if request.form.get('auto_report'): path = get_homedir() /'config'/ 'users' / (user + ".json") if os.path.isfile(path): diff --git a/website/web/templates/simple_capture.html b/website/web/templates/simple_capture.html index dbb6d09..e8810ca 100644 --- a/website/web/templates/simple_capture.html +++ b/website/web/templates/simple_capture.html @@ -33,9 +33,6 @@
- - -
From e824fea97dd81a75936eb5fd2e38d1c4168437d9 Mon Sep 17 00:00:00 2001 From: Adrian Maraj Date: Tue, 30 Apr 2024 15:49:30 +0200 Subject: [PATCH 04/11] chg: sample file name --- config/users/{adrian.json.sample => user.json.sample} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename config/users/{adrian.json.sample => user.json.sample} (100%) diff --git a/config/users/adrian.json.sample b/config/users/user.json.sample similarity index 100% rename from config/users/adrian.json.sample rename to config/users/user.json.sample From 427c6ed46551cd99dd238f372518760711cab492 Mon Sep 17 00:00:00 2001 From: Adrian Maraj Date: Tue, 30 Apr 2024 16:50:48 +0200 Subject: [PATCH 05/11] chg: fixes --- bin/async_capture.py | 2 +- lookyloo/lookyloo.py | 4 ++-- website/web/__init__.py | 7 +++++-- website/web/templates/simple_capture.html | 12 +----------- 4 files changed, 9 insertions(+), 16 deletions(-) diff --git a/bin/async_capture.py b/bin/async_capture.py index f7c9513..e7e8b5c 100755 --- a/bin/async_capture.py +++ b/bin/async_capture.py @@ -116,7 +116,7 @@ class AsyncCapture(AbstractManager): if send_report: self.lookyloo.send_mail(uuid, email=settings.get('email', ''), - comment=settings.get('comment'), email_prio= settings["email"] if settings["email"] else None) + comment=settings.get('comment'), recipient_mail= settings.get("recipient_mail")) lazy_cleanup = self.lookyloo.redis.pipeline() if queue and self.lookyloo.redis.zscore('queues', queue): diff --git a/lookyloo/lookyloo.py b/lookyloo/lookyloo.py index f46343b..aa93e43 100644 --- a/lookyloo/lookyloo.py +++ b/lookyloo/lookyloo.py @@ -864,7 +864,7 @@ class Lookyloo(): return f"Malicious capture according to {len(modules)} module(s): {', '.join(modules)}" - def send_mail(self, capture_uuid: str, /, email: str='', comment: str | None=None, email_prio: str | None = None) -> bool | dict[str, Any]: + def send_mail(self, capture_uuid: str, /, email: str='', comment: str | None=None, recipient_mail: str | None = None) -> bool | dict[str, Any]: '''Send an email notification regarding a specific capture''' if not get_config('generic', 'enable_mail_notification'): return {"error": "Unable to send mail: mail notification disabled"} @@ -913,7 +913,7 @@ class Lookyloo(): msg['From'] = email_config['from'] if email: msg['Reply-To'] = email - msg['To'] = email_config['to'] if not email_prio else email_prio + msg['To'] = email_config['to'] if not recipient_mail else recipient_mail msg['Subject'] = email_config['subject'] body = get_email_template() body = body.format( diff --git a/website/web/__init__.py b/website/web/__init__.py index 35a46b8..d416e3d 100644 --- a/website/web/__init__.py +++ b/website/web/__init__.py @@ -12,6 +12,7 @@ import json import logging import logging.config import os +import re import sys import time @@ -1638,8 +1639,10 @@ def simple_capture() -> str | Response | WerkzeugResponse: user = flask_login.current_user.get_id() else: user = src_request_ip(request) - if request.method == 'POST': + if not re.match("^[A-Za-z]+$", user): + flash('User is not permitted.', 'error') + return redirect(url_for('simple_capture')) if not (request.form.get('url') or request.form.get('urls')): flash('Invalid submission: please submit at least a URL.', 'error') return render_template('simple_capture.html') @@ -1649,7 +1652,7 @@ def simple_capture() -> str | Response | WerkzeugResponse: path = get_homedir() /'config'/ 'users' / (user + ".json") if os.path.isfile(path): email = get_config(user, 'email') - capture_query['auto_report'] = {"email": email} + capture_query['auto_report'] = {"recipient_mail": email} else: capture_query['auto_report'] = True if request.form.get('url'): diff --git a/website/web/templates/simple_capture.html b/website/web/templates/simple_capture.html index e8810ca..ec54fd1 100644 --- a/website/web/templates/simple_capture.html +++ b/website/web/templates/simple_capture.html @@ -63,20 +63,10 @@
- -
- - {% if default_public %} - By default, the capture is private. If you do not want that, untick the box at the top of the form. - {% else %} - By default, the capture is private (not visible on the index page). If you want it to be public tick the box at the top of the form. - {% endif %} -
-
- +
From 37c78ef2460e76982d1b68f50b92e7e88fe30337 Mon Sep 17 00:00:00 2001 From: Adrian Maraj Date: Thu, 2 May 2024 10:26:56 +0200 Subject: [PATCH 06/11] chg: login required for simple_capture --- website/web/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/website/web/__init__.py b/website/web/__init__.py index d416e3d..e5c6298 100644 --- a/website/web/__init__.py +++ b/website/web/__init__.py @@ -1634,6 +1634,7 @@ def capture_web() -> str | Response | WerkzeugResponse: return _prepare_capture_template(user_ua=request.headers.get('User-Agent')) @app.route('/simple_capture', methods=['GET','POST']) +@flask_login.login_required def simple_capture() -> str | Response | WerkzeugResponse: if flask_login.current_user.is_authenticated: user = flask_login.current_user.get_id() From 00e331ec5a71522556d4fe3c93acba5ae87caf65 Mon Sep 17 00:00:00 2001 From: Adrian Maraj Date: Thu, 2 May 2024 10:29:46 +0200 Subject: [PATCH 07/11] make mypy happy --- website/web/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/web/__init__.py b/website/web/__init__.py index e5c6298..28eb6c9 100644 --- a/website/web/__init__.py +++ b/website/web/__init__.py @@ -1634,7 +1634,7 @@ def capture_web() -> str | Response | WerkzeugResponse: return _prepare_capture_template(user_ua=request.headers.get('User-Agent')) @app.route('/simple_capture', methods=['GET','POST']) -@flask_login.login_required +@flask_login.login_required # type: ignore[misc] def simple_capture() -> str | Response | WerkzeugResponse: if flask_login.current_user.is_authenticated: user = flask_login.current_user.get_id() From 5ab46abb21dd6e6a31dc0a3c0a89c1e43eb53d05 Mon Sep 17 00:00:00 2001 From: Antonia Koch <146366740+AntoniaBK@users.noreply.github.com> Date: Thu, 2 May 2024 12:08:10 +0200 Subject: [PATCH 08/11] Fix: validate username --- website/web/__init__.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/website/web/__init__.py b/website/web/__init__.py index 28eb6c9..89eea04 100644 --- a/website/web/__init__.py +++ b/website/web/__init__.py @@ -1636,14 +1636,13 @@ def capture_web() -> str | Response | WerkzeugResponse: @app.route('/simple_capture', methods=['GET','POST']) @flask_login.login_required # type: ignore[misc] def simple_capture() -> str | Response | WerkzeugResponse: - if flask_login.current_user.is_authenticated: - user = flask_login.current_user.get_id() - else: - user = src_request_ip(request) + user = flask_login.current_user.get_id() + if not re.match("^[A-Za-z0-9]+$", user): + # Username has been manipulated + flash('User is not permitted.', 'error') + return redirect(url_for('submit_capture')) + if request.method == 'POST': - if not re.match("^[A-Za-z]+$", user): - flash('User is not permitted.', 'error') - return redirect(url_for('simple_capture')) if not (request.form.get('url') or request.form.get('urls')): flash('Invalid submission: please submit at least a URL.', 'error') return render_template('simple_capture.html') From 10f0adb7a512d104241508691a3857a1ad2d8bcb Mon Sep 17 00:00:00 2001 From: Antonia Koch <146366740+AntoniaBK@users.noreply.github.com> Date: Thu, 2 May 2024 13:24:47 +0200 Subject: [PATCH 09/11] add: validate username at login --- website/web/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/website/web/__init__.py b/website/web/__init__.py index 89eea04..3168003 100644 --- a/website/web/__init__.py +++ b/website/web/__init__.py @@ -53,7 +53,7 @@ else: all_timezones_set = available_timezones() from .genericapi import api as generic_api -from .helpers import (User, build_users_table, get_secret_key, +from .helpers import (User, valid_username, build_users_table, get_secret_key, load_user_from_request, src_request_ip, sri_load, get_lookyloo_instance) from .proxied import ReverseProxied @@ -107,6 +107,9 @@ def login() -> WerkzeugResponse | str | Response: ''' username = request.form['username'] + if not valid_username(username): + flash('User is not permitted.', 'error') + return redirect(url_for('login')) users_table = build_users_table() if username in users_table and check_password_hash(users_table[username]['password'], request.form['password']): user = User() From db0a11185abca9be754c64c99381c01a1f4b22e7 Mon Sep 17 00:00:00 2001 From: Antonia Koch <146366740+AntoniaBK@users.noreply.github.com> Date: Thu, 2 May 2024 13:27:03 +0200 Subject: [PATCH 10/11] add: function that checks if username is valid --- website/web/helpers.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/website/web/helpers.py b/website/web/helpers.py index 4a107a8..3a49fd1 100644 --- a/website/web/helpers.py +++ b/website/web/helpers.py @@ -49,6 +49,8 @@ def load_user_from_request(request: Request) -> User | None: return user return None +def valid_username(username: str) -> bool: + return re.match("^[A-Za-z0-9]+$", username) @lru_cache(64) def build_keys_table() -> dict[str, str]: From 7b06ce399b59167f6344f811c417d4d5b488d39f Mon Sep 17 00:00:00 2001 From: Adrian Maraj Date: Thu, 2 May 2024 13:33:33 +0200 Subject: [PATCH 11/11] fix: function name --- website/web/__init__.py | 6 +++--- website/web/helpers.py | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/website/web/__init__.py b/website/web/__init__.py index 3168003..8df3169 100644 --- a/website/web/__init__.py +++ b/website/web/__init__.py @@ -53,7 +53,7 @@ else: all_timezones_set = available_timezones() from .genericapi import api as generic_api -from .helpers import (User, valid_username, build_users_table, get_secret_key, +from .helpers import (User, is_valid_username, build_users_table, get_secret_key, load_user_from_request, src_request_ip, sri_load, get_lookyloo_instance) from .proxied import ReverseProxied @@ -107,7 +107,7 @@ def login() -> WerkzeugResponse | str | Response: ''' username = request.form['username'] - if not valid_username(username): + if not is_valid_username(username): flash('User is not permitted.', 'error') return redirect(url_for('login')) users_table = build_users_table() @@ -1640,7 +1640,7 @@ def capture_web() -> str | Response | WerkzeugResponse: @flask_login.login_required # type: ignore[misc] def simple_capture() -> str | Response | WerkzeugResponse: user = flask_login.current_user.get_id() - if not re.match("^[A-Za-z0-9]+$", user): + if not is_valid_username(user): # Username has been manipulated flash('User is not permitted.', 'error') return redirect(url_for('submit_capture')) diff --git a/website/web/helpers.py b/website/web/helpers.py index 3a49fd1..cc8aacc 100644 --- a/website/web/helpers.py +++ b/website/web/helpers.py @@ -5,6 +5,7 @@ from __future__ import annotations import hashlib import json import os +import re from functools import lru_cache from pathlib import Path @@ -49,7 +50,7 @@ def load_user_from_request(request: Request) -> User | None: return user return None -def valid_username(username: str) -> bool: +def is_valid_username(username: str) -> bool: return re.match("^[A-Za-z0-9]+$", username) @lru_cache(64)