diff --git a/poetry.lock b/poetry.lock index bc523ed0..a15c97d1 100644 --- a/poetry.lock +++ b/poetry.lock @@ -120,7 +120,7 @@ lxml = ["lxml"] [[package]] name = "bootstrap-flask" -version = "2.1.0" +version = "2.2.0" description = "Bootstrap 4 & 5 helper for your Flask projects." category = "main" optional = false @@ -673,7 +673,7 @@ python-versions = ">=3.8" [[package]] name = "numpy" -version = "1.23.4" +version = "1.23.5" description = "NumPy is the fundamental package for array computing with Python." category = "main" optional = false @@ -1141,7 +1141,7 @@ jupyter = ["ipywidgets (>=7.5.1,<8.0.0)"] [[package]] name = "setuptools" -version = "65.5.1" +version = "65.6.0" description = "Easily download, build, install, upgrade, and uninstall Python packages" category = "main" optional = false @@ -1435,7 +1435,7 @@ misp = ["python-magic", "pydeep2"] [metadata] lock-version = "1.1" python-versions = ">=3.8,<3.12" -content-hash = "6faef544e363edbfb356b99c5ec94eb36b17c7dbe09cd818528221b2cec4097d" +content-hash = "bd26768e9bf334e9fa647c4a0a541498b1533a30f79c810c74687a210dc37761" [metadata.files] aiohttp = [ @@ -1578,8 +1578,8 @@ beautifulsoup4 = [ {file = "beautifulsoup4-4.11.1.tar.gz", hash = "sha256:ad9aa55b65ef2808eb405f46cf74df7fcb7044d5cbc26487f96eb2ef2e436693"}, ] bootstrap-flask = [ - {file = "Bootstrap-Flask-2.1.0.tar.gz", hash = "sha256:dc4f9c463727f3a59a6bfb17b7f9d13fd07646ba852f94285542c6a1e4e457e3"}, - {file = "Bootstrap_Flask-2.1.0-py2.py3-none-any.whl", hash = "sha256:52e360421aafbf117f59ed8237391b2a4c77592e35ab7b4566d4fde3b277825a"}, + {file = "Bootstrap-Flask-2.2.0.tar.gz", hash = "sha256:40f3bf7d5802e429b2fca286a548bdde816c1185c28aa8df3051cff35cd05cac"}, + {file = "Bootstrap_Flask-2.2.0-py2.py3-none-any.whl", hash = "sha256:b1fd9be1d22f4f1714a8a596ea2a4a5db9c957ba6db4aea4cb38ae90555233f3"}, ] cchardet = [ {file = "cchardet-2.1.7-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:c6f70139aaf47ffb94d89db603af849b82efdf756f187cdd3e566e30976c519f"}, @@ -2188,13 +2188,34 @@ numpy = [ {file = "numpy-1.23.3-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:91b8d6768a75247026e951dce3b2aac79dc7e78622fc148329135ba189813584"}, {file = "numpy-1.23.3-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:94c15ca4e52671a59219146ff584488907b1f9b3fc232622b47e2cf832e94fb8"}, {file = "numpy-1.23.3.tar.gz", hash = "sha256:51bf49c0cd1d52be0a240aa66f3458afc4b95d8993d2d04f0d91fa60c10af6cd"}, - {file = "numpy-1.23.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:95d79ada05005f6f4f337d3bb9de8a7774f259341c70bc88047a1f7b96a4bcb2"}, - {file = "numpy-1.23.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:926db372bc4ac1edf81cfb6c59e2a881606b409ddc0d0920b988174b2e2a767f"}, - {file = "numpy-1.23.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c237129f0e732885c9a6076a537e974160482eab8f10db6292e92154d4c67d71"}, - {file = "numpy-1.23.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a8365b942f9c1a7d0f0dc974747d99dd0a0cdfc5949a33119caf05cb314682d3"}, - {file = "numpy-1.23.4-cp310-cp310-win32.whl", hash = "sha256:2341f4ab6dba0834b685cce16dad5f9b6606ea8a00e6da154f5dbded70fdc4dd"}, - {file = "numpy-1.23.4-cp310-cp310-win_amd64.whl", hash = "sha256:d331afac87c92373826af83d2b2b435f57b17a5c74e6268b79355b970626e329"}, - {file = "numpy-1.23.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:488a66cb667359534bc70028d653ba1cf307bae88eab5929cd707c761ff037db"}, + {file = "numpy-1.23.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9c88793f78fca17da0145455f0d7826bcb9f37da4764af27ac945488116efe63"}, + {file = "numpy-1.23.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e9f4c4e51567b616be64e05d517c79a8a22f3606499941d97bb76f2ca59f982d"}, + {file = "numpy-1.23.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7903ba8ab592b82014713c491f6c5d3a1cde5b4a3bf116404e08f5b52f6daf43"}, + {file = "numpy-1.23.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5e05b1c973a9f858c74367553e236f287e749465f773328c8ef31abe18f691e1"}, + {file = "numpy-1.23.5-cp310-cp310-win32.whl", hash = "sha256:522e26bbf6377e4d76403826ed689c295b0b238f46c28a7251ab94716da0b280"}, + {file = "numpy-1.23.5-cp310-cp310-win_amd64.whl", hash = "sha256:dbee87b469018961d1ad79b1a5d50c0ae850000b639bcb1b694e9981083243b6"}, + {file = "numpy-1.23.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ce571367b6dfe60af04e04a1834ca2dc5f46004ac1cc756fb95319f64c095a96"}, + {file = "numpy-1.23.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:56e454c7833e94ec9769fa0f86e6ff8e42ee38ce0ce1fa4cbb747ea7e06d56aa"}, + {file = "numpy-1.23.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5039f55555e1eab31124a5768898c9e22c25a65c1e0037f4d7c495a45778c9f2"}, + {file = "numpy-1.23.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:58f545efd1108e647604a1b5aa809591ccd2540f468a880bedb97247e72db387"}, + {file = "numpy-1.23.5-cp311-cp311-win32.whl", hash = "sha256:b2a9ab7c279c91974f756c84c365a669a887efa287365a8e2c418f8b3ba73fb0"}, + {file = "numpy-1.23.5-cp311-cp311-win_amd64.whl", hash = "sha256:0cbe9848fad08baf71de1a39e12d1b6310f1d5b2d0ea4de051058e6e1076852d"}, + {file = "numpy-1.23.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:f063b69b090c9d918f9df0a12116029e274daf0181df392839661c4c7ec9018a"}, + {file = "numpy-1.23.5-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:0aaee12d8883552fadfc41e96b4c82ee7d794949e2a7c3b3a7201e968c7ecab9"}, + {file = "numpy-1.23.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:92c8c1e89a1f5028a4c6d9e3ccbe311b6ba53694811269b992c0b224269e2398"}, + {file = "numpy-1.23.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d208a0f8729f3fb790ed18a003f3a57895b989b40ea4dce4717e9cf4af62c6bb"}, + {file = "numpy-1.23.5-cp38-cp38-win32.whl", hash = "sha256:06005a2ef6014e9956c09ba07654f9837d9e26696a0470e42beedadb78c11b07"}, + {file = "numpy-1.23.5-cp38-cp38-win_amd64.whl", hash = "sha256:ca51fcfcc5f9354c45f400059e88bc09215fb71a48d3768fb80e357f3b457e1e"}, + {file = "numpy-1.23.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8969bfd28e85c81f3f94eb4a66bc2cf1dbdc5c18efc320af34bffc54d6b1e38f"}, + {file = "numpy-1.23.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a7ac231a08bb37f852849bbb387a20a57574a97cfc7b6cabb488a4fc8be176de"}, + {file = "numpy-1.23.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bf837dc63ba5c06dc8797c398db1e223a466c7ece27a1f7b5232ba3466aafe3d"}, + {file = "numpy-1.23.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:33161613d2269025873025b33e879825ec7b1d831317e68f4f2f0f84ed14c719"}, + {file = "numpy-1.23.5-cp39-cp39-win32.whl", hash = "sha256:af1da88f6bc3d2338ebbf0e22fe487821ea4d8e89053e25fa59d1d79786e7481"}, + {file = "numpy-1.23.5-cp39-cp39-win_amd64.whl", hash = "sha256:09b7847f7e83ca37c6e627682f145856de331049013853f344f37b0c9690e3df"}, + {file = "numpy-1.23.5-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:abdde9f795cf292fb9651ed48185503a2ff29be87770c3b8e2a14b0cd7aa16f8"}, + {file = "numpy-1.23.5-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f9a909a8bae284d46bbfdefbdd4a262ba19d3bc9921b1e76126b1d21c3c34135"}, + {file = "numpy-1.23.5-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:01dd17cbb340bf0fc23981e52e1d18a9d4050792e8fb8363cecbf066a84b827d"}, + {file = "numpy-1.23.5.tar.gz", hash = "sha256:1b1766d6f397c18153d40015ddfc79ddb715cabadc04d2d228d4e5a8bc4ded1a"}, ] packaging = [ {file = "packaging-21.3-py3-none-any.whl", hash = "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522"}, @@ -2512,8 +2533,8 @@ rich = [ {file = "rich-12.6.0.tar.gz", hash = "sha256:ba3a3775974105c221d31141f2c116f4fd65c5ceb0698657a11e9f295ec93fd0"}, ] setuptools = [ - {file = "setuptools-65.5.1-py3-none-any.whl", hash = "sha256:d0b9a8433464d5800cbe05094acf5c6d52a91bfac9b52bcfc4d41382be5d5d31"}, - {file = "setuptools-65.5.1.tar.gz", hash = "sha256:e197a19aa8ec9722928f2206f8de752def0e4c9fc6953527360d1c36d94ddb2f"}, + {file = "setuptools-65.6.0-py3-none-any.whl", hash = "sha256:6211d2f5eddad8757bd0484923ca7c0a6302ebc4ab32ea5e94357176e0ca0840"}, + {file = "setuptools-65.6.0.tar.gz", hash = "sha256:d1eebf881c6114e51df1664bc2c9133d022f78d12d5f4f665b9191f084e2862d"}, ] six = [ {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, diff --git a/pyproject.toml b/pyproject.toml index 1da3382f..ebbe3264 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,7 +42,7 @@ gunicorn = "^20.1.0" cchardet = "^2.1.7" redis = {version = "^4.3.4", extras = ["hiredis"]} beautifulsoup4 = "^4.11.1" -bootstrap-flask = "^2.1.0" +bootstrap-flask = "^2.2.0" defang = "^0.5.3" vt-py = "^0.17.3" pyeupi = "^1.1" diff --git a/website/web/__init__.py b/website/web/__init__.py index 3a46b6b7..47059fd5 100644 --- a/website/web/__init__.py +++ b/website/web/__init__.py @@ -15,6 +15,7 @@ from io import BytesIO, StringIO from typing import Any, Dict, List, Optional, Union, TypedDict from urllib.parse import quote_plus, unquote_plus, urlparse from uuid import uuid4 +from zipfile import ZipFile import flask_login # type: ignore from flask import (Flask, Response, flash, jsonify, redirect, render_template, @@ -871,14 +872,57 @@ def recapture(tree_uuid: str): def submit_capture(): if request.method == 'POST': - if 'har_file' not in request.files: - flash('Invalid submission: please submit at least an HAR file.', 'error') - else: - uuid = str(uuid4()) + listing = True if request.form.get('listing') else False + uuid = str(uuid4()) # NOTE: new UUID, because we do not want duplicates + har: Optional[Dict[str, Any]] = None + html: Optional[str] = None + last_redirected_url: Optional[str] = None + screenshot: Optional[bytes] = None + if 'har_file' in request.files and request.files['har_file']: har = json.loads(request.files['har_file'].stream.read()) - listing = True if request.form.get('listing') else False - lookyloo.store_capture(uuid, is_public=listing, har=har) + last_redirected_url = request.form.get('landing_page') + if 'screenshot_file' in request.files: + screenshot = request.files['screenshot_file'].stream.read() + if 'html_file' in request.files: + html = request.files['html_file'].stream.read().decode() + lookyloo.store_capture(uuid, is_public=listing, har=har, + last_redirected_url=last_redirected_url, + png=screenshot, html=html) return redirect(url_for('tree', tree_uuid=uuid)) + elif 'full_capture' in request.files and request.files['full_capture']: + # it *only* accepts a lookyloo export. + cookies: Optional[List[Dict[str, str]]] = None + has_error = False + with ZipFile(BytesIO(request.files['full_capture'].stream.read()), 'r') as lookyloo_capture: + for filename in lookyloo_capture.namelist(): + if filename.endswith('0.har'): + har = json.loads(lookyloo_capture.read(filename)) + elif filename.endswith('0.html'): + html = lookyloo_capture.read(filename).decode() + elif filename.endswith('0.last_redirect.txt'): + last_redirected_url = lookyloo_capture.read(filename).decode() + elif filename.endswith('0.png'): + screenshot = lookyloo_capture.read(filename) + elif filename.endswith('0.cookies.json'): + # Not required + cookies = json.loads(lookyloo_capture.read(filename)) + if not har or not html or not last_redirected_url or not screenshot: + has_error = True + if not har: + flash('Invalid submission: missing HAR file', 'error') + if not html: + flash('Invalid submission: missing HTML file', 'error') + if not last_redirected_url: + flash('Invalid submission: missing landing page', 'error') + if not screenshot: + flash('Invalid submission: missing screenshot', 'error') + if not has_error: + lookyloo.store_capture(uuid, is_public=listing, har=har, + last_redirected_url=last_redirected_url, + png=screenshot, html=html, cookies=cookies) + return redirect(url_for('tree', tree_uuid=uuid)) + else: + flash('Invalid submission: please submit at least an HAR file.', 'error') return render_template('submit_capture.html', default_public=get_config('generic', 'default_public'), diff --git a/website/web/templates/capture.html b/website/web/templates/capture.html index 7ffe5cba..eae9f72c 100644 --- a/website/web/templates/capture.html +++ b/website/web/templates/capture.html @@ -78,7 +78,7 @@ -
+