diff --git a/lookyloo/lookyloo.py b/lookyloo/lookyloo.py index 728c02d..298b2ed 100644 --- a/lookyloo/lookyloo.py +++ b/lookyloo/lookyloo.py @@ -499,6 +499,10 @@ class Lookyloo(): '''Get rendered HTML''' return self._get_raw(capture_uuid, 'html', all_html) + def get_data(self, capture_uuid: str, /) -> Tuple[str, BytesIO]: + '''Get the data''' + return self._get_raw(capture_uuid, 'data.filename', False).getvalue().decode(), self._get_raw(capture_uuid, 'data', False) + def get_cookies(self, capture_uuid: str, /, all_cookies: bool=False) -> BytesIO: '''Get the cookie(s)''' return self._get_raw(capture_uuid, 'cookies.json', all_cookies) @@ -911,6 +915,16 @@ class Lookyloo(): 'url_object': url, } + if url.empty_response: + if ct.root_hartree.rendered_node == url: + # check if a file is available + filename, data = self.get_data(capture_uuid) + if filename: + # we have a file to download + url.add_feature('has_dl_file', True) + url.add_feature('downloaded_filename', filename) + url.add_feature('downloaded_filesize', data.getbuffer().nbytes) + if not url.empty_response: # Index lookup # %%% Full body %%% diff --git a/poetry.lock b/poetry.lock index 8dc3489..2ef8a5d 100644 --- a/poetry.lock +++ b/poetry.lock @@ -270,7 +270,7 @@ python-versions = "*" [[package]] name = "executing" -version = "0.9.1" +version = "0.10.0" description = "Get the currently executing AST node of a frame, and other information" category = "dev" optional = false @@ -637,7 +637,7 @@ python-versions = "*" [[package]] name = "numpy" -version = "1.23.1" +version = "1.23.2" description = "NumPy is the fundamental package for array computing with Python." category = "main" optional = false @@ -977,7 +977,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [[package]] name = "pytz" -version = "2022.1" +version = "2022.2.1" description = "World timezone definitions, modern and historical" category = "main" optional = false @@ -1085,7 +1085,7 @@ python-versions = ">=3.6" [[package]] name = "stack-data" -version = "0.3.0" +version = "0.4.0" description = "Extract data from python stack frames and tracebacks for informative displays" category = "dev" optional = false @@ -1097,7 +1097,7 @@ executing = "*" pure-eval = "*" [package.extras] -tests = ["pytest", "typeguard", "pygments", "littleutils", "cython"] +tests = ["cython", "littleutils", "pygments", "typeguard", "pytest"] [[package]] name = "tldextract" @@ -1265,7 +1265,7 @@ python-versions = ">=3.7" [[package]] name = "tzdata" -version = "2022.1" +version = "2022.2" description = "Provider of IANA time zone data" category = "main" optional = false @@ -1410,7 +1410,7 @@ misp = ["python-magic", "pydeep2"] [metadata] lock-version = "1.1" python-versions = ">=3.8,<3.11" -content-hash = "cc137ed5eb94f794573512b67b705800ce699132bd8b52d5e7801e8ec20364c5" +content-hash = "9d723e8a69ebeac1f5c2da62b90e2abc5fd6f675ff200f69cef1638ed20effee" [metadata.files] aiohttp = [ @@ -1619,8 +1619,8 @@ ete3 = [ {file = "ete3-3.1.2.tar.gz", hash = "sha256:4fc987b8c529889d6608fab1101f1455cb5cbd42722788de6aea9c7d0a8e59e9"}, ] executing = [ - {file = "executing-0.9.1-py2.py3-none-any.whl", hash = "sha256:4ce4d6082d99361c0231fc31ac1a0f56979363cc6819de0b1410784f99e49105"}, - {file = "executing-0.9.1.tar.gz", hash = "sha256:ea278e2cf90cbbacd24f1080dd1f0ac25b71b2e21f50ab439b7ba45dd3195587"}, + {file = "executing-0.10.0-py2.py3-none-any.whl", hash = "sha256:9c745f80cda11eb22b62cbecf21156491a794eb56ab06f9d286a44e62822b24e"}, + {file = "executing-0.10.0.tar.gz", hash = "sha256:d1cd87c2e371e9966261410c5b3769d6df2f9e4a79a83eebd2662dd3388f9833"}, ] filelock = [ {file = "filelock-3.8.0-py3-none-any.whl", hash = "sha256:617eb4e5eedc82fc5f47b6d61e4d11cb837c56cb4544e39081099fa17ad109d4"}, @@ -2085,7 +2085,36 @@ mypy-extensions = [ {file = "mypy_extensions-0.4.3-py2.py3-none-any.whl", hash = "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d"}, {file = "mypy_extensions-0.4.3.tar.gz", hash = "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"}, ] -numpy = [] +numpy = [ + {file = "numpy-1.23.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e603ca1fb47b913942f3e660a15e55a9ebca906857edfea476ae5f0fe9b457d5"}, + {file = "numpy-1.23.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:633679a472934b1c20a12ed0c9a6c9eb167fbb4cb89031939bfd03dd9dbc62b8"}, + {file = "numpy-1.23.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:17e5226674f6ea79e14e3b91bfbc153fdf3ac13f5cc54ee7bc8fdbe820a32da0"}, + {file = "numpy-1.23.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bdc02c0235b261925102b1bd586579b7158e9d0d07ecb61148a1799214a4afd5"}, + {file = "numpy-1.23.2-cp310-cp310-win32.whl", hash = "sha256:df28dda02c9328e122661f399f7655cdcbcf22ea42daa3650a26bce08a187450"}, + {file = "numpy-1.23.2-cp310-cp310-win_amd64.whl", hash = "sha256:8ebf7e194b89bc66b78475bd3624d92980fca4e5bb86dda08d677d786fefc414"}, + {file = "numpy-1.23.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:dc76bca1ca98f4b122114435f83f1fcf3c0fe48e4e6f660e07996abf2f53903c"}, + {file = "numpy-1.23.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ecfdd68d334a6b97472ed032b5b37a30d8217c097acfff15e8452c710e775524"}, + {file = "numpy-1.23.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5593f67e66dea4e237f5af998d31a43e447786b2154ba1ad833676c788f37cde"}, + {file = "numpy-1.23.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac987b35df8c2a2eab495ee206658117e9ce867acf3ccb376a19e83070e69418"}, + {file = "numpy-1.23.2-cp311-cp311-win32.whl", hash = "sha256:d98addfd3c8728ee8b2c49126f3c44c703e2b005d4a95998e2167af176a9e722"}, + {file = "numpy-1.23.2-cp311-cp311-win_amd64.whl", hash = "sha256:8ecb818231afe5f0f568c81f12ce50f2b828ff2b27487520d85eb44c71313b9e"}, + {file = "numpy-1.23.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:909c56c4d4341ec8315291a105169d8aae732cfb4c250fbc375a1efb7a844f8f"}, + {file = "numpy-1.23.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:8247f01c4721479e482cc2f9f7d973f3f47810cbc8c65e38fd1bbd3141cc9842"}, + {file = "numpy-1.23.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b8b97a8a87cadcd3f94659b4ef6ec056261fa1e1c3317f4193ac231d4df70215"}, + {file = "numpy-1.23.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bd5b7ccae24e3d8501ee5563e82febc1771e73bd268eef82a1e8d2b4d556ae66"}, + {file = "numpy-1.23.2-cp38-cp38-win32.whl", hash = "sha256:9b83d48e464f393d46e8dd8171687394d39bc5abfe2978896b77dc2604e8635d"}, + {file = "numpy-1.23.2-cp38-cp38-win_amd64.whl", hash = "sha256:dec198619b7dbd6db58603cd256e092bcadef22a796f778bf87f8592b468441d"}, + {file = "numpy-1.23.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:4f41f5bf20d9a521f8cab3a34557cd77b6f205ab2116651f12959714494268b0"}, + {file = "numpy-1.23.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:806cc25d5c43e240db709875e947076b2826f47c2c340a5a2f36da5bb10c58d6"}, + {file = "numpy-1.23.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f9d84a24889ebb4c641a9b99e54adb8cab50972f0166a3abc14c3b93163f074"}, + {file = "numpy-1.23.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c403c81bb8ffb1c993d0165a11493fd4bf1353d258f6997b3ee288b0a48fce77"}, + {file = "numpy-1.23.2-cp39-cp39-win32.whl", hash = "sha256:cf8c6aed12a935abf2e290860af8e77b26a042eb7f2582ff83dc7ed5f963340c"}, + {file = "numpy-1.23.2-cp39-cp39-win_amd64.whl", hash = "sha256:5e28cd64624dc2354a349152599e55308eb6ca95a13ce6a7d5679ebff2962913"}, + {file = "numpy-1.23.2-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:806970e69106556d1dd200e26647e9bee5e2b3f1814f9da104a943e8d548ca38"}, + {file = "numpy-1.23.2-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2bd879d3ca4b6f39b7770829f73278b7c5e248c91d538aab1e506c628353e47f"}, + {file = "numpy-1.23.2-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:be6b350dfbc7f708d9d853663772a9310783ea58f6035eec649fb9c4371b5389"}, + {file = "numpy-1.23.2.tar.gz", hash = "sha256:b78d00e48261fbbd04aa0d7427cf78d18401ee0abd89c7559bbf422e5b1c7d01"}, +] packaging = [ {file = "packaging-21.3-py3-none-any.whl", hash = "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522"}, {file = "packaging-21.3.tar.gz", hash = "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb"}, @@ -2292,8 +2321,8 @@ python-magic = [ {file = "python_magic-0.4.27-py2.py3-none-any.whl", hash = "sha256:c212960ad306f700aa0d01e5d7a325d20548ff97eb9920dcd29513174f0294d3"}, ] pytz = [ - {file = "pytz-2022.1-py2.py3-none-any.whl", hash = "sha256:e68985985296d9a66a881eb3193b0906246245294a881e7c8afe623866ac6a5c"}, - {file = "pytz-2022.1.tar.gz", hash = "sha256:1e760e2fe6a8163bc0b3d9a19c4f84342afa0a2affebfaa84b01b978a02ecaa7"}, + {file = "pytz-2022.2.1-py2.py3-none-any.whl", hash = "sha256:220f481bdafa09c3955dfbdddb7b57780e9a94f5127e35456a48589b9e0c0197"}, + {file = "pytz-2022.2.1.tar.gz", hash = "sha256:cea221417204f2d1a2aa03ddae3e867921971d0d76f14d87abb4414415bbdcf5"}, ] pytz-deprecation-shim = [ {file = "pytz_deprecation_shim-0.1.0.post0-py2.py3-none-any.whl", hash = "sha256:8314c9692a636c8eb3bda879b9f119e350e93223ae83e70e80c31675a0fdc1a6"}, @@ -2400,8 +2429,8 @@ soupsieve = [ {file = "soupsieve-2.3.2.post1.tar.gz", hash = "sha256:fc53893b3da2c33de295667a0e19f078c14bf86544af307354de5fcf12a3f30d"}, ] stack-data = [ - {file = "stack_data-0.3.0-py3-none-any.whl", hash = "sha256:aa1d52d14d09c7a9a12bb740e6bdfffe0f5e8f4f9218d85e7c73a8c37f7ae38d"}, - {file = "stack_data-0.3.0.tar.gz", hash = "sha256:77bec1402dcd0987e9022326473fdbcc767304892a533ed8c29888dacb7dddbc"}, + {file = "stack_data-0.4.0-py3-none-any.whl", hash = "sha256:b94fed36d725cfabc6d09ed5886913e35eed9009766a1af1d5941b9da3a94aaa"}, + {file = "stack_data-0.4.0.tar.gz", hash = "sha256:a90ae7e260f7d15aefeceb46f0a028d4ccb9eb8856475c53e341945342d41ea7"}, ] tldextract = [ {file = "tldextract-3.3.1-py3-none-any.whl", hash = "sha256:35a0260570e214d8d3cfeeb403992fe9e2b686925f63c9b03c5933408ac2aa5a"}, @@ -2476,8 +2505,8 @@ typing-extensions = [ {file = "typing_extensions-4.3.0.tar.gz", hash = "sha256:e6d2677a32f47fc7eb2795db1dd15c1f34eff616bcaf2cfb5e997f854fa1c4a6"}, ] tzdata = [ - {file = "tzdata-2022.1-py2.py3-none-any.whl", hash = "sha256:238e70234214138ed7b4e8a0fab0e5e13872edab3be586ab8198c407620e2ab9"}, - {file = "tzdata-2022.1.tar.gz", hash = "sha256:8b536a8ec63dc0751342b3984193a3118f8fca2afe25752bb9b7fffd398552d3"}, + {file = "tzdata-2022.2-py2.py3-none-any.whl", hash = "sha256:c3119520447d68ef3eb8187a55a4f44fa455f30eb1b4238fa5691ba094f2b05b"}, + {file = "tzdata-2022.2.tar.gz", hash = "sha256:21f4f0d7241572efa7f7a4fdabb052e61b55dc48274e6842697ccdf5253e5451"}, ] tzlocal = [ {file = "tzlocal-4.2-py3-none-any.whl", hash = "sha256:89885494684c929d9191c57aa27502afc87a579be5cdd3225c77c463ea043745"}, diff --git a/pyproject.toml b/pyproject.toml index ea9b5ed..28a4e15 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,6 +66,7 @@ har2tree = "^1.14.1" playwrightcapture = "^1.14.0" passivetotal = "^2.5.9" werkzeug = "2.1.2" +filetype = "^1.1.0" [tool.poetry.extras] misp = ['python-magic', 'pydeep2'] diff --git a/website/web/__init__.py b/website/web/__init__.py index 2869054..9aaaa98 100644 --- a/website/web/__init__.py +++ b/website/web/__init__.py @@ -7,6 +7,8 @@ import logging import os import time +import filetype # type: ignore + from datetime import date, datetime, timedelta, timezone from io import BytesIO, StringIO from typing import Any, Dict, List, Optional, Union, TypedDict @@ -490,6 +492,21 @@ def image(tree_uuid: str): as_attachment=True, attachment_filename='image.png') +@app.route('/tree//data', methods=['GET']) +def data(tree_uuid: str): + filename, data = lookyloo.get_data(tree_uuid) + if len(filename) == 0: + # TODO: return something saying it is not a valid request + return + + if filetype.guess_mime(data.getvalue()) is None: + mime = 'application/octet-stream' + else: + mime = filetype.guess_mime(data.getvalue()) + return send_file(data, mimetype=mime, + as_attachment=True, attachment_filename=filename) + + @app.route('/tree//thumbnail/', defaults={'width': 64}, methods=['GET']) @app.route('/tree//thumbnail/', methods=['GET']) def thumbnail(tree_uuid: str, width: int): diff --git a/website/web/templates/hostname_popup.html b/website/web/templates/hostname_popup.html index 4e0efd2..5a14987 100644 --- a/website/web/templates/hostname_popup.html +++ b/website/web/templates/hostname_popup.html @@ -182,24 +182,29 @@

{{ popup_icons_response(url['url_object'], tree_uuid) }} - {% if url['url_object'].rendered_html %} - - Download rendered HTML page - - ({{ sizeof_fmt(url['url_object'].rendered_html.getbuffer().nbytes)}}) -
- - Download URLs in rendered HTML page - - {% endif %} - -
- {% if url['url_object'].empty_response %} - Empty body. + {% if url['url_object'].has_dl_file %} + + Download {{url['url_object'].downloaded_filename}} + ({{sizeof_fmt(url['url_object'].downloaded_filesize)}}) {% else %} - {{ ressource_legitimacy_details(url['legitimacy'], url['url_object'].body.getbuffer().nbytes) }} + {% if url['url_object'].rendered_html %} + + Download rendered HTML page + ({{ sizeof_fmt(url['url_object'].rendered_html.getbuffer().nbytes)}}) +
+ + Download URLs in rendered HTML page + + {% endif %} +
+ {% if url['url_object'].empty_response %} + Empty body. + {% else %} + {{ ressource_legitimacy_details(url['legitimacy'], url['url_object'].body.getbuffer().nbytes) }} + {% endif %} +
{%endif%} -
+ {% if url['known_content'] %} {{ known_content_details(url['known_content']) }}