From ec07429e6562c62ec69f5503f54bbae2ad80cc47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Thu, 25 Aug 2022 13:28:02 +0200 Subject: [PATCH] chg: Much better handling of downloaded files --- lookyloo/lookyloo.py | 43 ++++-------- poetry.lock | 17 +++-- pyproject.toml | 4 +- website/web/sri.txt | 4 +- website/web/static/tree.js | 84 +++++++++++++++-------- website/web/templates/hostname_popup.html | 5 +- 6 files changed, 85 insertions(+), 72 deletions(-) diff --git a/lookyloo/lookyloo.py b/lookyloo/lookyloo.py index 14a6794..90f5afd 100644 --- a/lookyloo/lookyloo.py +++ b/lookyloo/lookyloo.py @@ -521,25 +521,20 @@ class Lookyloo(): '''Get the thumbnail of the rendered page. Always crop to a square.''' to_return = BytesIO() size = width, width - filename, data = self.get_data(capture_uuid) - if filename: - download_img: Path = get_homedir() / 'website' / 'web' / 'static' / 'download.png' - to_thumbnail = Image.open(download_img) - else: - try: - s = self.get_screenshot(capture_uuid) - orig_screenshot = Image.open(s) - to_thumbnail = orig_screenshot.crop((0, 0, orig_screenshot.width, orig_screenshot.width)) - except Image.DecompressionBombError as e: - # The image is most probably too big: https://pillow.readthedocs.io/en/stable/reference/Image.html - self.logger.warning(f'Unable to generate the screenshot thumbnail of {capture_uuid}: image too big ({e}).') - error_img: Path = get_homedir() / 'website' / 'web' / 'static' / 'error_screenshot.png' - to_thumbnail = Image.open(error_img) - except UnidentifiedImageError as e: - # The image is most probably too big: https://pillow.readthedocs.io/en/stable/reference/Image.html - self.logger.warning(f'Unable to generate the screenshot thumbnail of {capture_uuid}: {e}.') - error_img = get_homedir() / 'website' / 'web' / 'static' / 'error_screenshot.png' - to_thumbnail = Image.open(error_img) + try: + s = self.get_screenshot(capture_uuid) + orig_screenshot = Image.open(s) + to_thumbnail = orig_screenshot.crop((0, 0, orig_screenshot.width, orig_screenshot.width)) + except Image.DecompressionBombError as e: + # The image is most probably too big: https://pillow.readthedocs.io/en/stable/reference/Image.html + self.logger.warning(f'Unable to generate the screenshot thumbnail of {capture_uuid}: image too big ({e}).') + error_img: Path = get_homedir() / 'website' / 'web' / 'static' / 'error_screenshot.png' + to_thumbnail = Image.open(error_img) + except UnidentifiedImageError as e: + # The image is most probably too big: https://pillow.readthedocs.io/en/stable/reference/Image.html + self.logger.warning(f'Unable to generate the screenshot thumbnail of {capture_uuid}: {e}.') + error_img = get_homedir() / 'website' / 'web' / 'static' / 'error_screenshot.png' + to_thumbnail = Image.open(error_img) to_thumbnail.thumbnail(size) to_thumbnail.save(to_return, 'png') @@ -930,16 +925,6 @@ class Lookyloo(): 'url_object': url, } - if url.empty_response: - if ct.root_hartree.rendered_node == url: - # check if a file is available - filename, data = self.get_data(capture_uuid) - if filename: - # we have a file to download - url.add_feature('has_dl_file', True) - url.add_feature('downloaded_filename', filename) - url.add_feature('downloaded_filesize', data.getbuffer().nbytes) - if not url.empty_response: # Index lookup # %%% Full body %%% diff --git a/poetry.lock b/poetry.lock index 26f9cab..c7d88db 100644 --- a/poetry.lock +++ b/poetry.lock @@ -403,7 +403,7 @@ tornado = ["tornado (>=0.2)"] [[package]] name = "har2tree" -version = "1.14.2" +version = "1.14.3" description = "HTTP Archive (HAR) to ETE Toolkit generator" category = "main" optional = false @@ -738,7 +738,7 @@ websockets = "10.1" [[package]] name = "playwrightcapture" -version = "1.14.3" +version = "1.14.4" description = "A simple library to capture websites using playwright" category = "main" optional = false @@ -746,7 +746,7 @@ python-versions = ">=3.8,<4.0" [package.dependencies] dateparser = ">=1.1.1,<2.0.0" -playwright = ">=1.25.1,<2.0.0" +playwright = ">=1.25.2,<2.0.0" [package.extras] recaptcha = ["SpeechRecognition (>=3.8.1,<4.0.0)", "pydub (>=0.25.1,<0.26.0)", "requests (>=2.28.1,<3.0.0)"] @@ -1427,7 +1427,7 @@ misp = ["python-magic", "pydeep2"] [metadata] lock-version = "1.1" python-versions = ">=3.8,<3.11" -content-hash = "35dd47ad4e1c44d02655af2d5074d27fabc5cf442e338c219e86064aed7f601f" +content-hash = "7eda63da4c3b4d62b1752e0f33f2b820e9773e9720c68609f661867f9ddfb6b6" [metadata.files] aiohttp = [ @@ -1789,8 +1789,8 @@ gunicorn = [ {file = "gunicorn-20.1.0.tar.gz", hash = "sha256:e0a968b5ba15f8a328fdfd7ab1fcb5af4470c28aaf7e55df02a99bc13138e6e8"}, ] har2tree = [ - {file = "har2tree-1.14.2-py3-none-any.whl", hash = "sha256:6d0068a8ebdbda0ba011f99fbdcf64e703b248b955fcf6c0d7a6d01f39897784"}, - {file = "har2tree-1.14.2.tar.gz", hash = "sha256:4f066bae7ee5737b51b96f77d87646f09992b1e2c2baac58af9b536033cb53b8"}, + {file = "har2tree-1.14.3-py3-none-any.whl", hash = "sha256:0d59af3523608eaaada54b2c3880c141230b276f0dd5d3e2d02b1377e7b0562e"}, + {file = "har2tree-1.14.3.tar.gz", hash = "sha256:937b755cef81a93fc3f3f46b984c93692dcc34bbf0bfe79d796b6454648e3b53"}, ] hiredis = [ {file = "hiredis-2.0.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:b4c8b0bc5841e578d5fb32a16e0c305359b987b850a06964bd5a62739d688048"}, @@ -1901,7 +1901,6 @@ lxml = [ {file = "lxml-4.9.1-cp27-cp27m-win_amd64.whl", hash = "sha256:4cfbe42c686f33944e12f45a27d25a492cc0e43e1dc1da5d6a87cbcaf2e95627"}, {file = "lxml-4.9.1-cp27-cp27mu-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:dad7b164905d3e534883281c050180afcf1e230c3d4a54e8038aa5cfcf312b84"}, {file = "lxml-4.9.1-cp27-cp27mu-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:a614e4afed58c14254e67862456d212c4dcceebab2eaa44d627c2ca04bf86837"}, - {file = "lxml-4.9.1-cp310-cp310-macosx_10_15_universal2.whl", hash = "sha256:49a866923e69bc7da45a0565636243707c22752fc38f6b9d5c8428a86121022c"}, {file = "lxml-4.9.1-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:f9ced82717c7ec65a67667bb05865ffe38af0e835cdd78728f1209c8fffe0cad"}, {file = "lxml-4.9.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:d9fc0bf3ff86c17348dfc5d322f627d78273eba545db865c3cd14b3f19e57fa5"}, {file = "lxml-4.9.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:e5f66bdf0976ec667fc4594d2812a00b07ed14d1b44259d19a41ae3fff99f2b8"}, @@ -2226,8 +2225,8 @@ playwright = [ {file = "playwright-1.25.2-py3-none-win_amd64.whl", hash = "sha256:68ae739f82b78717123eb9d1b28b4619f0b368b88ef73c633681e267680697cd"}, ] playwrightcapture = [ - {file = "PlaywrightCapture-1.14.3-py3-none-any.whl", hash = "sha256:5068f58726ebff1f7928e9793f3c91ead97ba1b096f05021261d126c9ce7fb9e"}, - {file = "PlaywrightCapture-1.14.3.tar.gz", hash = "sha256:7243b908caa16b9e50c662eb01f558a578fd4acb0541fbde7103714ef51219b5"}, + {file = "PlaywrightCapture-1.14.4-py3-none-any.whl", hash = "sha256:8f7e5ce27ff54920a5839dfee7a04c4a0b0fc2c6750cdd1771e607a6d1f8894a"}, + {file = "PlaywrightCapture-1.14.4.tar.gz", hash = "sha256:230e5d538bc51bd724fb808f5321436f743a0a1d379d1de1e8d3d42b020f2091"}, ] prompt-toolkit = [ {file = "prompt_toolkit-3.0.30-py3-none-any.whl", hash = "sha256:d8916d3f62a7b67ab353a952ce4ced6a1d2587dfe9ef8ebc30dd7c386751f289"}, diff --git a/pyproject.toml b/pyproject.toml index 94a4d56..c84d4ba 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -62,8 +62,8 @@ pyhashlookup = "^1.2.0" lief = "^0.12.1" ua-parser = "^0.16.0" Flask-Login = "^0.6.2" -har2tree = "^1.14.2" -playwrightcapture = "^1.14.3" +har2tree = "^1.14.3" +playwrightcapture = "^1.14.4" passivetotal = "^2.5.9" werkzeug = "2.1.2" filetype = "^1.1.0" diff --git a/website/web/sri.txt b/website/web/sri.txt index 959a717..31de328 100644 --- a/website/web/sri.txt +++ b/website/web/sri.txt @@ -12,7 +12,7 @@ "datatables.min.js": "VUgHTv8zUr1K/xtRWvjLwZx2RiyuMfhxefJuDMNFQ2KYT9Y/qGKxgLJ2PBSQzyqbb9uoCzYzccAeF2hC7Qa91Q==", "down.jpg": "LHRHJ5yCaSjNcDfEoChGIfh7K5HrMYbaGn7EOlxgZ8GoLIwb0nFBkpoOMG9gMHA/pBX2skkXMukvKJC6P6FBGg==", "down_left.jpg": "UwHkJaZGayY1LewuFM3bJHQCUPG1vYyrVeiGG5mCM9MD9FtAhdbD4hBY3JZNDWv93CXeEAbxL1kqEeHTKnyquQ==", - "download.png": "A0K7cyVs9BtVjVBEBJtbWYzdKRCp+HIHRO9S0PKe/VrhCMk5Z90J64F794Q/g1H8NoeUSB0KV545kOQWl9BaOw==", + "download.svg": "ufH75x06tbf0RQy2MgbHDycQUBkWXZp9LFR52Bggfs0hYI4Tpni0mRkJnq0UHzMcSPmfM4zAn3Ddgsds7dCBCA==", "empty.svg": "6tfMLNzDFV9P6t1rC2tDRQtOGzrxi/VtIBc8aV0jo4i3u+dn1fIe3/fySBFA6z13n+XjISF5bTRUNBsN3LWinQ==", "error_screenshot.png": "IkUKnQ47PYYreukA7Byvx+5ACkcCvqk+jYD0GZoQznsD9qDPWrKAMZxlIku7G3Re19vehIlYawep/THcV/ruTA==", "exe.png": "pWwo9nBLtEss/UJ173zHa6/RpySUyz/XMdNhWc6aRIvwwHMO6a+fLmu2K6TbvO3Jbg4VYL2Af4yhHPyhH3ZeTw==", @@ -35,7 +35,7 @@ "stats.css": "/kY943FwWBTne4IIyf7iBROSfbGd82TeBicEXqKkRwawMVRIvM/Pk5MRa7okUyGIxaDjFQGmV/U1vy+PhN6Jbw==", "stats_graph.js": "0OEouA6NAxLG2wMd7D2vtGoMrXKna7My98Euc6ecyfdO4/6mIJS87vzISOS4zSZ8u4ehpa+p7E0nWhsXXE7H/Q==", "tree.css": "THJ9LnnSJ91DSTvrYoOCxRrenGgwsgG5zKo+eZLH2rRFHn6lpX9UpmRhRic4th9ZYuM9/NJUS7LqYBDRPPnB1Q==", - "tree.js": "GU+5QryGm9FqVNFqPPmw872HA5xCJ30fW2QBe7zkZs89bmMnk7eU1TohmkJ/GKODacjyKvMJ3q/Yfjh3F4gAjQ==", + "tree.js": "N7JU+dJ+8pVPZSzw4zGxO+kzukGBC2BmQYpzfIP4fiENch9cbIJc8lu+7Pwomp8DxbA5RDDvfHauUKHb14JqRQ==", "up.jpg": "d1ljZJ9f5JekyM6RLFFH2Ua44j6neiQBdUIXOenRTjGppQr3JaeglpQIH6BjPCJL177+TH52U3UIRNS5YAyKIg==", "up_right.jpg": "OMmz+n+MxR34P8/fn5t4DkqKqdJRzQbXQ7fAi2lhkZIJGhVs2vIyY1f2hpYoBxDAX1OcYsSE2lqIR2vXNDGZsA==", "video.png": "gJtmkfr8I1Kw43pYEKjg6CAjgmhl1vIBKBQ3ZkxCu3wvxQm+6kf93iLrrFiY2WuiXzxEn2Leu52GJzmVN5id0g==", diff --git a/website/web/static/tree.js b/website/web/static/tree.js index b13f9ff..5511ae0 100644 --- a/website/web/static/tree.js +++ b/website/web/static/tree.js @@ -582,35 +582,63 @@ function update(root, computed_node_width=0) { const thumbnail_size = 64; if (d.data.contains_rendered_urlnode) { center_node = d.data.uuid; - d3.select(this).append("svg").append('rect') - .attr('x', selected_node_bbox.width/3) - .attr('y', node_height - 3) - .attr('width', thumbnail_size) - .attr('height', thumbnail_size) - .attr('fill', 'white') - .attr('stroke', 'black'); + if (d.data.downloaded_filename) { + d3.select(this).append("svg").append('rect') + .attr('x', selected_node_bbox.width/3) + .attr('y', node_height - 3) + .attr('width', thumbnail_size) + .attr('height', thumbnail_size) + .attr('fill', 'white') + .attr('stroke', 'black'); - d3.select(this).append('image') - .attr('x', selected_node_bbox.width/3) - .attr('y', node_height - 3) - .attr('id', 'screenshot_thumbnail') - .attr("width", thumbnail_size) - .attr("height", thumbnail_size) - .attr("xlink:href", `data:image/png;base64,${screenshot_thumbnail}`) - .attr('cursor', 'pointer') - .on('mouseover', (event, d) => { - d3.select('#tooltip') - .style('opacity', 1) - .style('left', `${event.pageX + 10}px`) - .style('top', `${event.pageY + 10}px`) - .text('Contains the URL rendered in the browser.'); - }) - .on('click', (event, d) => { - $("#screenshotModal").modal('toggle'); - }) - .on('mouseout', (event, d) => { - d3.select('#tooltip').style('opacity', 0) - }); + d3.select(this).append('image') + .attr('x', selected_node_bbox.width/3) + .attr('y', node_height - 3) + .attr('id', 'screenshot_thumbnail') + .attr("width", thumbnail_size) + .attr("height", thumbnail_size) + .attr("xlink:href", '/static/download.svg') + .on('mouseover', (event, d) => { + d3.select('#tooltip') + .style('opacity', 1) + .style('left', `${event.pageX + 10}px`) + .style('top', `${event.pageY + 10}px`) + .text(`Contains the downloaded file (${d.data.downloaded_filename}).`); + }) + .on('mouseout', (event, d) => { + d3.select('#tooltip').style('opacity', 0) + }); + } else { + d3.select(this).append("svg").append('rect') + .attr('x', selected_node_bbox.width/3) + .attr('y', node_height - 3) + .attr('width', thumbnail_size) + .attr('height', thumbnail_size) + .attr('fill', 'white') + .attr('stroke', 'black'); + + d3.select(this).append('image') + .attr('x', selected_node_bbox.width/3) + .attr('y', node_height - 3) + .attr('id', 'screenshot_thumbnail') + .attr("width", thumbnail_size) + .attr("height", thumbnail_size) + .attr("xlink:href", `data:image/png;base64,${screenshot_thumbnail}`) + .attr('cursor', 'pointer') + .on('mouseover', (event, d) => { + d3.select('#tooltip') + .style('opacity', 1) + .style('left', `${event.pageX + 10}px`) + .style('top', `${event.pageY + 10}px`) + .text('Contains the URL rendered in the browser.'); + }) + .on('click', (event, d) => { + $("#screenshotModal").modal('toggle'); + }) + .on('mouseout', (event, d) => { + d3.select('#tooltip').style('opacity', 0) + }); + } }; const http_icon_size = 24; diff --git a/website/web/templates/hostname_popup.html b/website/web/templates/hostname_popup.html index 250b14e..7c88b07 100644 --- a/website/web/templates/hostname_popup.html +++ b/website/web/templates/hostname_popup.html @@ -204,13 +204,14 @@

{{ popup_icons_response(url['url_object'], tree_uuid) }} - {% if url['url_object'].has_dl_file %} + {% if url['url_object'].downloaded_filename %} {% if has_pandora %} +
Downloaded file: {{url['url_object'].downloaded_filename}} ({{sizeof_fmt(url['url_object'].downloaded_file.getbuffer().nbytes)}})
{% else %} Download {{url['url_object'].downloaded_filename}} - ({{sizeof_fmt(url['url_object'].downloaded_filesize)}}) + ({{sizeof_fmt(url['url_object'].downloaded_file.getbuffer().nbytes)}}) {% endif%} {% else %}