From 3c6eca3567f85185be0d185e4a3d4cafed2784a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 9 Dec 2020 19:11:19 +0100 Subject: [PATCH] new: Add screenshot thumbnail on tree, move links to the end of the node --- lookyloo/lookyloo.py | 13 +++ poetry.lock | 40 ++++++++- pyproject.toml | 1 + website/web/__init__.py | 2 + website/web/static/tree.js | 144 ++++++++++++++++++-------------- website/web/templates/tree.html | 1 + 6 files changed, 136 insertions(+), 65 deletions(-) diff --git a/lookyloo/lookyloo.py b/lookyloo/lookyloo.py index f8c68cf4..53434c94 100644 --- a/lookyloo/lookyloo.py +++ b/lookyloo/lookyloo.py @@ -24,6 +24,7 @@ from defang import refang # type: ignore import dns.resolver import dns.rdatatype from har2tree import CrawledTree, Har2TreeError, HarFile, HostNode, URLNode +from PIL import Image from pymisp import MISPEvent from pymisp.tools import URLObject, FileObject from redis import Redis @@ -650,6 +651,18 @@ class Lookyloo(): def get_screenshot(self, capture_uuid: str, all_images: bool=False) -> BytesIO: return self._get_raw(capture_uuid, 'png', all_images) + def get_screenshot_thumbnail(self, capture_uuid: str, all_images: bool=False, for_datauri=False) -> Union[str, BytesIO]: + size = 64, 64 + screenshot = Image.open(self._get_raw(capture_uuid, 'png', False)) + c_screenshot = screenshot.crop((0, 0, screenshot.width, screenshot.width)) + c_screenshot.thumbnail(size) + to_return = BytesIO() + c_screenshot.save(to_return, 'png') + if for_datauri: + return base64.b64encode(to_return.getvalue()).decode() + else: + return to_return + def get_capture(self, capture_uuid: str) -> BytesIO: return self._get_raw(capture_uuid) diff --git a/poetry.lock b/poetry.lock index c740464d..4d0d77c9 100644 --- a/poetry.lock +++ b/poetry.lock @@ -596,6 +596,14 @@ category = "dev" optional = false python-versions = "*" +[[package]] +name = "pillow" +version = "8.0.1" +description = "Python Imaging Library (Fork)" +category = "main" +optional = false +python-versions = ">=3.6" + [[package]] name = "prompt-toolkit" version = "3.0.8" @@ -1127,7 +1135,7 @@ misp = ["python-magic", "pydeep"] [metadata] lock-version = "1.1" python-versions = "^3.7" -content-hash = "bd69eeeb9e624116448b67542215105ea09782d51240ec7ba4769adf77cdc521" +content-hash = "f35e1b5ea6f84ceadfe782d669bec094e9d486e30f4541b3c46390de46b7caa6" [metadata.files] aiohttp = [ @@ -1586,6 +1594,36 @@ pickleshare = [ {file = "pickleshare-0.7.5-py2.py3-none-any.whl", hash = "sha256:9649af414d74d4df115d5d718f82acb59c9d418196b7b4290ed47a12ce62df56"}, {file = "pickleshare-0.7.5.tar.gz", hash = "sha256:87683d47965c1da65cdacaf31c8441d12b8044cdec9aca500cd78fc2c683afca"}, ] +pillow = [ + {file = "Pillow-8.0.1-cp36-cp36m-macosx_10_10_x86_64.whl", hash = "sha256:b63d4ff734263ae4ce6593798bcfee6dbfb00523c82753a3a03cbc05555a9cc3"}, + {file = "Pillow-8.0.1-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:5f9403af9c790cc18411ea398a6950ee2def2a830ad0cfe6dc9122e6d528b302"}, + {file = "Pillow-8.0.1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:6b4a8fd632b4ebee28282a9fef4c341835a1aa8671e2770b6f89adc8e8c2703c"}, + {file = "Pillow-8.0.1-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:cc3ea6b23954da84dbee8025c616040d9aa5eaf34ea6895a0a762ee9d3e12e11"}, + {file = "Pillow-8.0.1-cp36-cp36m-win32.whl", hash = "sha256:d8a96747df78cda35980905bf26e72960cba6d355ace4780d4bdde3b217cdf1e"}, + {file = "Pillow-8.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:7ba0ba61252ab23052e642abdb17fd08fdcfdbbf3b74c969a30c58ac1ade7cd3"}, + {file = "Pillow-8.0.1-cp37-cp37m-macosx_10_10_x86_64.whl", hash = "sha256:795e91a60f291e75de2e20e6bdd67770f793c8605b553cb6e4387ce0cb302e09"}, + {file = "Pillow-8.0.1-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:0a2e8d03787ec7ad71dc18aec9367c946ef8ef50e1e78c71f743bc3a770f9fae"}, + {file = "Pillow-8.0.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:006de60d7580d81f4a1a7e9f0173dc90a932e3905cc4d47ea909bc946302311a"}, + {file = "Pillow-8.0.1-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:bd7bf289e05470b1bc74889d1466d9ad4a56d201f24397557b6f65c24a6844b8"}, + {file = "Pillow-8.0.1-cp37-cp37m-win32.whl", hash = "sha256:95edb1ed513e68bddc2aee3de66ceaf743590bf16c023fb9977adc4be15bd3f0"}, + {file = "Pillow-8.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:e38d58d9138ef972fceb7aeec4be02e3f01d383723965bfcef14d174c8ccd039"}, + {file = "Pillow-8.0.1-cp38-cp38-macosx_10_10_x86_64.whl", hash = "sha256:d3d07c86d4efa1facdf32aa878bd508c0dc4f87c48125cc16b937baa4e5b5e11"}, + {file = "Pillow-8.0.1-cp38-cp38-manylinux1_i686.whl", hash = "sha256:fbd922f702582cb0d71ef94442bfca57624352622d75e3be7a1e7e9360b07e72"}, + {file = "Pillow-8.0.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:92c882b70a40c79de9f5294dc99390671e07fc0b0113d472cbea3fde15db1792"}, + {file = "Pillow-8.0.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:7c9401e68730d6c4245b8e361d3d13e1035cbc94db86b49dc7da8bec235d0015"}, + {file = "Pillow-8.0.1-cp38-cp38-win32.whl", hash = "sha256:6c1aca8231625115104a06e4389fcd9ec88f0c9befbabd80dc206c35561be271"}, + {file = "Pillow-8.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:cc9ec588c6ef3a1325fa032ec14d97b7309db493782ea8c304666fb10c3bd9a7"}, + {file = "Pillow-8.0.1-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:eb472586374dc66b31e36e14720747595c2b265ae962987261f044e5cce644b5"}, + {file = "Pillow-8.0.1-cp39-cp39-manylinux1_i686.whl", hash = "sha256:0eeeae397e5a79dc088d8297a4c2c6f901f8fb30db47795113a4a605d0f1e5ce"}, + {file = "Pillow-8.0.1-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:81f812d8f5e8a09b246515fac141e9d10113229bc33ea073fec11403b016bcf3"}, + {file = "Pillow-8.0.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:895d54c0ddc78a478c80f9c438579ac15f3e27bf442c2a9aa74d41d0e4d12544"}, + {file = "Pillow-8.0.1-cp39-cp39-win32.whl", hash = "sha256:2fb113757a369a6cdb189f8df3226e995acfed0a8919a72416626af1a0a71140"}, + {file = "Pillow-8.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:59e903ca800c8cfd1ebe482349ec7c35687b95e98cefae213e271c8c7fffa021"}, + {file = "Pillow-8.0.1-pp36-pypy36_pp73-macosx_10_10_x86_64.whl", hash = "sha256:5abd653a23c35d980b332bc0431d39663b1709d64142e3652890df4c9b6970f6"}, + {file = "Pillow-8.0.1-pp36-pypy36_pp73-manylinux2010_x86_64.whl", hash = "sha256:4b0ef2470c4979e345e4e0cc1bbac65fda11d0d7b789dbac035e4c6ce3f98adb"}, + {file = "Pillow-8.0.1-pp37-pypy37_pp73-win32.whl", hash = "sha256:8de332053707c80963b589b22f8e0229f1be1f3ca862a932c1bcd48dafb18dd8"}, + {file = "Pillow-8.0.1.tar.gz", hash = "sha256:11c5c6e9b02c9dac08af04f093eb5a2f84857df70a7d4a6a6ad461aca803fb9e"}, +] prompt-toolkit = [ {file = "prompt_toolkit-3.0.8-py3-none-any.whl", hash = "sha256:7debb9a521e0b1ee7d2fe96ee4bd60ef03c6492784de0547337ca4433e46aa63"}, {file = "prompt_toolkit-3.0.8.tar.gz", hash = "sha256:25c95d2ac813909f813c93fde734b6e44406d1477a9faef7c915ff37d39c0a8c"}, diff --git a/pyproject.toml b/pyproject.toml index 8e17e0aa..3278886c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,6 +54,7 @@ pymisp = {version = "^2.4.135", extras = ["url"]} python-magic = {version = "^0.4.18", optional = true} # pydeep requires libfuzzy-dev, and is only used in the MISP export module pydeep = {version = "^0.4", optional = true} +Pillow = "^8.0.1" [tool.poetry.extras] misp = ['python-magic', 'pydeep'] diff --git a/website/web/__init__.py b/website/web/__init__.py index 66dd5cc4..3abb4e0a 100644 --- a/website/web/__init__.py +++ b/website/web/__init__.py @@ -355,8 +355,10 @@ def tree(tree_uuid: str, urlnode_uuid: Optional[str]=None): try: tree_json, start_time, user_agent, root_url, meta = lookyloo.load_tree(tree_uuid) + b64_thumbnail = lookyloo.get_screenshot_thumbnail(tree_uuid, for_datauri=True) return render_template('tree.html', tree_json=tree_json, start_time=start_time, user_agent=user_agent, root_url=root_url, tree_uuid=tree_uuid, + screenshot_thumbnail=b64_thumbnail, meta=meta, enable_mail_notification=enable_mail_notification, enable_context_by_users=enable_context_by_users, enable_categorization=enable_categorization, diff --git a/website/web/static/tree.js b/website/web/static/tree.js index 7bde001e..61bc724b 100644 --- a/website/web/static/tree.js +++ b/website/web/static/tree.js @@ -380,35 +380,6 @@ function update(root, computed_node_width=0) { .attr("id", d => `node_${d.data.uuid}`) .attr("transform", `translate(${root.y0}, ${root.x0})`); - node_group - // Add Circle for the nodes - .append('circle') - .attr('class', 'node') - .attr('r', 1e-6) - .style("fill", d => d._children ? "lightsteelblue" : "#fff") - .on('mouseover', (event, d) => { - if (d.children || d._children) { - d3.select('#tooltip') - .style('opacity', 1) - .style('left', `${event.pageX + 10}px`) - .style('top', `${event.pageY + 10}px`) - .text(d.children ? 'Collapse the childrens of this node.' : 'Expand the childrens of this node.'); - }; - } - ) - .on('mouseout', (event, d) => { - if (d.children || d._children) { - d3.select('#tooltip').style('opacity', 0) - }; - } - ) - .on('click', (event, d) => { - if (d.children || d._children) { - toggle_children_collapse(event, d) - }; - } - ); - let node_data = node_group .append('svg') .attr('class', 'node_data') @@ -418,7 +389,7 @@ function update(root, computed_node_width=0) { node_data.append('rect') .attr("rx", 6) .attr("ry", 6) - .attr('x', 12) + .attr('x', 0) .attr('y', 0) .attr('width', 0) .style("opacity", "0.5") @@ -426,15 +397,14 @@ function update(root, computed_node_width=0) { .attr('stroke-opacity', "0.8") .attr("stroke-width", "2") .attr("stroke-linecap", "round") - .attr("fill", "white"); + .attr("fill", "white") // Set Hostname text node_data - .append(d => text_entry(15, 5, d)); // Popup + .append(d => text_entry(10, 5, d)); // Popup // Set list of icons node_data - .append(d => icon_list(17, 35, d)); - + .append(d => icon_list(12, 35, d)); node_group.select('.node_data').each(function(d){ // set position of icons based of their length @@ -450,8 +420,10 @@ function update(root, computed_node_width=0) { .attr('height', node_height + 5) .attr('width', selected_node_bbox_init.width + 50); + // Set the width for all the nodes let selected_node_bbox = d3.select(this).select('rect').node().getBoundingClientRect(); // Required, as the node width need to include the rectangle + d.node_width = selected_node_bbox.width; node_width = node_width > selected_node_bbox.width ? node_width : selected_node_bbox.width; // Set Bookmark @@ -501,6 +473,39 @@ function update(root, computed_node_width=0) { }) .on('mouseout', (event, d) => d3.select('#tooltip').style('opacity', 0)); }; + var thumbnail_size = 64; + if (d.data.contains_rendered_urlnode) { + d3.select(this).append("svg").append('rect') + .attr('x', selected_node_bbox.width/3) + .attr('y', node_height - 10) + .attr('width', thumbnail_size) + .attr('height', thumbnail_size) + .attr('fill', 'white') + .attr('stroke', 'black'); + + d3.select(this).append('image') + .attr('x', selected_node_bbox.width/3) + .attr('y', node_height - 10) + .attr('id', 'screenshot_thumbnail') + .attr("width", thumbnail_size) + .attr("height", thumbnail_size) + .attr("xlink:href", `data:image/png;base64,${screenshot_thumbnail}`) + .attr('cursor', 'pointer') + .on('mouseover', (event, d) => { + d3.select('#tooltip') + .style('opacity', 1) + .style('left', `${event.pageX + 10}px`) + .style('top', `${event.pageY + 10}px`) + .text('Contains the URL rendered in the browser.'); + }) + .on('click', (event, d) => { + $("#screenshotModal").modal('toggle'); + }) + .on('mouseout', (event, d) => { + d3.select('#tooltip').style('opacity', 0) + }); + }; + const context_icon_size = 24; if (d.data.malicious) { // set bomb @@ -515,7 +520,7 @@ function update(root, computed_node_width=0) { d3.select(this).append('image') .attr('x', selected_node_bbox.width - 22 - http_icon_size) .attr('y', selected_node_bbox.height - 13) - .attr('id', 'insecure_image') + .attr('id', 'malicious_image') .attr("width", context_icon_size) .attr("height", context_icon_size) .attr("xlink:href", '/static/bomb.svg') @@ -540,7 +545,7 @@ function update(root, computed_node_width=0) { d3.select(this).append('image') .attr('x', selected_node_bbox.width - 22 - http_icon_size) .attr('y', selected_node_bbox.height - 13) - .attr('id', 'insecure_image') + .attr('id', 'known_image') .attr("width", context_icon_size) .attr("height", context_icon_size) .attr("xlink:href", '/static/check.svg') @@ -565,7 +570,7 @@ function update(root, computed_node_width=0) { d3.select(this).append('image') .attr('x', selected_node_bbox.width - 22 - http_icon_size) .attr('y', selected_node_bbox.height - 13) - .attr('id', 'insecure_image') + .attr('id', 'empty_image') .attr("width", context_icon_size) .attr("height", context_icon_size) .attr("xlink:href", '/static/empty.svg') @@ -578,6 +583,38 @@ function update(root, computed_node_width=0) { }) .on('mouseout', (event, d) => d3.select('#tooltip').style('opacity', 0)); }; + if (d.children || d._children) { + d3.select(this) + // Add Circle for the nodes + .append('circle') + .attr('class', 'node') + .attr('r', 1e-6) + .attr('cx', d => d.node_width) + .attr('cy', d => node_height/2) + .style("fill", d => d._children ? "lightsteelblue" : "#fff") + .on('mouseover', (event, d) => { + if (d.children || d._children) { + d3.select('#tooltip') + .style('opacity', 1) + .style('left', `${event.pageX + 10}px`) + .style('top', `${event.pageY + 10}px`) + .text(d.children ? 'Collapse the childrens of this node.' : 'Expand the childrens of this node.'); + }; + } + ) + .on('mouseout', (event, d) => { + if (d.children || d._children) { + d3.select('#tooltip').style('opacity', 0) + }; + } + ) + .on('click', (event, d) => { + if (d.children || d._children) { + toggle_children_collapse(event, d) + }; + } + ); + }; }); return node_group; @@ -620,40 +657,19 @@ function update(root, computed_node_width=0) { const link = node_container.selectAll('path.link').data(links, d => d.id); // Creates a curved (diagonal) path from parent to the child nodes - let diagonal = (s, d) => { - return `M ${s.y} ${s.x} - C ${(s.y + d.y) / 2} ${s.x}, - ${(s.y + d.y) / 2} ${d.x}, - ${d.y} ${d.x}` - }; + let diagonal = d3.linkHorizontal() + .source(d => {return [d.y, d.x]}) + .target(d => {return [d.parent.y + d.parent.node_width, d.parent.x]}); link.join( enter => enter // Enter any new links at the parent's previous position. .insert('path', "g") .attr("class", "link") - .attr('d', d => { - let o = { - x: d.x0, - y: d.y0 - }; - return diagonal(o, o) - }), + .attr('d', diagonal), update => update, - exit => exit - .call(exit => exit - .attr('d', d => { - let o = { - x: d.x0, - y: d.y0 - }; - return diagonal(o, o) - }) - .remove() - ) - ).call(link => link - .attr('d', d => diagonal(d, d.parent)) - ); + exit => exit.call(exit => exit.attr('d', diagonal).remove()) + ).call(link => link.attr('d', diagonal)); if (computed_node_width === 0) { update(root, node_width) diff --git a/website/web/templates/tree.html b/website/web/templates/tree.html index a3455f0d..11dc3483 100644 --- a/website/web/templates/tree.html +++ b/website/web/templates/tree.html @@ -68,6 +68,7 @@