new: Add screenshot thumbnail on tree, move links to the end of the node

lookyloo/lookyloo.py

@@ -24,6 +24,7 @@ from defang import refang  # type: ignore
 import dns.resolver
 import dns.rdatatype
 from har2tree import CrawledTree, Har2TreeError, HarFile, HostNode, URLNode
+from PIL import Image
 from pymisp import MISPEvent
 from pymisp.tools import URLObject, FileObject
 from redis import Redis
@@ -650,6 +651,18 @@ class Lookyloo():
     def get_screenshot(self, capture_uuid: str, all_images: bool=False) -> BytesIO:
         return self._get_raw(capture_uuid, 'png', all_images)
 
+    def get_screenshot_thumbnail(self, capture_uuid: str, all_images: bool=False, for_datauri=False) -> Union[str, BytesIO]:
+        size = 64, 64
+        screenshot = Image.open(self._get_raw(capture_uuid, 'png', False))
+        c_screenshot = screenshot.crop((0, 0, screenshot.width, screenshot.width))
+        c_screenshot.thumbnail(size)
+        to_return = BytesIO()
+        c_screenshot.save(to_return, 'png')
+        if for_datauri:
+            return base64.b64encode(to_return.getvalue()).decode()
+        else:
+            return to_return
+
     def get_capture(self, capture_uuid: str) -> BytesIO:
         return self._get_raw(capture_uuid)
 

poetry.lock

@@ -596,6 +596,14 @@ category = "dev"
 optional = false
 python-versions = "*"
 
+[[package]]
+name = "pillow"
+version = "8.0.1"
+description = "Python Imaging Library (Fork)"
+category = "main"
+optional = false
+python-versions = ">=3.6"
+
 [[package]]
 name = "prompt-toolkit"
 version = "3.0.8"
@@ -1127,7 +1135,7 @@ misp = ["python-magic", "pydeep"]
 [metadata]
 lock-version = "1.1"
 python-versions = "^3.7"
-content-hash = "bd69eeeb9e624116448b67542215105ea09782d51240ec7ba4769adf77cdc521"
+content-hash = "f35e1b5ea6f84ceadfe782d669bec094e9d486e30f4541b3c46390de46b7caa6"
 
 [metadata.files]
 aiohttp = [
@@ -1586,6 +1594,36 @@ pickleshare = [
     {file = "pickleshare-0.7.5-py2.py3-none-any.whl", hash = "sha256:9649af414d74d4df115d5d718f82acb59c9d418196b7b4290ed47a12ce62df56"},
     {file = "pickleshare-0.7.5.tar.gz", hash = "sha256:87683d47965c1da65cdacaf31c8441d12b8044cdec9aca500cd78fc2c683afca"},
 ]
+pillow = [
+    {file = "Pillow-8.0.1-cp36-cp36m-macosx_10_10_x86_64.whl", hash = "sha256:b63d4ff734263ae4ce6593798bcfee6dbfb00523c82753a3a03cbc05555a9cc3"},
+    {file = "Pillow-8.0.1-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:5f9403af9c790cc18411ea398a6950ee2def2a830ad0cfe6dc9122e6d528b302"},
+    {file = "Pillow-8.0.1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:6b4a8fd632b4ebee28282a9fef4c341835a1aa8671e2770b6f89adc8e8c2703c"},
+    {file = "Pillow-8.0.1-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:cc3ea6b23954da84dbee8025c616040d9aa5eaf34ea6895a0a762ee9d3e12e11"},
+    {file = "Pillow-8.0.1-cp36-cp36m-win32.whl", hash = "sha256:d8a96747df78cda35980905bf26e72960cba6d355ace4780d4bdde3b217cdf1e"},
+    {file = "Pillow-8.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:7ba0ba61252ab23052e642abdb17fd08fdcfdbbf3b74c969a30c58ac1ade7cd3"},
+    {file = "Pillow-8.0.1-cp37-cp37m-macosx_10_10_x86_64.whl", hash = "sha256:795e91a60f291e75de2e20e6bdd67770f793c8605b553cb6e4387ce0cb302e09"},
+    {file = "Pillow-8.0.1-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:0a2e8d03787ec7ad71dc18aec9367c946ef8ef50e1e78c71f743bc3a770f9fae"},
+    {file = "Pillow-8.0.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:006de60d7580d81f4a1a7e9f0173dc90a932e3905cc4d47ea909bc946302311a"},
+    {file = "Pillow-8.0.1-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:bd7bf289e05470b1bc74889d1466d9ad4a56d201f24397557b6f65c24a6844b8"},
+    {file = "Pillow-8.0.1-cp37-cp37m-win32.whl", hash = "sha256:95edb1ed513e68bddc2aee3de66ceaf743590bf16c023fb9977adc4be15bd3f0"},
+    {file = "Pillow-8.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:e38d58d9138ef972fceb7aeec4be02e3f01d383723965bfcef14d174c8ccd039"},
+    {file = "Pillow-8.0.1-cp38-cp38-macosx_10_10_x86_64.whl", hash = "sha256:d3d07c86d4efa1facdf32aa878bd508c0dc4f87c48125cc16b937baa4e5b5e11"},
+    {file = "Pillow-8.0.1-cp38-cp38-manylinux1_i686.whl", hash = "sha256:fbd922f702582cb0d71ef94442bfca57624352622d75e3be7a1e7e9360b07e72"},
+    {file = "Pillow-8.0.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:92c882b70a40c79de9f5294dc99390671e07fc0b0113d472cbea3fde15db1792"},
+    {file = "Pillow-8.0.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:7c9401e68730d6c4245b8e361d3d13e1035cbc94db86b49dc7da8bec235d0015"},
+    {file = "Pillow-8.0.1-cp38-cp38-win32.whl", hash = "sha256:6c1aca8231625115104a06e4389fcd9ec88f0c9befbabd80dc206c35561be271"},
+    {file = "Pillow-8.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:cc9ec588c6ef3a1325fa032ec14d97b7309db493782ea8c304666fb10c3bd9a7"},
+    {file = "Pillow-8.0.1-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:eb472586374dc66b31e36e14720747595c2b265ae962987261f044e5cce644b5"},
+    {file = "Pillow-8.0.1-cp39-cp39-manylinux1_i686.whl", hash = "sha256:0eeeae397e5a79dc088d8297a4c2c6f901f8fb30db47795113a4a605d0f1e5ce"},
+    {file = "Pillow-8.0.1-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:81f812d8f5e8a09b246515fac141e9d10113229bc33ea073fec11403b016bcf3"},
+    {file = "Pillow-8.0.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:895d54c0ddc78a478c80f9c438579ac15f3e27bf442c2a9aa74d41d0e4d12544"},
+    {file = "Pillow-8.0.1-cp39-cp39-win32.whl", hash = "sha256:2fb113757a369a6cdb189f8df3226e995acfed0a8919a72416626af1a0a71140"},
+    {file = "Pillow-8.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:59e903ca800c8cfd1ebe482349ec7c35687b95e98cefae213e271c8c7fffa021"},
+    {file = "Pillow-8.0.1-pp36-pypy36_pp73-macosx_10_10_x86_64.whl", hash = "sha256:5abd653a23c35d980b332bc0431d39663b1709d64142e3652890df4c9b6970f6"},
+    {file = "Pillow-8.0.1-pp36-pypy36_pp73-manylinux2010_x86_64.whl", hash = "sha256:4b0ef2470c4979e345e4e0cc1bbac65fda11d0d7b789dbac035e4c6ce3f98adb"},
+    {file = "Pillow-8.0.1-pp37-pypy37_pp73-win32.whl", hash = "sha256:8de332053707c80963b589b22f8e0229f1be1f3ca862a932c1bcd48dafb18dd8"},
+    {file = "Pillow-8.0.1.tar.gz", hash = "sha256:11c5c6e9b02c9dac08af04f093eb5a2f84857df70a7d4a6a6ad461aca803fb9e"},
+]
 prompt-toolkit = [
     {file = "prompt_toolkit-3.0.8-py3-none-any.whl", hash = "sha256:7debb9a521e0b1ee7d2fe96ee4bd60ef03c6492784de0547337ca4433e46aa63"},
     {file = "prompt_toolkit-3.0.8.tar.gz", hash = "sha256:25c95d2ac813909f813c93fde734b6e44406d1477a9faef7c915ff37d39c0a8c"},

pyproject.toml

@@ -54,6 +54,7 @@ pymisp = {version = "^2.4.135", extras = ["url"]}
 python-magic = {version = "^0.4.18", optional = true}
 # pydeep requires libfuzzy-dev, and is only used in the MISP export module
 pydeep = {version = "^0.4", optional = true}
+Pillow = "^8.0.1"
 
 [tool.poetry.extras]
 misp = ['python-magic', 'pydeep']

website/web/__init__.py

@@ -355,8 +355,10 @@ def tree(tree_uuid: str, urlnode_uuid: Optional[str]=None):
 
     try:
         tree_json, start_time, user_agent, root_url, meta = lookyloo.load_tree(tree_uuid)
+        b64_thumbnail = lookyloo.get_screenshot_thumbnail(tree_uuid, for_datauri=True)
         return render_template('tree.html', tree_json=tree_json, start_time=start_time,
                                user_agent=user_agent, root_url=root_url, tree_uuid=tree_uuid,
+                               screenshot_thumbnail=b64_thumbnail,
                                meta=meta, enable_mail_notification=enable_mail_notification,
                                enable_context_by_users=enable_context_by_users,
                                enable_categorization=enable_categorization,

website/web/static/tree.js

@@ -380,35 +380,6 @@ function update(root, computed_node_width=0) {
                 .attr("id", d => `node_${d.data.uuid}`)
                 .attr("transform", `translate(${root.y0}, ${root.x0})`);
 
-            node_group
-                // Add Circle for the nodes
-                .append('circle')
-                .attr('class', 'node')
-                .attr('r', 1e-6)
-                .style("fill", d => d._children ? "lightsteelblue" : "#fff")
-                .on('mouseover', (event, d) => {
-                    if (d.children || d._children) {
-                      d3.select('#tooltip')
-                        .style('opacity', 1)
-                        .style('left', `${event.pageX + 10}px`)
-                        .style('top', `${event.pageY + 10}px`)
-                        .text(d.children ? 'Collapse the childrens of this node.' : 'Expand the childrens of this node.');
-                    };
-                  }
-                )
-                .on('mouseout', (event, d) => {
-                    if (d.children || d._children) {
-                      d3.select('#tooltip').style('opacity', 0)
-                    };
-                  }
-                )
-                .on('click', (event, d) => {
-                    if (d.children || d._children) {
-                      toggle_children_collapse(event, d)
-                    };
-                  }
-                );
-
             let node_data = node_group
               .append('svg')
               .attr('class', 'node_data')
@@ -418,7 +389,7 @@ function update(root, computed_node_width=0) {
             node_data.append('rect')
               .attr("rx", 6)
               .attr("ry", 6)
-              .attr('x', 12)
+              .attr('x', 0)
               .attr('y', 0)
               .attr('width', 0)
               .style("opacity", "0.5")
@@ -426,15 +397,14 @@ function update(root, computed_node_width=0) {
               .attr('stroke-opacity', "0.8")
               .attr("stroke-width", "2")
               .attr("stroke-linecap", "round")
-              .attr("fill", "white");
+              .attr("fill", "white")
 
             // Set Hostname text
             node_data
-              .append(d => text_entry(15, 5, d));  // Popup
+              .append(d => text_entry(10, 5, d));  // Popup
             // Set list of icons
             node_data
-              .append(d => icon_list(17, 35, d));
+              .append(d => icon_list(12, 35, d));
-
 
             node_group.select('.node_data').each(function(d){
                 // set position of icons based of their length
@@ -450,8 +420,10 @@ function update(root, computed_node_width=0) {
                   .attr('height', node_height + 5)
                   .attr('width', selected_node_bbox_init.width + 50);
 
+
                 // Set the width for all the nodes
                 let selected_node_bbox = d3.select(this).select('rect').node().getBoundingClientRect();  // Required, as the node width need to include the rectangle
+                d.node_width = selected_node_bbox.width;
                 node_width = node_width > selected_node_bbox.width ? node_width : selected_node_bbox.width;
 
                 // Set Bookmark
@@ -501,6 +473,39 @@ function update(root, computed_node_width=0) {
                         })
                         .on('mouseout', (event, d) => d3.select('#tooltip').style('opacity', 0));
                 };
+                var thumbnail_size = 64;
+                if (d.data.contains_rendered_urlnode) {
+                  d3.select(this).append("svg").append('rect')
+                      .attr('x', selected_node_bbox.width/3)
+                      .attr('y', node_height - 10)
+                      .attr('width', thumbnail_size)
+                      .attr('height', thumbnail_size)
+                      .attr('fill', 'white')
+                      .attr('stroke', 'black');
+
+                  d3.select(this).append('image')
+                      .attr('x', selected_node_bbox.width/3)
+                      .attr('y', node_height - 10)
+                      .attr('id', 'screenshot_thumbnail')
+                      .attr("width", thumbnail_size)
+                      .attr("height", thumbnail_size)
+                      .attr("xlink:href", `data:image/png;base64,${screenshot_thumbnail}`)
+                      .attr('cursor', 'pointer')
+                      .on('mouseover', (event, d) => {
+                          d3.select('#tooltip')
+                            .style('opacity', 1)
+                            .style('left', `${event.pageX + 10}px`)
+                            .style('top', `${event.pageY + 10}px`)
+                            .text('Contains the URL rendered in the browser.');
+                      })
+                      .on('click', (event, d) => {
+                          $("#screenshotModal").modal('toggle');
+                      })
+                      .on('mouseout', (event, d) => {
+                          d3.select('#tooltip').style('opacity', 0)
+                      });
+                };
+
                 const context_icon_size = 24;
                 if (d.data.malicious) {
                     // set bomb
@@ -515,7 +520,7 @@ function update(root, computed_node_width=0) {
                     d3.select(this).append('image')
                         .attr('x', selected_node_bbox.width - 22 - http_icon_size)
                         .attr('y', selected_node_bbox.height - 13)
-                        .attr('id', 'insecure_image')
+                        .attr('id', 'malicious_image')
                         .attr("width", context_icon_size)
                         .attr("height", context_icon_size)
                         .attr("xlink:href", '/static/bomb.svg')
@@ -540,7 +545,7 @@ function update(root, computed_node_width=0) {
                     d3.select(this).append('image')
                         .attr('x', selected_node_bbox.width - 22 - http_icon_size)
                         .attr('y', selected_node_bbox.height - 13)
-                        .attr('id', 'insecure_image')
+                        .attr('id', 'known_image')
                         .attr("width", context_icon_size)
                         .attr("height", context_icon_size)
                         .attr("xlink:href", '/static/check.svg')
@@ -565,7 +570,7 @@ function update(root, computed_node_width=0) {
                 d3.select(this).append('image')
                     .attr('x', selected_node_bbox.width - 22 - http_icon_size)
                     .attr('y', selected_node_bbox.height - 13)
-                    .attr('id', 'insecure_image')
+                    .attr('id', 'empty_image')
                     .attr("width", context_icon_size)
                     .attr("height", context_icon_size)
                     .attr("xlink:href", '/static/empty.svg')
@@ -578,6 +583,38 @@ function update(root, computed_node_width=0) {
                     })
                     .on('mouseout', (event, d) => d3.select('#tooltip').style('opacity', 0));
               };
+              if (d.children || d._children) {
+                d3.select(this)
+                  // Add Circle for the nodes
+                  .append('circle')
+                  .attr('class', 'node')
+                  .attr('r', 1e-6)
+                  .attr('cx', d => d.node_width)
+                  .attr('cy', d => node_height/2)
+                  .style("fill", d => d._children ? "lightsteelblue" : "#fff")
+                  .on('mouseover', (event, d) => {
+                      if (d.children || d._children) {
+                        d3.select('#tooltip')
+                          .style('opacity', 1)
+                          .style('left', `${event.pageX + 10}px`)
+                          .style('top', `${event.pageY + 10}px`)
+                          .text(d.children ? 'Collapse the childrens of this node.' : 'Expand the childrens of this node.');
+                      };
+                    }
+                  )
+                  .on('mouseout', (event, d) => {
+                      if (d.children || d._children) {
+                        d3.select('#tooltip').style('opacity', 0)
+                      };
+                    }
+                  )
+                  .on('click', (event, d) => {
+                      if (d.children || d._children) {
+                        toggle_children_collapse(event, d)
+                      };
+                    }
+                  );
+              };
             });
 
             return node_group;
@@ -620,40 +657,19 @@ function update(root, computed_node_width=0) {
   const link = node_container.selectAll('path.link').data(links, d => d.id);
 
   // Creates a curved (diagonal) path from parent to the child nodes
-  let diagonal = (s, d) => {
+  let diagonal = d3.linkHorizontal()
-    return `M ${s.y} ${s.x}
+                        .source(d => {return [d.y, d.x]})
-            C ${(s.y + d.y) / 2} ${s.x},
+                        .target(d => {return [d.parent.y + d.parent.node_width, d.parent.x]});
-              ${(s.y + d.y) / 2} ${d.x},
-              ${d.y} ${d.x}`
-  };
 
   link.join(
     enter => enter
         // Enter any new links at the parent's previous position.
         .insert('path', "g")
         .attr("class", "link")
-        .attr('d', d => {
+        .attr('d', diagonal),
-          let o = {
-              x: d.x0,
-              y: d.y0
-          };
-          return diagonal(o, o)
-        }),
     update => update,
-    exit => exit
+    exit => exit.call(exit => exit.attr('d', diagonal).remove())
-      .call(exit => exit
+  ).call(link => link.attr('d', diagonal));
-                .attr('d', d => {
-                    let o = {
-                        x: d.x0,
-                        y: d.y0
-                    };
-                    return diagonal(o, o)
-                })
-      .remove()
-      )
-  ).call(link => link
-    .attr('d', d => diagonal(d, d.parent))
-  );
 
   if (computed_node_width === 0) {
     update(root, node_width)

website/web/templates/tree.html

@@ -68,6 +68,7 @@
 
 <script>
   var treeUUID = "{{ tree_uuid }}";
+  var screenshot_thumbnail = "{{ screenshot_thumbnail }}";
   var enable_bookmark = {{ enable_bookmark|tojson }};
   var treeData = {{ tree_json | safe }};
 </script>