new: Support for rendered nodes with HTML and downloaded file

lookyloo/lookyloo.py

@@ -913,6 +913,12 @@ class Lookyloo():
             error_img: Path = get_homedir() / 'website' / 'web' / 'static' / 'error_screenshot.png'
             to_thumbnail = Image.open(error_img)
         except UnidentifiedImageError as e:
+            # We might have a direct download link, and no screenshot. Assign the thumbnail accordingly.
+            try:
+                filename, data = self.get_data(capture_uuid)
+                self.logger.info(f'{capture_uuid} is is a download link, set thumbnail.')
+                error_img = get_homedir() / 'website' / 'web' / 'static' / 'download.png'
+            except Exception:
                 # The capture probably doesn't have a screenshot at all, no need to log that as a warning.
                 self.logger.debug(f'Unable to generate the screenshot thumbnail of {capture_uuid}: {e}.')
                 error_img = get_homedir() / 'website' / 'web' / 'static' / 'error_screenshot.png'

website/web/sri.txt

@@ -12,6 +12,7 @@
     "datatables.min.js": "8ua4soo5FXHNIbuX9zDsc7L/JOQE5q5tqJ+GOofQN4/8sDnJ3l5zNj54dIWKoppTfrthPBjetweH/PPf97P/bg==",
     "down.jpg": "LHRHJ5yCaSjNcDfEoChGIfh7K5HrMYbaGn7EOlxgZ8GoLIwb0nFBkpoOMG9gMHA/pBX2skkXMukvKJC6P6FBGg==",
     "down_left.jpg": "UwHkJaZGayY1LewuFM3bJHQCUPG1vYyrVeiGG5mCM9MD9FtAhdbD4hBY3JZNDWv93CXeEAbxL1kqEeHTKnyquQ==",
+    "download.png": "J8y1gDKURf3AhgYDuqCnfaVLKRG2MI6k37xSvR5pJBAZ3aNmA6dDw6+UGf65hLBN3eGksaBJUeroBW/LDlUTqQ==",
     "download.svg": "8Dmi0Z56+uecmE4mW03JEP6IdxeFFaQcUgdnqAzrYbVLED0YxnQTuiKqn5qeLZNlK1XQpGnDC47YzHvE7zAKig==",
     "empty.svg": "6tfMLNzDFV9P6t1rC2tDRQtOGzrxi/VtIBc8aV0jo4i3u+dn1fIe3/fySBFA6z13n+XjISF5bTRUNBsN3LWinQ==",
     "error_screenshot.png": "IkUKnQ47PYYreukA7Byvx+5ACkcCvqk+jYD0GZoQznsD9qDPWrKAMZxlIku7G3Re19vehIlYawep/THcV/ruTA==",
@@ -35,7 +36,7 @@
     "stats.css": "/kY943FwWBTne4IIyf7iBROSfbGd82TeBicEXqKkRwawMVRIvM/Pk5MRa7okUyGIxaDjFQGmV/U1vy+PhN6Jbw==",
     "stats_graph.js": "0OEouA6NAxLG2wMd7D2vtGoMrXKna7My98Euc6ecyfdO4/6mIJS87vzISOS4zSZ8u4ehpa+p7E0nWhsXXE7H/Q==",
     "tree.css": "R/pWQnE8kMacDrzGy/NpA1tJoHAZpOjFiX6dqtBe+PqAnhYMn1CIQzODh8Ifvh6hBTPLRWX3bsabfEvfaI7Z6A==",
-    "tree.js": "jvaOjno3+4vX3Caam4PSXeZf4G3o9uh8whfOsND2zTfDxLQwo1P+umNQQx7JDPsf5Fan7tHDGOnBTupb9CjLlA==",
+    "tree.js": "67R8M7znCyonDtD3GdqyF7uq4p18SLWB/BSwR1dK45M63SHmksENlBEPl9WS3OepzmnfW8q00sH/UBEWqGE8rQ==",
     "up.jpg": "d1ljZJ9f5JekyM6RLFFH2Ua44j6neiQBdUIXOenRTjGppQr3JaeglpQIH6BjPCJL177+TH52U3UIRNS5YAyKIg==",
     "up_right.jpg": "OMmz+n+MxR34P8/fn5t4DkqKqdJRzQbXQ7fAi2lhkZIJGhVs2vIyY1f2hpYoBxDAX1OcYsSE2lqIR2vXNDGZsA==",
     "video.png": "gJtmkfr8I1Kw43pYEKjg6CAjgmhl1vIBKBQ3ZkxCu3wvxQm+6kf93iLrrFiY2WuiXzxEn2Leu52GJzmVN5id0g==",

website/web/static/tree.js

@@ -582,33 +582,6 @@ function update(root, computed_node_width=0) {
                 const thumbnail_size = 64;
                 if (d.data.contains_rendered_urlnode) {
                   center_node = d.data.uuid;
-                  if (d.data.downloaded_filename) {
-                    d3.select(this).append("svg").append('rect')
-                        .attr('x', selected_node_bbox.width/3)
-                        .attr('y', node_height - 3)
-                        .attr('width', thumbnail_size)
-                        .attr('height', thumbnail_size)
-                        .attr('fill', 'white')
-                        .attr('stroke', 'black');
-
-                    d3.select(this).append('image')
-                        .attr('x', selected_node_bbox.width/3)
-                        .attr('y', node_height - 3)
-                        .attr('id', 'screenshot_thumbnail')
-                        .attr("width", thumbnail_size)
-                        .attr("height", thumbnail_size)
-                        .attr("xlink:href", '/static/download.svg')
-                        .on('mouseover', (event, d) => {
-                            d3.select('#tooltip')
-                              .style('opacity', 1)
-                              .style('left', `${event.pageX + 10}px`)
-                              .style('top', `${event.pageY + 10}px`)
-                              .text(`Contains the downloaded file (${d.data.downloaded_filename}).`);
-                        })
-                        .on('mouseout', (event, d) => {
-                            d3.select('#tooltip').style('opacity', 0)
-                        });
-                  } else {
                   if (favicon) {
                     d3.select(this).append('image')
                       .attr('x', selected_node_bbox.width/6)
@@ -626,7 +599,6 @@ function update(root, computed_node_width=0) {
                             .text('Potential favicon.');
                       });
                   }
-                    if (screenshot_thumbnail) {
                   d3.select(this).append("svg").append('rect')
                     .attr('x', selected_node_bbox.width/2)
                     .attr('y', node_height - 3)
@@ -645,10 +617,11 @@ function update(root, computed_node_width=0) {
                     .attr('cursor', 'pointer')
                     .on('mouseover', (event, d) => {
                         d3.select('#tooltip')
+                          .data(d)
                           .style('opacity', 1)
                           .style('left', `${event.pageX + 10}px`)
                           .style('top', `${event.pageY + 10}px`)
-                              .text('Contains the URL rendered in the browser.');
+                          .text(d => d.data.downloaded_filename ? 'Contains the URL rendered in the browser. It also downloaded a file.': 'Contains the URL rendered in the browser.');
                     })
                     .on('click', (event, d) => {
                         $("#screenshotModal").modal('toggle');
@@ -656,8 +629,6 @@ function update(root, computed_node_width=0) {
                     .on('mouseout', (event, d) => {
                         d3.select('#tooltip').style('opacity', 0)
                     });
-                    }
-                  }
                 };
 
                 const http_icon_size = 24;

website/web/templates/hostname_popup.html

@@ -241,6 +241,7 @@
           {{ popup_icons_response(url['url_object'], tree_uuid) }}
 
           {% if url['url_object'].downloaded_filename %}
+          <div>
             {% if has_pandora %}
               <div> Downloaded file: <b>{{url['url_object'].downloaded_filename}}</b> ({{sizeof_fmt(url['url_object'].downloaded_file.getbuffer().nbytes)}})</div>
               {{ pandora_submit() }}
@@ -249,9 +250,11 @@
                 Download {{url['url_object'].downloaded_filename}}
               </a> ({{sizeof_fmt(url['url_object'].downloaded_file.getbuffer().nbytes)}})
             {% endif%}
+          </div>
+          {% endif%}
 
-          {% else %}
           {% if url['url_object'].rendered_html %}
+          <div>
             <a href="{{ url_for('urlnode_rendered_content', tree_uuid=tree_uuid, node_uuid=url['url_object'].uuid) }}">
                 Download rendered HTML page
             </a>({{ sizeof_fmt(url['url_object'].rendered_html.getbuffer().nbytes)}})
@@ -259,10 +262,12 @@
             <a href="{{ url_for('urlnode_urls_in_rendered_content', tree_uuid=tree_uuid, node_uuid=url['url_object'].uuid) }}">
                 Download URLs in rendered HTML page
             </a>
+          </div>
           {% endif %}
+
           <div>
             {% if url['url_object'].empty_response %}
-            Empty body.
+            Empty HTML body.
             {% else %}
               {{ ressource_legitimacy_details(url['legitimacy'], url['url_object'].body.getbuffer().nbytes) }}
               {% if has_pandora %}
@@ -270,7 +275,6 @@
               {% endif %}
             {% endif %}
           </div>
-          {%endif%}
 
 
           {% if url['known_content'] %}