chg: Improve pop-up, cleanup old code.

lookyloo/modules.py

@@ -39,6 +39,13 @@ class SaneJavaScript():
         "bd9ab35dde3a5242b04c159187732e13b0a6da50ddcff7015dfb78cdd68743e191eaf5cddedd49bef7d2d5a642c217272a40e5ba603fe24ca676a53f8c417c5d": "This is a 1*1 pixel GIF",
         "d052ecec2839340876eb57247cfc2e777dd7f2e868dc37cd3f3f740c8deb94917a0c9f2a4fc8229987a0b91b04726de2d1e9f6bcbe3f9bef0e4b7e0d7f65ea12": "This is a 1*1 pixel GIF",
         "8717074ddf1198d27b9918132a550cb4ba343794cc3d304a793f9d78c9ff6c4929927b414141d40b6f6ad296725520f4c63edeb660ed530267766c2ab74ee4a9": "This is a 1*1 pixel GIF",
+        "6834f1548f26b94357fcc3312a3491e8c87080a84f678f990beb2c745899a01e239964521e64a534d7d5554222f728af966ec6ec8291bc64d2005861bcfd78ec": "This is a 1*1 pixel GIF",
+        "3be8176915593e79bc280d08984a16c29c495bc53be9b439276094b8dcd3764a3c72a046106a06b958e08e67451fe02743175c621a1faa261fe7a9691cc77141": "This is a 1*1 pixel GIF",
+        "826225fc21717d8861a05b9d2f959539aad2d2b131b2afed75d88fbca535e1b0d5a0da8ac69713a0876a0d467848a37a0a7f926aeafad8cf28201382d16466ab": "This is a 1*1 pixel GIF",
+        "202612457d9042fe853daab3ddcc1f0f960c5ffdbe8462fa435713e4d1d85ff0c3f197daf8dba15bda9f5266d7e1f9ecaeee045cbc156a4892d2f931fe6fa1bb": "This is a 1*1 pixel GIF",
+        "b82c6aa1ae927ade5fadbbab478cfaef26d21c1ac441f48e69cfc04cdb779b1e46d7668b4368b933213276068e52f9060228907720492a70fd9bc897191ee77c": "This is a 1*1 pixel GIF",
+        "763de1053a56a94eef4f72044adb2aa370b98ffa6e0add0b1cead7ee27da519e223921c681ae1db3311273f45d0dd3dc022d102d42ce210c90cb3e761b178438": "This is a 1*1 pixel GIF",
+        # "": "This is a 1*1 pixel GIF",
         "f1c33e72643ce366fd578e3b5d393799e8c9ea27b180987826af43b4fc00b65a4eaae5e6426a23448956fee99e3108c6a86f32fb4896c156e24af0571a11c498": "This is a 1*1 pixel PNG",
         "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e": "This is an empty file"
     }

poetry.lock

@@ -306,7 +306,7 @@ publicsuffix2 = "^2.20191221"
 six = "^1.14.0"
 
 [package.source]
-reference = "38fe9bd74a0e8c92cd17d79981196c69d329b3b8"
+reference = "d65554d3a733790431aedb0d4209907471978421"
 type = "git"
 url = "https://github.com/viper-framework/har2tree.git"
 [[package]]
@@ -899,7 +899,7 @@ description = "Python 2 and 3 compatibility utilities"
 name = "six"
 optional = false
 python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*"
-version = "1.14.0"
+version = "1.15.0"
 
 [[package]]
 category = "main"
@@ -1539,8 +1539,8 @@ service-identity = [
     {file = "service_identity-18.1.0.tar.gz", hash = "sha256:0858a54aabc5b459d1aafa8a518ed2081a285087f349fe3e55197989232e2e2d"},
 ]
 six = [
-    {file = "six-1.14.0-py2.py3-none-any.whl", hash = "sha256:8f3cd2e254d8f793e7f3d6d9df77b92252b52637291d0f0da013c76ea2724b6c"},
+    {file = "six-1.15.0-py2.py3-none-any.whl", hash = "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced"},
-    {file = "six-1.14.0.tar.gz", hash = "sha256:236bdbdce46e6e6a3d61a337c0f8b763ca1e8717c03b369e87a7ec7ce1319c0a"},
+    {file = "six-1.15.0.tar.gz", hash = "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259"},
 ]
 soupsieve = [
     {file = "soupsieve-1.9.6-py2.py3-none-any.whl", hash = "sha256:feb1e937fa26a69e08436aad4a9037cd7e1d4c7212909502ba30701247ff8abd"},

website/web/__init__.py

@@ -6,6 +6,7 @@ from io import BytesIO
 import os
 from pathlib import Path
 from datetime import datetime, timedelta
+import json
 
 from flask import Flask, render_template, request, send_file, redirect, url_for, Response, flash
 from flask_bootstrap import Bootstrap  # type: ignore
@@ -46,6 +47,19 @@ time_delta_on_index = lookyloo.get_config('time_delta_on_index')
 logging.basicConfig(level=lookyloo.get_config('loglevel'))
 
 
+# Method to make sizes in bytes human readable
+# Source: https://stackoverflow.com/questions/1094841/reusable-library-to-get-human-readable-version-of-file-size
+def sizeof_fmt(num, suffix='B'):
+    for unit in ['', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi']:
+        if abs(num) < 1024.0:
+            return "%3.1f%s%s" % (num, unit, suffix)
+        num /= 1024.0
+    return "%.1f%s%s" % (num, 'Yi', suffix)
+
+
+app.jinja_env.globals.update(sizeof_fmt=sizeof_fmt)
+
+
 @auth.get_password
 def get_pw(username: str) -> Optional[str]:
     if username in user:
@@ -131,7 +145,7 @@ def hostnode_popup(tree_uuid: str, node_uuid: str):
     if not capture_dir:
         return
     hostnode = lookyloo.get_hostnode_from_tree(capture_dir, node_uuid)
-    table_keys = {
+    keys_response = {
         'js': "/static/javascript.png",
         'exe': "/static/exe.png",
         'css': "/static/css.png",
@@ -142,11 +156,13 @@ def hostnode_popup(tree_uuid: str, node_uuid: str):
         'image': "/static/img.png",
         'unknown_mimetype': "/static/wtf.png",
         'video': "/static/video.png",
-        'request_cookie': "/static/cookie_read.png",
         'response_cookie': "/static/cookie_received.png",
         'redirect': "/static/redirect.png",
         'redirect_to_nothing': "/static/cookie_in_url.png"
     }
+    keys_request = {
+        'request_cookie': "/static/cookie_read.png",
+    }
 
     urls = []
     if lookyloo.sanejs.available:
@@ -169,7 +185,31 @@ def hostnode_popup(tree_uuid: str, node_uuid: str):
                            hostname_uuid=node_uuid,
                            hostname=hostnode.name,
                            urls=urls,
-                           keys=table_keys)
+                           keys_response=keys_response,
+                           keys_request=keys_request)
+
+
+@app.route('/tree/<string:tree_uuid>/url/<string:node_uuid>/posted_data', methods=['GET'])
+def urlnode_post_request(tree_uuid: str, node_uuid: str):
+    capture_dir = lookyloo.lookup_capture_dir(tree_uuid)
+    if not capture_dir:
+        return
+    urlnode = lookyloo.get_urlnode_from_tree(capture_dir, node_uuid)
+    if not urlnode.posted_data:
+        return
+    if isinstance(urlnode.posted_data, (dict, list)):
+        # JSON blob, pretty print.
+        posted = json.dumps(urlnode.posted_data, indent=2)
+    else:
+        posted = urlnode.posted_data
+
+    if isinstance(posted, bytes):
+        to_return = BytesIO(posted)
+    else:
+        to_return = BytesIO(posted.encode())
+    to_return.seek(0)
+    return send_file(to_return, mimetype='text/plain',
+                     as_attachment=True, attachment_filename='posted_data.txt')
 
 
 @app.route('/tree/<string:tree_uuid>/url/<string:node_uuid>', methods=['GET'])

website/web/static/tree.js

@@ -110,165 +110,6 @@ function ProcessChildMessage(message) {
         .style('font-size', '16px');
 };
 
-// What happen when clicking on a domain (load a modal display)
-function hostnode_click(d) {
-    // Move the node to the front (end of the list)
-    var cur_node = d3.select("#node_" + d.data.uuid).moveToFront();
-    // Avoid duplicating overlays
-    cur_node.selectAll('.overlay').remove();
-    // Insert new svg element at this position
-    var overlay_hostname = cur_node.append('g')
-                                .attr('class', 'overlay');
-
-    cur_node.append('line')
-                .attr('id', 'overlay_link')
-                .style("opacity", "0.95")
-                .attr("stroke-width", "2")
-                .style("stroke", "gray");
-
-    var top_margin = 15;
-    var overlay_header_height = 50;
-    var left_margin = 30;
-
-    overlay_hostname
-        .datum({x: 0, y: 0, overlay_uuid: d.data.uuid})
-        .attr('id', 'overlay_' + d.data.uuid)
-        .attr("transform", "translate(" + 10 + "," + 15 + ")")
-        .call(d3.drag().on("drag", function(d, i) {
-            if (typeof d.x === 'undefined') { d.x = 0; }  // Any real JS dev would kill me fo that, right?
-            if (typeof d.y === 'undefined') { d.y = 0; }  // Maybe even twice.
-            d.x += d3.event.dx
-            d.y += d3.event.dy
-            d3.select(this)
-                .attr("transform", "translate(" + d.x + "," + d.y + ")");
-            cur_node.select('#overlay_link')
-                .attr("x2", d.x + left_margin + 10)
-                .attr("y2", d.y + top_margin + 15);
-        }));
-
-    overlay_hostname.append('rect')
-        .attr("rx", 6)
-        .attr("ry", 6)
-        .attr('x', 15)
-        .attr('y', 10)
-        .style("opacity", "0.95")
-        .attr("stroke", "black")
-        .attr('stroke-opacity', "0.8")
-        .attr("stroke-width", "2")
-        .attr("stroke-linecap", "round")
-        .attr("fill", "white");
-
-    // Modal display
-    var url = "/tree/hostname/" + d.data.uuid;
-    d3.json(url, {credentials: 'same-origin'}).then(urls => {
-        overlay_hostname
-            .append('circle')
-            .attr('id', 'overlay_circle_' + d.data.uuid)
-            .attr('height', overlay_header_height)
-            .attr('cx', left_margin + 10)
-            .attr('cy', top_margin + 15)
-            .attr('r', 2);
-
-        overlay_hostname
-            .append('text')
-            .attr('id', 'overlay_close_' + d.data.uuid)
-            .attr('height', overlay_header_height)
-            .attr('x', left_margin + 500)  // Value updated based on the size of the rectangle max: max_overlay_width
-            .attr('y', top_margin + 25)
-            .style("font-size", '30px')
-            .text('\u2716')
-            .attr('cursor', 'pointer')
-            .on("click", () => {
-                    main_svg.selectAll('#overlay_' + d.data.uuid).remove();
-                    cur_node.select('#overlay_link').remove();
-                }
-            );
-
-        overlay_hostname.append('line')
-            .attr('id', 'overlay_separator_header' + d.data.uuid)
-            .style("stroke", "black")
-            .style('stroke-width', "1px")
-            .attr('x1', 20)
-            .attr('y1', overlay_header_height)
-            .attr('x2', 500)
-            .attr('y2', overlay_header_height);
-
-        var url_entries = overlay_hostname.append('svg');
-
-        var interval_entries = 10;
-        urls.forEach((url, index, array) => {
-            var jdata = JSON.parse(url);
-            var url_data = url_entries.append('svg')
-                                      .attr('class', 'url_data');
-            url_data.datum({'data': jdata});
-            url_data.append(d => text_entry(left_margin, top_margin + overlay_header_height + (interval_entries * index), urlnode_click, d));
-            url_data.append(d => icon_list(left_margin + 5, top_margin + 20 + overlay_header_height + (interval_entries * index), d, url_view=true));
-        });
-
-        var cur_url_data_height = 0;
-        url_entries.selectAll('.url_data').each(function(p, j){
-            d3.select(this).attr('y', cur_url_data_height);
-            cur_url_data_height += d3.select(this).node().getBBox().height;
-            var cur_icon_list_len = 0;
-            // set position of icons based of their length
-            d3.select(this).selectAll('.icon').each(function(p, j){
-                d3.select(this).attr('x', cur_icon_list_len);
-                cur_icon_list_len += d3.select(this).node().getBBox().width;
-            });
-        });
-
-        var overlay_bbox = overlay_hostname.node().getBBox()
-        overlay_hostname.append('line')
-            .attr('id', 'overlay_separator_footer' + d.data.uuid)
-            .style("stroke", "black")
-            .style('stroke-width', "1px")
-            .attr('x1', 20)
-            .attr('y1', overlay_bbox.height + 20)
-            .attr('x2', 500)
-            .attr('y2', overlay_bbox.height + 20);
-
-        var overlay_bbox = overlay_hostname.node().getBBox()
-        overlay_hostname
-            .append('text')
-            .attr('id', 'overlay_download_' + d.data.uuid)
-            .attr('height', overlay_header_height - 10)
-            .attr('x', left_margin)
-            .attr('y', overlay_bbox.height + overlay_header_height)
-            .style("font-size", '20px')
-            .text('Download URLs as text')
-            .attr('cursor', 'pointer')
-            .on("click", () => {
-                var url = "/tree/hostname/" + d.data.uuid + '/text';
-                d3.blob(url, {credentials: 'same-origin'}).then(data => {
-                    saveAs(data, 'file.md');
-                });
-            });
-
-        var overlay_bbox = overlay_hostname.node().getBBox();
-        overlay_hostname.select('rect')
-            .attr('width', () => {
-                optimal_size = overlay_bbox.width + left_margin
-                return optimal_size < max_overlay_width ? optimal_size : max_overlay_width;
-            })
-            .attr('height', overlay_bbox.height + overlay_header_height);
-
-        overlay_hostname.select('#overlay_close_' + d.data.uuid)
-            .attr('x', overlay_hostname.select('rect').node().getBBox().width - 20);
-
-        overlay_hostname.select('#overlay_separator_header' + d.data.uuid)
-            .attr('x2', overlay_hostname.select('rect').node().getBBox().width + 10);
-        overlay_hostname.select('#overlay_separator_footer' + d.data.uuid)
-            .attr('x2', overlay_hostname.select('rect').node().getBBox().width + 10);
-
-
-        cur_node.select('#overlay_link')
-                    .attr("x1", 10)
-                    .attr("y1", 0)
-                    .attr("x2", left_margin + 3)
-                    .attr("y2", top_margin + 7);
-    });
-};
-
 function icon(key, icon_path, d, icon_size){
     var iconContent = d3.create("svg") // WARNING: svg is required there, "g" doesn't have getBBox
                         .attr('class', 'icon');
@@ -501,7 +342,6 @@ function update(root, computed_node_width=0) {
 
             // Set Hostname text
             node_data
-              // .append(d => text_entry(15, 5, hostnode_click, d));  // In-tree block
               .append(d => text_entry(15, 5, hostnode_click_popup, d));  // Popup
             // Set list of icons
             node_data

website/web/templates/hostname_popup.html

@@ -18,37 +18,61 @@
   </center>
   <p>Click on the URL to get the content of the response</p>
   <ul class="list-group-flush">
-      {% for url in urls %}
+    {% for url in urls %}
-      <li class="list-group-item">
+    <li class="list-group-item">
-        <a href="{{ url_for('urlnode_details', tree_uuid=tree_uuid, node_uuid=url.uuid) }}">{{ url.name }}</a>
+      <p class="h3">{{ url.name }}</p>
-        <div>
+      <ul class="list-group">
-        {% for key, path in keys.items() %}
+        <li class="list-group-item">
-          {% if url[key] %}
+          <p class="h4">Response</p>
-            <img src="{{ path }}" alt="{{ key }}" width="21" height="21"/>
+          <div>
-          {%endif%}
+          {% for key, path in keys_response.items() %}
-        {% endfor %}
+            {% if url[key] %}
-        </div>
+              <img src="{{ path }}" alt="{{ key }}" width="21" height="21"/>
-        {% if url.body %}
-        <div>
-          Body size: {{ url.body.getbuffer().nbytes}}
-        </div>
-        {% else %}
-          Empty body.
-        {%endif%}
-        {% if url.sane_js_details_to_print %}
-        <div>
-          {% if url.sane_js_details_to_print is string %}
-            {{ url.sane_js_details_to_print }}
-          {% else %}
-            This file is known as part of <b>{{ url.sane_js_details_to_print[0] }}</b>
-            version <b>{{ url.sane_js_details_to_print[1] }}</b>: <b>{{ url.sane_js_details_to_print[2] }}</b>.
-            {% if url.sane_js_details_to_print[3] > 1%}
-            It is also present in <b>{{ url.sane_js_details_to_print[3] -1 }}</b> other libraries.
             {%endif%}
+          {% endfor %}
+          </div>
+          {% if not url.empty_response %}
+          <div>
+            <a href="{{ url_for('urlnode_details', tree_uuid=tree_uuid, node_uuid=url.uuid) }}">
+              Download response body.
+            </a></br>
+            Body size: {{ sizeof_fmt(url.body.getbuffer().nbytes) }}
+          </div>
+          {% else %}
+            Empty body.
           {%endif%}
-        </div>
+          {% if url.sane_js_details_to_print %}
-        {% endif %}
+          <div>
-      </li>
+            {% if url.sane_js_details_to_print is string %}
-      {% endfor %}
+              {{ url.sane_js_details_to_print }}
+            {% else %}
+              This file is known as part of <b>{{ url.sane_js_details_to_print[0] }}</b>
+              version <b>{{ url.sane_js_details_to_print[1] }}</b>: <b>{{ url.sane_js_details_to_print[2] }}</b>.
+              {% if url.sane_js_details_to_print[3] > 1%}
+              It is also present in <b>{{ url.sane_js_details_to_print[3] -1 }}</b> other libraries.
+              {%endif%}
+            {%endif%}
+          </div>
+          {% endif %}
+        </li>
+        <li class="list-group-item">
+          <p class="h4">Request</p>
+          <div>
+          {% for key, path in keys_request.items() %}
+            {% if url[key] %}
+              <img src="{{ path }}" alt="{{ key }}" width="21" height="21"/>
+            {%endif%}
+          {% endfor %}
+          </div>
+          {% if url.posted_data %}
+          <a href="{{ url_for('urlnode_post_request', tree_uuid=tree_uuid, node_uuid=url.uuid) }}">
+              Download posted data
+          </a></br>
+          Posted data size: {{ sizeof_fmt(url.posted_data|length) }}
+          {% endif %}
+        </li>
+      </ul>
+    </li>
+    {% endfor %}
   </ul>
 {% endblock %}