diff --git a/lookyloo/modules.py b/lookyloo/modules.py index 5c33388..4efa976 100644 --- a/lookyloo/modules.py +++ b/lookyloo/modules.py @@ -39,6 +39,13 @@ class SaneJavaScript(): "bd9ab35dde3a5242b04c159187732e13b0a6da50ddcff7015dfb78cdd68743e191eaf5cddedd49bef7d2d5a642c217272a40e5ba603fe24ca676a53f8c417c5d": "This is a 1*1 pixel GIF", "d052ecec2839340876eb57247cfc2e777dd7f2e868dc37cd3f3f740c8deb94917a0c9f2a4fc8229987a0b91b04726de2d1e9f6bcbe3f9bef0e4b7e0d7f65ea12": "This is a 1*1 pixel GIF", "8717074ddf1198d27b9918132a550cb4ba343794cc3d304a793f9d78c9ff6c4929927b414141d40b6f6ad296725520f4c63edeb660ed530267766c2ab74ee4a9": "This is a 1*1 pixel GIF", + "6834f1548f26b94357fcc3312a3491e8c87080a84f678f990beb2c745899a01e239964521e64a534d7d5554222f728af966ec6ec8291bc64d2005861bcfd78ec": "This is a 1*1 pixel GIF", + "3be8176915593e79bc280d08984a16c29c495bc53be9b439276094b8dcd3764a3c72a046106a06b958e08e67451fe02743175c621a1faa261fe7a9691cc77141": "This is a 1*1 pixel GIF", + "826225fc21717d8861a05b9d2f959539aad2d2b131b2afed75d88fbca535e1b0d5a0da8ac69713a0876a0d467848a37a0a7f926aeafad8cf28201382d16466ab": "This is a 1*1 pixel GIF", + "202612457d9042fe853daab3ddcc1f0f960c5ffdbe8462fa435713e4d1d85ff0c3f197daf8dba15bda9f5266d7e1f9ecaeee045cbc156a4892d2f931fe6fa1bb": "This is a 1*1 pixel GIF", + "b82c6aa1ae927ade5fadbbab478cfaef26d21c1ac441f48e69cfc04cdb779b1e46d7668b4368b933213276068e52f9060228907720492a70fd9bc897191ee77c": "This is a 1*1 pixel GIF", + "763de1053a56a94eef4f72044adb2aa370b98ffa6e0add0b1cead7ee27da519e223921c681ae1db3311273f45d0dd3dc022d102d42ce210c90cb3e761b178438": "This is a 1*1 pixel GIF", + # "": "This is a 1*1 pixel GIF", "f1c33e72643ce366fd578e3b5d393799e8c9ea27b180987826af43b4fc00b65a4eaae5e6426a23448956fee99e3108c6a86f32fb4896c156e24af0571a11c498": "This is a 1*1 pixel PNG", "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e": "This is an empty file" } diff --git a/poetry.lock b/poetry.lock index 195f8b7..00f3f1c 100644 --- a/poetry.lock +++ b/poetry.lock @@ -306,7 +306,7 @@ publicsuffix2 = "^2.20191221" six = "^1.14.0" [package.source] -reference = "38fe9bd74a0e8c92cd17d79981196c69d329b3b8" +reference = "d65554d3a733790431aedb0d4209907471978421" type = "git" url = "https://github.com/viper-framework/har2tree.git" [[package]] @@ -899,7 +899,7 @@ description = "Python 2 and 3 compatibility utilities" name = "six" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" -version = "1.14.0" +version = "1.15.0" [[package]] category = "main" @@ -1539,8 +1539,8 @@ service-identity = [ {file = "service_identity-18.1.0.tar.gz", hash = "sha256:0858a54aabc5b459d1aafa8a518ed2081a285087f349fe3e55197989232e2e2d"}, ] six = [ - {file = "six-1.14.0-py2.py3-none-any.whl", hash = "sha256:8f3cd2e254d8f793e7f3d6d9df77b92252b52637291d0f0da013c76ea2724b6c"}, - {file = "six-1.14.0.tar.gz", hash = "sha256:236bdbdce46e6e6a3d61a337c0f8b763ca1e8717c03b369e87a7ec7ce1319c0a"}, + {file = "six-1.15.0-py2.py3-none-any.whl", hash = "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced"}, + {file = "six-1.15.0.tar.gz", hash = "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259"}, ] soupsieve = [ {file = "soupsieve-1.9.6-py2.py3-none-any.whl", hash = "sha256:feb1e937fa26a69e08436aad4a9037cd7e1d4c7212909502ba30701247ff8abd"}, diff --git a/website/web/__init__.py b/website/web/__init__.py index 0575b56..f3e1d0d 100644 --- a/website/web/__init__.py +++ b/website/web/__init__.py @@ -6,6 +6,7 @@ from io import BytesIO import os from pathlib import Path from datetime import datetime, timedelta +import json from flask import Flask, render_template, request, send_file, redirect, url_for, Response, flash from flask_bootstrap import Bootstrap # type: ignore @@ -46,6 +47,19 @@ time_delta_on_index = lookyloo.get_config('time_delta_on_index') logging.basicConfig(level=lookyloo.get_config('loglevel')) +# Method to make sizes in bytes human readable +# Source: https://stackoverflow.com/questions/1094841/reusable-library-to-get-human-readable-version-of-file-size +def sizeof_fmt(num, suffix='B'): + for unit in ['', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi']: + if abs(num) < 1024.0: + return "%3.1f%s%s" % (num, unit, suffix) + num /= 1024.0 + return "%.1f%s%s" % (num, 'Yi', suffix) + + +app.jinja_env.globals.update(sizeof_fmt=sizeof_fmt) + + @auth.get_password def get_pw(username: str) -> Optional[str]: if username in user: @@ -131,7 +145,7 @@ def hostnode_popup(tree_uuid: str, node_uuid: str): if not capture_dir: return hostnode = lookyloo.get_hostnode_from_tree(capture_dir, node_uuid) - table_keys = { + keys_response = { 'js': "/static/javascript.png", 'exe': "/static/exe.png", 'css': "/static/css.png", @@ -142,11 +156,13 @@ def hostnode_popup(tree_uuid: str, node_uuid: str): 'image': "/static/img.png", 'unknown_mimetype': "/static/wtf.png", 'video': "/static/video.png", - 'request_cookie': "/static/cookie_read.png", 'response_cookie': "/static/cookie_received.png", 'redirect': "/static/redirect.png", 'redirect_to_nothing': "/static/cookie_in_url.png" } + keys_request = { + 'request_cookie': "/static/cookie_read.png", + } urls = [] if lookyloo.sanejs.available: @@ -169,7 +185,31 @@ def hostnode_popup(tree_uuid: str, node_uuid: str): hostname_uuid=node_uuid, hostname=hostnode.name, urls=urls, - keys=table_keys) + keys_response=keys_response, + keys_request=keys_request) + + +@app.route('/tree//url//posted_data', methods=['GET']) +def urlnode_post_request(tree_uuid: str, node_uuid: str): + capture_dir = lookyloo.lookup_capture_dir(tree_uuid) + if not capture_dir: + return + urlnode = lookyloo.get_urlnode_from_tree(capture_dir, node_uuid) + if not urlnode.posted_data: + return + if isinstance(urlnode.posted_data, (dict, list)): + # JSON blob, pretty print. + posted = json.dumps(urlnode.posted_data, indent=2) + else: + posted = urlnode.posted_data + + if isinstance(posted, bytes): + to_return = BytesIO(posted) + else: + to_return = BytesIO(posted.encode()) + to_return.seek(0) + return send_file(to_return, mimetype='text/plain', + as_attachment=True, attachment_filename='posted_data.txt') @app.route('/tree//url/', methods=['GET']) diff --git a/website/web/static/tree.js b/website/web/static/tree.js index ed9978b..d237c93 100644 --- a/website/web/static/tree.js +++ b/website/web/static/tree.js @@ -110,165 +110,6 @@ function ProcessChildMessage(message) { .style('font-size', '16px'); }; -// What happen when clicking on a domain (load a modal display) -function hostnode_click(d) { - // Move the node to the front (end of the list) - var cur_node = d3.select("#node_" + d.data.uuid).moveToFront(); - // Avoid duplicating overlays - cur_node.selectAll('.overlay').remove(); - // Insert new svg element at this position - var overlay_hostname = cur_node.append('g') - .attr('class', 'overlay'); - - cur_node.append('line') - .attr('id', 'overlay_link') - .style("opacity", "0.95") - .attr("stroke-width", "2") - .style("stroke", "gray"); - - var top_margin = 15; - var overlay_header_height = 50; - var left_margin = 30; - - overlay_hostname - .datum({x: 0, y: 0, overlay_uuid: d.data.uuid}) - .attr('id', 'overlay_' + d.data.uuid) - .attr("transform", "translate(" + 10 + "," + 15 + ")") - .call(d3.drag().on("drag", function(d, i) { - if (typeof d.x === 'undefined') { d.x = 0; } // Any real JS dev would kill me fo that, right? - if (typeof d.y === 'undefined') { d.y = 0; } // Maybe even twice. - d.x += d3.event.dx - d.y += d3.event.dy - d3.select(this) - .attr("transform", "translate(" + d.x + "," + d.y + ")"); - cur_node.select('#overlay_link') - .attr("x2", d.x + left_margin + 10) - .attr("y2", d.y + top_margin + 15); - })); - - overlay_hostname.append('rect') - .attr("rx", 6) - .attr("ry", 6) - .attr('x', 15) - .attr('y', 10) - .style("opacity", "0.95") - .attr("stroke", "black") - .attr('stroke-opacity', "0.8") - .attr("stroke-width", "2") - .attr("stroke-linecap", "round") - .attr("fill", "white"); - - // Modal display - var url = "/tree/hostname/" + d.data.uuid; - d3.json(url, {credentials: 'same-origin'}).then(urls => { - overlay_hostname - .append('circle') - .attr('id', 'overlay_circle_' + d.data.uuid) - .attr('height', overlay_header_height) - .attr('cx', left_margin + 10) - .attr('cy', top_margin + 15) - .attr('r', 2); - - overlay_hostname - .append('text') - .attr('id', 'overlay_close_' + d.data.uuid) - .attr('height', overlay_header_height) - .attr('x', left_margin + 500) // Value updated based on the size of the rectangle max: max_overlay_width - .attr('y', top_margin + 25) - .style("font-size", '30px') - .text('\u2716') - .attr('cursor', 'pointer') - .on("click", () => { - main_svg.selectAll('#overlay_' + d.data.uuid).remove(); - cur_node.select('#overlay_link').remove(); - } - ); - - overlay_hostname.append('line') - .attr('id', 'overlay_separator_header' + d.data.uuid) - .style("stroke", "black") - .style('stroke-width', "1px") - .attr('x1', 20) - .attr('y1', overlay_header_height) - .attr('x2', 500) - .attr('y2', overlay_header_height); - - var url_entries = overlay_hostname.append('svg'); - - var interval_entries = 10; - urls.forEach((url, index, array) => { - var jdata = JSON.parse(url); - var url_data = url_entries.append('svg') - .attr('class', 'url_data'); - url_data.datum({'data': jdata}); - url_data.append(d => text_entry(left_margin, top_margin + overlay_header_height + (interval_entries * index), urlnode_click, d)); - url_data.append(d => icon_list(left_margin + 5, top_margin + 20 + overlay_header_height + (interval_entries * index), d, url_view=true)); - }); - - var cur_url_data_height = 0; - url_entries.selectAll('.url_data').each(function(p, j){ - d3.select(this).attr('y', cur_url_data_height); - cur_url_data_height += d3.select(this).node().getBBox().height; - var cur_icon_list_len = 0; - // set position of icons based of their length - d3.select(this).selectAll('.icon').each(function(p, j){ - d3.select(this).attr('x', cur_icon_list_len); - cur_icon_list_len += d3.select(this).node().getBBox().width; - }); - }); - - var overlay_bbox = overlay_hostname.node().getBBox() - overlay_hostname.append('line') - .attr('id', 'overlay_separator_footer' + d.data.uuid) - .style("stroke", "black") - .style('stroke-width', "1px") - .attr('x1', 20) - .attr('y1', overlay_bbox.height + 20) - .attr('x2', 500) - .attr('y2', overlay_bbox.height + 20); - - var overlay_bbox = overlay_hostname.node().getBBox() - overlay_hostname - .append('text') - .attr('id', 'overlay_download_' + d.data.uuid) - .attr('height', overlay_header_height - 10) - .attr('x', left_margin) - .attr('y', overlay_bbox.height + overlay_header_height) - .style("font-size", '20px') - .text('Download URLs as text') - .attr('cursor', 'pointer') - .on("click", () => { - var url = "/tree/hostname/" + d.data.uuid + '/text'; - d3.blob(url, {credentials: 'same-origin'}).then(data => { - saveAs(data, 'file.md'); - }); - }); - - var overlay_bbox = overlay_hostname.node().getBBox(); - overlay_hostname.select('rect') - .attr('width', () => { - optimal_size = overlay_bbox.width + left_margin - return optimal_size < max_overlay_width ? optimal_size : max_overlay_width; - }) - .attr('height', overlay_bbox.height + overlay_header_height); - - overlay_hostname.select('#overlay_close_' + d.data.uuid) - .attr('x', overlay_hostname.select('rect').node().getBBox().width - 20); - - overlay_hostname.select('#overlay_separator_header' + d.data.uuid) - .attr('x2', overlay_hostname.select('rect').node().getBBox().width + 10); - overlay_hostname.select('#overlay_separator_footer' + d.data.uuid) - .attr('x2', overlay_hostname.select('rect').node().getBBox().width + 10); - - - cur_node.select('#overlay_link') - .attr("x1", 10) - .attr("y1", 0) - .attr("x2", left_margin + 3) - .attr("y2", top_margin + 7); - }); -}; - function icon(key, icon_path, d, icon_size){ var iconContent = d3.create("svg") // WARNING: svg is required there, "g" doesn't have getBBox .attr('class', 'icon'); @@ -501,7 +342,6 @@ function update(root, computed_node_width=0) { // Set Hostname text node_data - // .append(d => text_entry(15, 5, hostnode_click, d)); // In-tree block .append(d => text_entry(15, 5, hostnode_click_popup, d)); // Popup // Set list of icons node_data diff --git a/website/web/templates/hostname_popup.html b/website/web/templates/hostname_popup.html index d82702e..0972b23 100644 --- a/website/web/templates/hostname_popup.html +++ b/website/web/templates/hostname_popup.html @@ -18,37 +18,61 @@

Click on the URL to get the content of the response

    - {% for url in urls %} -
  • - {{ url.name }} -
    - {% for key, path in keys.items() %} - {% if url[key] %} - {{ key }} - {%endif%} - {% endfor %} -
    - {% if url.body %} -
    - Body size: {{ url.body.getbuffer().nbytes}} -
    - {% else %} - Empty body. - {%endif%} - {% if url.sane_js_details_to_print %} -
    - {% if url.sane_js_details_to_print is string %} - {{ url.sane_js_details_to_print }} - {% else %} - This file is known as part of {{ url.sane_js_details_to_print[0] }} - version {{ url.sane_js_details_to_print[1] }}: {{ url.sane_js_details_to_print[2] }}. - {% if url.sane_js_details_to_print[3] > 1%} - It is also present in {{ url.sane_js_details_to_print[3] -1 }} other libraries. + {% for url in urls %} +
  • +

    {{ url.name }}

    +
      +
    • +

      Response

      +
      + {% for key, path in keys_response.items() %} + {% if url[key] %} + {{ key }} {%endif%} + {% endfor %} +
      + {% if not url.empty_response %} +
      + + Download response body. +
      + Body size: {{ sizeof_fmt(url.body.getbuffer().nbytes) }} +
      + {% else %} + Empty body. {%endif%} - - {% endif %} -
      • - {% endfor %} + {% if url.sane_js_details_to_print %} +
      + {% if url.sane_js_details_to_print is string %} + {{ url.sane_js_details_to_print }} + {% else %} + This file is known as part of {{ url.sane_js_details_to_print[0] }} + version {{ url.sane_js_details_to_print[1] }}: {{ url.sane_js_details_to_print[2] }}. + {% if url.sane_js_details_to_print[3] > 1%} + It is also present in {{ url.sane_js_details_to_print[3] -1 }} other libraries. + {%endif%} + {%endif%} +
      + {% endif %} + +
    • +

      Request

      +
      + {% for key, path in keys_request.items() %} + {% if url[key] %} + {{ key }} + {%endif%} + {% endfor %} +
      + {% if url.posted_data %} + + Download posted data +
      + Posted data size: {{ sizeof_fmt(url.posted_data|length) }} + {% endif %} +
      • +
    +
    • + {% endfor %}
{% endblock %}