From a8b0464afad9451b0d61cec78c069d3512e15652 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Wed, 11 Dec 2024 17:04:00 +0100 Subject: [PATCH] chg: Improve and normalize hostnode popup --- website/web/__init__.py | 22 ++- website/web/helpers.py | 8 +- website/web/templates/body_hash.html | 20 ++- website/web/templates/hostname_popup.html | 159 +++++++++----------- website/web/templates/macros.html | 118 +++++++-------- website/web/templates/ressources.html | 4 +- website/web/templates/tree_body_hashes.html | 2 +- website/web/templates/tree_hostnames.html | 2 - 8 files changed, 169 insertions(+), 166 deletions(-) diff --git a/website/web/__init__.py b/website/web/__init__.py index 124cc37d..fb7beda3 100644 --- a/website/web/__init__.py +++ b/website/web/__init__.py @@ -279,6 +279,7 @@ def get_icon(icon_id: str) -> Icon | None: app.jinja_env.globals.update(get_icon=get_icon) +app.jinja_env.globals.update(generic_type=mimetype_to_generic) def get_tz_info() -> tuple[str | None, str, set[str]]: @@ -366,8 +367,7 @@ def get_all_body_hashes(capture_uuid: str, /) -> dict[str, dict[str, int | str | continue if node.body_hash not in to_return: total_captures = get_indexing(flask_login.current_user).get_captures_body_hash_count(node.body_hash) - generic_type = mimetype_to_generic(node.mimetype) - to_return[node.body_hash] = {'total_captures': total_captures, 'generic_type': generic_type, 'nodes': []} + to_return[node.body_hash] = {'total_captures': total_captures, 'mimetype': node.mimetype, 'nodes': []} to_return[node.body_hash]['nodes'].append((node, False)) # type: ignore[union-attr] # get embedded retources (if any) - need their type too if 'embedded_ressources' in node.features: @@ -375,8 +375,7 @@ def get_all_body_hashes(capture_uuid: str, /) -> dict[str, dict[str, int | str | for h, blob in blobs: if h not in to_return: total_captures = get_indexing(flask_login.current_user).get_captures_body_hash_count(h) - generic_type = mimetype_to_generic(mimetype) - to_return[h] = {'total_captures': total_captures, 'generic_type': generic_type, 'nodes': []} + to_return[h] = {'total_captures': total_captures, 'mimetype': mimetype, 'nodes': []} to_return[h]['nodes'].append((node, True)) # type: ignore[union-attr] return to_return @@ -548,7 +547,7 @@ def get_hostnode_investigator(capture_uuid: str, /, node_uuid: str) -> tuple[Hos # Index lookup # %%% Full body %%% if freq := get_indexing(flask_login.current_user).get_captures_body_hash_count(url.body_hash): - to_append['body_hash_details'] = {'hash_freq': freq} + to_append['body_hash_freq'] = freq # %%% Embedded ressources %%% if hasattr(url, 'embedded_ressources') and url.embedded_ressources: @@ -1778,7 +1777,18 @@ def favicon_detail(favicon_sha512: str) -> str: @app.route('/body_hashes/', methods=['GET']) def body_hash_details(body_hash: str) -> str: from_popup = True if (request.args.get('from_popup') and request.args.get('from_popup') == 'True') else False - return render_template('body_hash.html', body_hash=body_hash, from_popup=from_popup) + filename = '' + mimetype = '' + b64 = '' + if uuids := get_indexing(flask_login.current_user).get_hash_uuids(body_hash): + # got UUIDs for this hash + capture_uuid, urlnode_uuid = uuids + if ressource := lookyloo.get_ressource(capture_uuid, urlnode_uuid, body_hash): + filename, body, mimetype = ressource + if mimetype_to_generic(mimetype) == 'image': + b64 = base64.b64encode(body.read()).decode() + return render_template('body_hash.html', body_hash=body_hash, from_popup=from_popup, + filename=filename, mimetype=mimetype, b64=b64) @app.route('/urls/', methods=['GET']) diff --git a/website/web/helpers.py b/website/web/helpers.py index 64beeefe..f2b9bd36 100644 --- a/website/web/helpers.py +++ b/website/web/helpers.py @@ -128,8 +128,10 @@ def get_indexing(user: User | None) -> Indexing: return get_indexing_cache(full=bool(user and user.is_authenticated)) -def mimetype_to_generic(mimetype: str) -> str: - if 'javascript' in mimetype or 'ecmascript' in mimetype or mimetype.startswith('js'): +def mimetype_to_generic(mimetype: str | None) -> str: + if not mimetype or mimetype == 'none': + return 'unset_mimetype' + elif 'javascript' in mimetype or 'ecmascript' in mimetype or mimetype.startswith('js'): return 'js' elif (mimetype.startswith('image') or mimetype.startswith('img') @@ -180,7 +182,5 @@ def mimetype_to_generic(mimetype: str) -> str: elif ('application/gzip' in mimetype or 'application/zip' in mimetype): return 'archive' - elif not mimetype or mimetype == 'none': - return 'unset_mimetype' else: return 'unknown_mimetype' diff --git a/website/web/templates/body_hash.html b/website/web/templates/body_hash.html index dcccb1e4..8bb31ef8 100644 --- a/website/web/templates/body_hash.html +++ b/website/web/templates/body_hash.html @@ -17,8 +17,24 @@ {%endif%}
-
{{ body_hash }}
- Download +

+ {% if filename %} + Filename: {{ filename }}
+ {% endif %} + {% if mimetype %} + Mimetype: {{ mimetype }}
+ {% endif %} + Hash:
{{ body_hash }}
+ {% if b64 %} + Preview:
+ + + + {% else %} + Download + {% endif %} +

diff --git a/website/web/templates/hostname_popup.html b/website/web/templates/hostname_popup.html index 15b53b6a..5c2a4834 100644 --- a/website/web/templates/hostname_popup.html +++ b/website/web/templates/hostname_popup.html @@ -2,11 +2,12 @@ {% from "macros.html" import known_content_details %} {% from "macros.html" import ressource_legitimacy_details %} {% from "macros.html" import indexed_cookies %} -{% from "macros.html" import popup_icons_request %} -{% from "macros.html" import popup_icons_response %} +{% from "macros.html" import request_cookies_icon %} +{% from "macros.html" import response_cookies_icon %} +{% from "macros.html" import hash_info%} +{% from "macros.html" import redirect_response %} {% from "macros.html" import shorten_string %} {% from "macros.html" import other_captures_table %} -{% from "macros.html" import get_ressource_button %} {% from "macros.html" import context_form %} {% from "macros.html" import pandora_submit %} @@ -127,7 +128,7 @@
{# Start list of URLs #} -
    +
      {% for url in urls %} {# URL Display #}
    • @@ -156,34 +157,47 @@ {% endif %} {% if url['url_object'].security_details %} -
      TLS certificate details: -
        - {% for k, v in url['url_object'].security_details.items() %} -
      • {{k}}: {{v}}
        • - {% endfor%} -
      +
      +
      +

      + +

      +
      +
      +
        + {% for k, v in url['url_object'].security_details.items() %} +
      • {{k}}: {{v}}
        • + {% endfor%} +
      +
      +
      +
      {% endif %} -
        - + {% if url['url_object'].posted_data or url['cookies_sent'] %}
      • -

        Request

        - {{ popup_icons_request(url['url_object'], tree_uuid) }} +

        Request {{ request_cookies_icon(url['url_object'], tree_uuid) }}

        +
        +
        {% if url['url_object'].posted_data %} +
        Download posted data
        {% if url['url_object'].posted_data is string %} Posted data size: {{ sizeof_fmt(url['url_object'].posted_data|length) }} {% endif %} +
        {% endif %} {% if url['cookies_sent'] %}
        -

        This request contains cookies. - @@ -195,7 +209,9 @@

        {% endif %} +
      • + {% endif %}
    • {# Details of the response #} @@ -206,9 +222,37 @@ - Load time: {{ url['url_object'].time.total_seconds() }}s + {{response_cookies_icon(url['url_object'], tree_uuid)}}

      - {{ popup_icons_response(url['url_object'], tree_uuid) }} +
      + {% if url['url_object'].rendered_html %} +
      + + Download rendered HTML page + ({{ sizeof_fmt(url['url_object'].rendered_html.getbuffer().nbytes)}}) +
      + + Download URLs in rendered HTML page + +
      + {% endif %} + + {{ redirect_response(url['url_object'], tree_uuid) }} + {% if url['url_object'].empty_response %} + Empty HTML body. + {% else %} + {{ hash_info(tree_uuid, url['url_object'].uuid, url['url_object'].mimetype, + url['url_object'].body_hash, url['url_object'].body.getbuffer().nbytes, + url.get('body_hash_freq', 0), has_pandora) }} + {{ ressource_legitimacy_details(url.get('legitimacy')) }} + {{ known_content_details(url.get('known_content')) }} + + {% if enable_context_by_users %} + {{ context_form(tree_uuid, url['url_object'].uuid, + url['url_object'].body_hash, 'hostnode_popup') }} + {% endif %} + {% endif %} {% if url['url_object'].downloaded_filename %}
      @@ -223,65 +267,19 @@
      {% endif%} - {% if url['url_object'].rendered_html %} -
      - - Download rendered HTML page - ({{ sizeof_fmt(url['url_object'].rendered_html.getbuffer().nbytes)}}) -
      - - Download URLs in rendered HTML page - -
      - {% endif %} - -
      - {% if url['url_object'].empty_response %} - Empty HTML body. - {% else %} - {{ ressource_legitimacy_details(url['legitimacy'], url['url_object'].body.getbuffer().nbytes) }} - {% if has_pandora %} - {{ pandora_submit(tree_uuid, url['url_object'].uuid) }} - {% endif %} - {% endif %} -
      - - - {% if url['known_content'] %} - {{ known_content_details(url['known_content']) }} - {% endif %} - - {# Everything we know about the response content #} - {% if url['body_hash_details'] and url['body_hash_details']['hash_freq'] %} -
      - This file can be found {{ url['body_hash_details']['hash_freq'] }} times - across all the captures on this lookyloo instance. - -

      - - Show more information about this response body. - -

      -
      {% if url['url_object'].hhhash %}

      - See other captures with the same HTTP Headers Hash + Show other captures with the same HTTP Headers Hash

      {% endif %} - {% endif %} - {% if enable_context_by_users %} -
      - {{ context_form(tree_uuid, url['url_object'].uuid, - url['url_object'].body_hash, 'hostnode_popup') }} - {% endif %} - +
      {% if url['embedded_ressources'] %} {# Details on embedded resources #} -
      This response contains embedded ressources - @@ -290,28 +288,12 @@
      {% for hash, details in url['embedded_ressources'].items() %}
      - {% if details['known_content'] %} - {{ known_content_details(details['known_content']) }} - {% endif %} - {{ ressource_legitimacy_details(details['legitimacy'], details['body_size']) }} -
      - {% if has_pandora %} - {{ pandora_submit(tree_uuid, url['url_object'].uuid, hash) }} - {% endif %} -
      - This file {% if details['type'] %}({{ details['type'] }}){% endif %} can be found {{ details['hash_freq'] }} times - across all the captures on this lookyloo instance. - {{ get_ressource_button(tree_uuid, url['url_object'].uuid, hash, - 'Download the embedded ressource', - details['type'] and details['type'].startswith('image')) }} -
      + {{hash_info(tree_uuid, url['url_object'].uuid, details['type'], hash, details['body_size'], details['hash_freq'] ) }}
      + {{ ressource_legitimacy_details(details.get('legitimacy')) }} + {{ known_content_details(details.get('known_content')) }} {% if enable_context_by_users %} {{ context_form(tree_uuid, url['url_object'].uuid, hash, 'hostnode_popup') }} {% endif %} - -

      - Show more information about this embedded content. -

      {% endfor %}
      @@ -320,8 +302,8 @@ {% if url['cookies_received'] %}
      -

      This response contains cookies. - @@ -335,6 +317,7 @@

      {% endif %} +
    diff --git a/website/web/templates/macros.html b/website/web/templates/macros.html index 94b263fa..ce5012b1 100644 --- a/website/web/templates/macros.html +++ b/website/web/templates/macros.html @@ -81,21 +81,23 @@ {% macro known_content_details(details) %} -
    -{% if details is string %} - {{ details }} -{% else %} - This file is known as part of {{ details[0] }} - version {{ details[1] }}: {{ details[2] }}. - {% if details[3] > 1%} - It is also present in {{ details[3] -1 }} other libraries. +{% if details %} +
    + {% if details is string %} + {{ details }} + {% else %} + This file is known as part of {{ details[0] }} + version {{ details[1] }}: {{ details[2] }}. + {% if details[3] > 1%} + It is also present in {{ details[3] -1 }} other libraries. + {%endif%} {%endif%} +
    {%endif%} -
    {% endmacro %} {% macro context_form(tree_uuid, urlnode_uuid, hash, callback_str) %} - @@ -150,22 +152,10 @@ {% endmacro %} -{% macro get_ressource_button(capture_uuid, urlnode_uuid, hash, text, can_preview=False) %} -
    - - -{% endmacro %} - -{% macro ressource_legitimacy_details(details, ressource_size) %} +{% macro ressource_legitimacy_details(details) %} {% if details and details[0] == False %} {%endif%} -Body size (in the HTTP response): {{ sizeof_fmt(ressource_size) }} {% if details %} {% if details[0] %} - This file is known legitimate on the following domains: {{ ', '.join(details[1]) }}. @@ -215,17 +205,15 @@ {% endif %} {% endmacro %} -{% macro popup_icons_request(urlnode, tree_uuid) %} -
    +{% macro request_cookies_icon(urlnode, tree_uuid) %} {% if urlnode.request_cookie %} {% set icon_info = get_icon("request_cookie") %} - + {{ icon_info['tooltip'] }} + width="21" height="21"/ + title="Download all the cookies in the request to the server"> {% endif %} -
    {% endmacro %} @@ -241,47 +229,57 @@ {% endmacro %} -{% macro hash_icon(tree_uuid, urlnode_uuid, generic_type, hash) %} -{% if generic_type in ["js", "exe", "css", "font", "html", "json", "image", "video", - "unknown_mimetype", "text", "unset_mimetype", "octet-stream", - "livestream"] %} - {% set icon_info = get_icon(generic_type) %} - - {{ icon_info['tooltip'] }}
    Click to download.' - {% else %} - data-bs-toggle="tooltip" data-bs-placement="bottom" data-bs-html="true" - title="{{icon_info['tooltip']}}
    Click to download." - {% endif %} - /> -     -{%endif%} +{% macro hash_info(tree_uuid, urlnode_uuid, mimetype, hash, ressource_size, + nb_occurrences, has_pandora)%} + +{{ hash_icon(tree_uuid, urlnode_uuid, mimetype, hash) }} + +Body size (in the HTTP response): {{ sizeof_fmt(ressource_size) }} + +{% if nb_occurrences > 0 %} +
    + This file can be found {{ nb_occurrences }} times across all the captures on this lookyloo instance. +

    + + Show more information about this ressource. + +

    +
    +{% endif %} + +{% if has_pandora %} + {{ pandora_submit(tree_uuid, urlnode_uuid, hash) }} +{% endif %} {% endmacro %} - -{% macro popup_icons_response(urlnode, tree_uuid) %} -
    - {% if urlnode.generic_type in ["js", "exe", "css", "font", "html", "json", "image", "video", - "unknown_mimetype", "text", "unset_mimetype", "octet-stream", - "livestream"] %} - {% set icon_info = get_icon(urlnode.generic_type) %} - +{% macro hash_icon(tree_uuid, urlnode_uuid, mimetype, hash) %} +{% if mimetype %} + {% set gt = generic_type(mimetype) %} + {% if gt in ["js", "exe", "css", "font", "html", "json", "image", "video", + "unknown_mimetype", "text", "unset_mimetype", "octet-stream", + "livestream"] %} + {% set icon_info = get_icon(gt) %} + {{ icon_info['tooltip'] }}
    Click to download.' + title='
    Click to download.' {% else %} data-bs-toggle="tooltip" data-bs-placement="bottom" data-bs-html="true" title="{{icon_info['tooltip']}}
    Click to download." {% endif %} />     + Mimetype: {{mimetype}} +
    {%endif%} +{%endif%} +{% endmacro %} + + +{% macro response_cookies_icon(urlnode, tree_uuid) %} {% if urlnode.response_cookie %} {% set icon_info = get_icon("response_cookie") %} {% endif %} +{% endmacro %} -
    - -
    +{% macro redirect_response(urlnode, tree_uuid) %} {% if urlnode["redirect"] %} {% set icon_info = get_icon('redirect') %} {% for child in urlnode.children if child.name == urlnode.redirect_url %} @@ -309,7 +306,6 @@ width="21" height="21"/> {% endfor %} {%endif%} -
    {% endmacro %} {% macro shorten_string(string, cut_length, with_title=False) %} diff --git a/website/web/templates/ressources.html b/website/web/templates/ressources.html index 3e156b96..e7710f48 100644 --- a/website/web/templates/ressources.html +++ b/website/web/templates/ressources.html @@ -2,7 +2,7 @@ {% from 'bootstrap5/utils.html' import render_messages %} {% from 'macros.html' import shorten_string %} -{% from 'macros.html' import get_ressource_button %} +{% from 'macros.html' import hash_icon %} {% from 'macros.html' import context_form %} {% block title %}Ressources{% endblock %} @@ -42,7 +42,7 @@
{{ shorten_string(h, 10) }}
- {{ get_ressource_button(capture_uuid, urlnode_uuid, h, 'Download sample', mimetype and mimetype.startswith('image')) }} + {{ hash_icon(capture_uuid, urlnode_uuid, mimetype, h) }} 		{{ freq }} {{ context['type'] }} - {{ context['details'] }}
diff --git a/website/web/templates/tree_body_hashes.html b/website/web/templates/tree_body_hashes.html index a784c52e..bd67992a 100644 --- a/website/web/templates/tree_body_hashes.html +++ b/website/web/templates/tree_body_hashes.html @@ -14,7 +14,7 @@
{{ info['total_captures'] }} - {{hash_icon(tree_uuid, info['nodes'][0][0].uuid, info['generic_type'], body_hash)}} + {{hash_icon(tree_uuid, info['nodes'][0][0].uuid, info['mimetype'], body_hash)}}
    diff --git a/website/web/templates/tree_hostnames.html b/website/web/templates/tree_hostnames.html index 597ce490..45dadd4a 100644 --- a/website/web/templates/tree_hostnames.html +++ b/website/web/templates/tree_hostnames.html @@ -1,5 +1,3 @@ -{% from "macros.html" import popup_icons_response %} -