mirror of https://github.com/CIRCL/lookyloo
new: Display embedded resources too in the modal
Raphaël Vinot
parent
cddc205d3c
commit
489bb52e12
|
|
@ -1,4 +1,4 @@
|
|||
# This file is automatically @generated by Poetry 1.8.4 and should not be changed by hand.
|
||||
# This file is automatically @generated by Poetry 1.8.5 and should not be changed by hand.
|
||||
|
||||
[[package]]
|
||||
name = "aiobotocore"
|
||||
|
|
@ -1341,13 +1341,13 @@ tornado = ["tornado (>=0.2)"]
|
|||
|
||||
[[package]]
|
||||
name = "har2tree"
|
||||
version = "1.27.3"
|
||||
version = "1.27.4"
|
||||
description = "HTTP Archive (HAR) to ETE Toolkit generator"
|
||||
optional = false
|
||||
python-versions = "<3.14,>=3.9"
|
||||
files = [
|
||||
{file = "har2tree-1.27.3-py3-none-any.whl", hash = "sha256:85474ede9467f316ceabd35b5561ec6d3d42e9f1da98397ff9dd1b70172d0dad"},
|
||||
{file = "har2tree-1.27.3.tar.gz", hash = "sha256:6639b7eb4ab368174e2cf7cb5b36c0376e146aae2859ccaf2352975142d30624"},
|
||||
{file = "har2tree-1.27.4-py3-none-any.whl", hash = "sha256:1fbae6a1dea8dea32e968a895e7cc883c823a05488a3ef4dde5c6065fbf1311f"},
|
||||
{file = "har2tree-1.27.4.tar.gz", hash = "sha256:2e735c11e7d923d3599ccbe8eb5631af82823bf769a24002e34feea8ca1c5ff0"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
|
|
@ -1358,13 +1358,13 @@ filetype = ">=1.2.0,<2.0.0"
|
|||
markupsafe = ">=3.0.2,<4.0.0"
|
||||
numpy = [
|
||||
{version = "<2.1", markers = "python_version < \"3.10\""},
|
||||
{version = ">=2.1,<3.0", markers = "python_version >= \"3.10\""},
|
||||
{version = ">=2.2,<3.0", markers = "python_version >= \"3.10\""},
|
||||
]
|
||||
publicsuffixlist = ">=1.0.2.20241124,<2.0.0.0"
|
||||
publicsuffixlist = ">=1.0.2.20241210,<2.0.0.0"
|
||||
w3lib = ">=2.2.1,<3.0.0"
|
||||
|
||||
[package.extras]
|
||||
docs = ["Sphinx (>=8,<9)", "six (>=1.16.0,<2.0.0)"]
|
||||
docs = ["Sphinx (>=8,<9)", "six (>=1.17.0,<2.0.0)"]
|
||||
|
||||
[[package]]
|
||||
name = "hiredis"
|
||||
|
|
@ -2839,13 +2839,13 @@ files = [
|
|||
|
||||
[[package]]
|
||||
name = "publicsuffixlist"
|
||||
version = "1.0.2.20241207"
|
||||
version = "1.0.2.20241210"
|
||||
description = "publicsuffixlist implement"
|
||||
optional = false
|
||||
python-versions = ">=3.5"
|
||||
files = [
|
||||
{file = "publicsuffixlist-1.0.2.20241207-py2.py3-none-any.whl", hash = "sha256:7213e69d0a2c9d7948b9bc304dbffa17a1450eccd2ba1de30c278a07134f39fd"},
|
||||
{file = "publicsuffixlist-1.0.2.20241207.tar.gz", hash = "sha256:2b6d70074b00886d3098e7ed5f8eba8c3d1f3c2429eb8ecaf98362595496de04"},
|
||||
{file = "publicsuffixlist-1.0.2.20241210-py2.py3-none-any.whl", hash = "sha256:7643f6089900a143f32b58687f95d0ebcf3d5868cd6a551235897439f1107007"},
|
||||
{file = "publicsuffixlist-1.0.2.20241210.tar.gz", hash = "sha256:4863508968378c572557b365d025d33af6b2bf935a1c34860dfb54331859bdca"},
|
||||
]
|
||||
|
||||
[package.extras]
|
||||
|
|
@ -4694,4 +4694,4 @@ type = ["pytest-mypy"]
|
|||
[metadata]
|
||||
lock-version = "2.0"
|
||||
python-versions = ">=3.9,<3.14"
|
||||
content-hash = "56281ebde1cc394ab04458d4982d6ddd17b7bcbb5c58271eb06da85e0a10474e"
|
||||
content-hash = "ec29c06fccdf5d83fce9ba51fcf49044c9f45e93cf67f2559302c047ea5bc6c3"
|
||||
|
|
|
|||
|
|
@ -63,7 +63,7 @@ pyhashlookup = "^1.2.5"
|
|||
lief = "^0.15"
|
||||
ua-parser = {extras = ["regex"], version = "^1.0.0"}
|
||||
Flask-Login = "^0.6.3"
|
||||
har2tree = "^1.27.3"
|
||||
har2tree = "^1.27.4"
|
||||
passivetotal = "^2.5.9"
|
||||
werkzeug = "^3.1.3"
|
||||
filetype = "^1.2.0"
|
||||
|
|
@ -71,7 +71,7 @@ pypandora = "^1.9.1"
|
|||
lacuscore = "^1.12.6"
|
||||
pylacus = "^1.12.0"
|
||||
pyipasnhistory = "^2.1.3"
|
||||
publicsuffixlist = "^1.0.2.20241207"
|
||||
publicsuffixlist = "^1.0.2.20241210"
|
||||
pyfaup = "^1.2"
|
||||
chardet = "^5.2.0"
|
||||
pysecuritytxt = "^1.3.2"
|
||||
|
|
|
|||
|
|
@ -53,7 +53,8 @@ from zoneinfo import available_timezones
|
|||
from .genericapi import api as generic_api
|
||||
from .helpers import (User, build_users_table, get_secret_key,
|
||||
load_user_from_request, src_request_ip, sri_load,
|
||||
get_lookyloo_instance, get_indexing, build_keys_table)
|
||||
get_lookyloo_instance, get_indexing, build_keys_table,
|
||||
mimetype_to_generic)
|
||||
from .proxied import ReverseProxied
|
||||
|
||||
logging.config.dictConfig(get_config('logging'))
|
||||
|
|
@ -357,15 +358,26 @@ def _get_body_hash_investigator(body_hash: str, offset: int | None=None, limit:
|
|||
return total, captures
|
||||
|
||||
|
||||
def get_all_body_hashes(capture_uuid: str, /) -> dict[str, dict[str, URLNode | int]]:
|
||||
def get_all_body_hashes(capture_uuid: str, /) -> dict[str, dict[str, int | str | list[tuple[URLNode, bool]]]]:
|
||||
ct = lookyloo.get_crawled_tree(capture_uuid)
|
||||
to_return: dict[str, dict[str, URLNode | int]] = defaultdict()
|
||||
to_return: dict[str, dict[str, int | str | list[tuple[URLNode, bool]]]] = defaultdict()
|
||||
for node in ct.root_hartree.url_tree.traverse():
|
||||
if node.empty_response or node.body_hash in to_return:
|
||||
# If we have the same hash more than once, skip
|
||||
if node.empty_response:
|
||||
continue
|
||||
total_captures = get_indexing(flask_login.current_user).get_captures_body_hash_count(node.body_hash)
|
||||
to_return[node.body_hash] = {'node': node, 'total_captures': total_captures}
|
||||
if node.body_hash not in to_return:
|
||||
total_captures = get_indexing(flask_login.current_user).get_captures_body_hash_count(node.body_hash)
|
||||
generic_type = mimetype_to_generic(node.mimetype)
|
||||
to_return[node.body_hash] = {'total_captures': total_captures, 'generic_type': generic_type, 'nodes': []}
|
||||
to_return[node.body_hash]['nodes'].append((node, False)) # type: ignore[union-attr]
|
||||
# get embedded retources (if any) - need their type too
|
||||
if 'embedded_ressources' in node.features:
|
||||
for mimetype, blobs in node.embedded_ressources.items():
|
||||
for h, blob in blobs:
|
||||
if h not in to_return:
|
||||
total_captures = get_indexing(flask_login.current_user).get_captures_body_hash_count(h)
|
||||
generic_type = mimetype_to_generic(mimetype)
|
||||
to_return[h] = {'total_captures': total_captures, 'generic_type': generic_type, 'nodes': []}
|
||||
to_return[h]['nodes'].append((node, True)) # type: ignore[union-attr]
|
||||
return to_return
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -126,3 +126,61 @@ def get_indexing(user: User | None) -> Indexing:
|
|||
It is only accessible to the admin user.
|
||||
'''
|
||||
return get_indexing_cache(full=bool(user and user.is_authenticated))
|
||||
|
||||
|
||||
def mimetype_to_generic(mimetype: str) -> str:
|
||||
if 'javascript' in mimetype or 'ecmascript' in mimetype or mimetype.startswith('js'):
|
||||
return 'js'
|
||||
elif (mimetype.startswith('image')
|
||||
or mimetype.startswith('img')
|
||||
or 'webp' in mimetype):
|
||||
return 'image'
|
||||
elif mimetype.startswith('text/css'):
|
||||
return 'css'
|
||||
elif 'json' in mimetype:
|
||||
return 'json'
|
||||
elif 'html' in mimetype:
|
||||
return 'html'
|
||||
elif ('font' in mimetype
|
||||
or 'woff' in mimetype
|
||||
or 'opentype' in mimetype):
|
||||
return 'font'
|
||||
elif ('octet-stream' in mimetype
|
||||
or 'application/x-protobuf' in mimetype
|
||||
or 'application/pkix-cert' in mimetype
|
||||
or 'application/x-123' in mimetype
|
||||
or 'application/x-binary' in mimetype
|
||||
or 'application/x-msdownload' in mimetype
|
||||
or 'application/x-thrift' in mimetype
|
||||
or 'application/x-troff-man' in mimetype
|
||||
or 'application/x-typekit-augmentation' in mimetype
|
||||
or 'application/grpc-web' in mimetype
|
||||
or 'model/gltf-binary' in mimetype
|
||||
or 'model/obj' in mimetype
|
||||
or 'application/wasm' in mimetype):
|
||||
return 'octet-stream'
|
||||
elif ('text' in mimetype or 'xml' in mimetype
|
||||
or mimetype.startswith('multipart')
|
||||
or mimetype.startswith('message')
|
||||
or 'application/x-www-form-urlencoded' in mimetype
|
||||
or 'application/vnd.oasis.opendocument.formula-template' in mimetype):
|
||||
return 'text'
|
||||
elif 'video' in mimetype:
|
||||
return 'video'
|
||||
elif ('audio' in mimetype or 'ogg' in mimetype):
|
||||
return 'audio'
|
||||
elif ('mpegurl' in mimetype
|
||||
or 'application/vnd.yt-ump' in mimetype):
|
||||
return 'livestream'
|
||||
elif ('application/x-shockwave-flash' in mimetype
|
||||
or 'application/x-shockware-flash' in mimetype): # Yes, shockwaRe
|
||||
return 'flash'
|
||||
elif 'application/pdf' in mimetype:
|
||||
return 'pdf'
|
||||
elif ('application/gzip' in mimetype
|
||||
or 'application/zip' in mimetype):
|
||||
return 'archive'
|
||||
elif not mimetype or mimetype == 'none':
|
||||
return 'unset_mimetype'
|
||||
else:
|
||||
return 'unknown_mimetype'
|
||||
|
|
|
|||
|
|
@ -241,6 +241,27 @@
|
|||
</div>
|
||||
{% endmacro %}
|
||||
|
||||
{% macro hash_icon(tree_uuid, urlnode_uuid, generic_type, hash) %}
|
||||
{% if generic_type in ["js", "exe", "css", "font", "html", "json", "image", "video",
|
||||
"unknown_mimetype", "text", "unset_mimetype", "octet-stream",
|
||||
"livestream"] %}
|
||||
{% set icon_info = get_icon(generic_type) %}
|
||||
<a href="{{ url_for('get_ressource', tree_uuid=tree_uuid, node_uuid=urlnode_uuid) }}">
|
||||
<img src="{{ url_for('static', filename=icon_info['icon']) }}" alt="{{ icon_info['tooltip'] }}"
|
||||
width="21" height="21"
|
||||
{% if generic_type == "image" %}
|
||||
data-bs-toggle="tooltip" data-bs-placement="bottom" data-bs-html="true"
|
||||
title='<img class="ressource_preview" src="{{ url_for('get_ressource_preview', tree_uuid=tree_uuid, node_uuid=urlnode_uuid, h_ressource=hash) }}"/> <br>Click to download.'
|
||||
{% else %}
|
||||
data-bs-toggle="tooltip" data-bs-placement="bottom" data-bs-html="true"
|
||||
title="{{icon_info['tooltip']}} <br/>Click to download."
|
||||
{% endif %}
|
||||
/>
|
||||
</a>
|
||||
{%endif%}
|
||||
|
||||
{% endmacro %}
|
||||
|
||||
|
||||
{% macro popup_icons_response(urlnode, tree_uuid) %}
|
||||
<div>
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
{% from "macros.html" import popup_icons_response %}
|
||||
{% from "macros.html" import hash_icon %}
|
||||
|
||||
<table id="bodyHashesTable" class="table table-striped" style="width:100%">
|
||||
<thead>
|
||||
|
|
@ -13,12 +13,18 @@
|
|||
{% for body_hash, info in body_hashes.items() %}
|
||||
<tr>
|
||||
<td>{{ info['total_captures'] }}</td>
|
||||
<td data-order="{{info['node'].generic_type}}">
|
||||
{{popup_icons_response(info['node'], tree_uuid)}}
|
||||
<td data-order="{{info['generic_type']}}">
|
||||
{{hash_icon(tree_uuid, info['nodes'][0][0].uuid, info['generic_type'], body_hash)}}
|
||||
</td>
|
||||
<td>
|
||||
<p class="text-break">{{ info['node'].name }}</p>
|
||||
<button type="button" class="btn btn-link openNewTab" data-capture="{{tree_uuid}}" data-hostnode="{{info['node'].uuid}}">Show on tree</button>
|
||||
<ul>
|
||||
{% for node, embedded in info['nodes'] %}
|
||||
<li>
|
||||
<span class="text-break">{{ node.name }} {% if embedded %}<b>(embedded)</b>{%endif%}</span>
|
||||
<a href="#" role="button" class="btn btn-link openNewTab"
|
||||
data-capture="{{tree_uuid}}" data-hostnode="{{node.uuid}}">Show on tree</a>
|
||||
</li>
|
||||
{% endfor %}
|
||||
</td>
|
||||
<td>
|
||||
<span class="d-inline-block text-truncate" style="max-width: 200px;">
|
||||
|
|
|
|||
Loading…
Reference in New Issue