new: Initial listing of all resources loaded on the capture

pull/497/head
Raphaël Vinot 2022-08-01 17:51:43 +02:00
parent 0c81f6b218
commit fdfa2e25cb
10 changed files with 1543 additions and 228 deletions

View File

@ -574,6 +574,18 @@ class Lookyloo():
break
return details, body_content
def get_all_body_hashes(self, capture_uuid: str, /) -> Dict[str, Dict[str, Union[URLNode, int]]]:
ct = self.get_crawled_tree(capture_uuid)
to_return: Dict[str, Dict[str, Union[URLNode, int]]] = defaultdict()
for node in ct.root_hartree.url_tree.traverse():
if node.empty_response or node.body_hash in to_return:
# If we have the same hash more than once, skip
continue
total_captures, details = self.indexing.get_body_hash_captures(node.body_hash, limit=-1)
# Note for future: mayeb get url, capture title, something better than just the hash to show to the user
to_return[node.body_hash] = {'node': node, 'total_captures': total_captures}
return to_return
def get_latest_url_capture(self, url: str, /) -> Optional[CaptureCache]:
'''Get the most recent capture with this URL'''
captures = self.sorted_capture_cache(self.indexing.get_captures_url(url))

1511
poetry.lock generated

File diff suppressed because it is too large Load Diff

View File

@ -47,7 +47,7 @@ defang = "^0.5.3"
vt-py = "^0.14.0"
pyeupi = "^1.1"
pysanejs = "^2.0.1"
pylookyloo = "^1.13.0"
pylookyloo = "^1.14.1"
dnspython = "^2.2.1"
pytaxonomies = "^1.5.0"
pymisp = {version = "^2.4.159", extras = ["url"]}
@ -63,8 +63,8 @@ pyhashlookup = "^1.2.0"
lief = "^0.12.1"
ua-parser = "^0.15.0"
Flask-Login = "^0.6.2"
har2tree = "^1.13.2"
playwrightcapture = "^1.13.4"
har2tree = "^1.14.1"
playwrightcapture = "^1.14.0"
passivetotal = "^2.5.9"
werkzeug = "2.1.2"
filetype = "^1.1.0"
@ -75,14 +75,14 @@ misp = ['python-magic', 'pydeep2']
[tool.poetry.dev-dependencies]
mypy = "^0.971"
ipython = "^8.4.0"
types-redis = "^4.3.11"
types-requests = "^2.28.5"
types-redis = "^4.3.14"
types-requests = "^2.28.8"
types-Flask = "^1.1.6"
types-pkg-resources = "^0.1.3"
types-Deprecated = "^1.2.9"
types-python-dateutil = "^2.8.19"
types-beautifulsoup4 = "^4.11.4"
types-setuptools = "^63.2.2"
types-setuptools = "^63.4.1"
types-Pillow = "^9.2.1"
[build-system]

View File

@ -9,7 +9,7 @@ import time
import filetype
from datetime import date, datetime, timedelta, timezone
from io import BytesIO, StringIO
from typing import Any, Dict, List, Optional, Union
from typing import Any, Dict, List, Optional, Union, TypedDict
from urllib.parse import quote_plus, unquote_plus, urlparse
import flask_login # type: ignore
@ -161,6 +161,38 @@ def get_sri(directory: str, filename: str) -> str:
app.jinja_env.globals.update(get_sri=get_sri)
class Icon(TypedDict):
icon: str
tooltip: str
def get_icon(icon_id: str) -> Optional[Icon]:
available_icons: Dict[str, Icon] = {
'js': {'icon': "javascript.png", 'tooltip': 'The content of the response is a javascript'},
'exe': {'icon': "exe.png", 'tooltip': 'The content of the response is an executable'},
'css': {'icon': "css.png", 'tooltip': 'The content of the response is a CSS'},
'font': {'icon': "font.png", 'tooltip': 'The content of the response is a font'},
'html': {'icon': "html.png", 'tooltip': 'The content of the response is a HTML document'},
'json': {'icon': "json.png", 'tooltip': 'The content of the response is a Json'},
'text': {'icon': "json.png", 'tooltip': 'The content of the response is a text'}, # FIXME: Need new icon
'iframe': {'icon': "ifr.png", 'tooltip': 'This content is loaded from an Iframe'},
'image': {'icon': "img.png", 'tooltip': 'The content of the response is an image'},
'unset_mimetype': {'icon': "wtf.png", 'tooltip': 'The type of content of the response is not set'},
'octet-stream': {'icon': "wtf.png", 'tooltip': 'The type of content of the response is a binary blob'},
'unknown_mimetype': {'icon': "wtf.png", 'tooltip': 'The type of content of the response is of an unknown type'},
'video': {'icon': "video.png", 'tooltip': 'The content of the response is a video'},
'livestream': {'icon': "video.png", 'tooltip': 'The content of the response is a livestream'},
'response_cookie': {'icon': "cookie_received.png", 'tooltip': 'There are cookies in the response'},
'request_cookie': {'icon': "cookie_read.png", 'tooltip': 'There are cookies in the request'},
'redirect': {'icon': "redirect.png", 'tooltip': 'The request is redirected'},
'redirect_to_nothing': {'icon': "cookie_in_url.png", 'tooltip': 'The request is redirected to an URL we do not have in the capture'}
}
return available_icons.get(icon_id)
app.jinja_env.globals.update(get_icon=get_icon)
# ##### Generic/configuration methods #####
@app.after_request
@ -202,30 +234,6 @@ def urls_hostnode(tree_uuid: str, node_uuid: str):
@app.route('/tree/<string:tree_uuid>/host/<string:node_uuid>', methods=['GET'])
def hostnode_popup(tree_uuid: str, node_uuid: str):
keys_response = {
'js': {'icon': "javascript.png", 'tooltip': 'The content of the response is a javascript'},
'exe': {'icon': "exe.png", 'tooltip': 'The content of the response is an executable'},
'css': {'icon': "css.png", 'tooltip': 'The content of the response is a CSS'},
'font': {'icon': "font.png", 'tooltip': 'The content of the response is a font'},
'html': {'icon': "html.png", 'tooltip': 'The content of the response is a HTML document'},
'json': {'icon': "json.png", 'tooltip': 'The content of the response is a Json'},
'text': {'icon': "json.png", 'tooltip': 'The content of the response is a text'}, # FIXME: Need new icon
'iframe': {'icon': "ifr.png", 'tooltip': 'This content is loaded from an Iframe'},
'image': {'icon': "img.png", 'tooltip': 'The content of the response is an image'},
'unset_mimetype': {'icon': "wtf.png", 'tooltip': 'The type of content of the response is not set'},
'octet-stream': {'icon': "wtf.png", 'tooltip': 'The type of content of the response is a binary blob'},
'unknown_mimetype': {'icon': "wtf.png", 'tooltip': 'The type of content of the response is of an unknown type'},
'video': {'icon': "video.png", 'tooltip': 'The content of the response is a video'},
'livestream': {'icon': "video.png", 'tooltip': 'The content of the response is a livestream'},
'response_cookie': {'icon': "cookie_received.png", 'tooltip': 'There are cookies in the response'},
# redirect has to be last
'redirect': {'icon': "redirect.png", 'tooltip': 'The request is redirected'},
'redirect_to_nothing': {'icon': "cookie_in_url.png", 'tooltip': 'The request is redirected to an URL we do not have in the capture'}
}
keys_request = {
'request_cookie': {'icon': "cookie_read.png", 'tooltip': 'There are cookies in the request'}
}
try:
hostnode, urls = lookyloo.get_hostnode_investigator(tree_uuid, node_uuid)
except IndexError:
@ -236,8 +244,6 @@ def hostnode_popup(tree_uuid: str, node_uuid: str):
hostnode_uuid=node_uuid,
hostnode=hostnode,
urls=urls,
keys_response=keys_response,
keys_request=keys_request,
enable_context_by_users=enable_context_by_users,
uwhois_available=lookyloo.uwhois.available)
@ -695,6 +701,12 @@ def mark_as_legitimate(tree_uuid: str):
return jsonify({'message': 'Legitimate entry added.'})
@app.route('/tree/<string:tree_uuid>/body_hashes', methods=['GET'])
def tree_body_hashes(tree_uuid: str):
body_hashes = lookyloo.get_all_body_hashes(tree_uuid)
return render_template('tree_body_hashes.html', tree_uuid=tree_uuid, body_hashes=body_hashes)
# ##### helpers #####
def index_generic(show_hidden: bool=False, show_error: bool=True, category: Optional[str]=None):

View File

@ -17,7 +17,7 @@
"exe.png": "pWwo9nBLtEss/UJ173zHa6/RpySUyz/XMdNhWc6aRIvwwHMO6a+fLmu2K6TbvO3Jbg4VYL2Af4yhHPyhH3ZeTw==",
"favicon.ico": "KOmrfwRbOQqhhwSeBkNpMRAxSVMmmLg+2kRMg9iSv7OWjE9spJc7x4MKB4AE/hi0knaV7UBVctAU6XZ7AC72ZA==",
"font.png": "RwoQkj9dT9SLUL2F7cAA16Nat9t2hDb58eQlHF9ThUar829p0INUXG+5XuDaFOC8SsmCZK5vw2f+YAQ6mLC1Qw==",
"generic.css": "6pYUMp7DzXI/O531PJ2PIB0/ce0TdIWEOEC4RfpcbMK2SRgKkZfGn12aixsTJEzAkF8Ao0/Bz405v3Bl0f7RUQ==",
"generic.css": "Sh/BcxFMLYYaLdCluVt9efGvJ9CF5d+YJ7lkL2M24PRGu8VZHI9lJiUlFObIocjQgwss3Ve2U5cUAE5WiAdpQQ==",
"generic.js": "UmFl4fHmB/UjMdUuYdFy9BfzQlJTyeImNHCFyBO4SdLxBCwCGxkF3NQvel1PKqW8JTnoPlPpq/n9d+vCfPeegA==",
"html.png": "T7pZrb8MMDsA/JV/51hu+TOglTqlxySuEVY0rpDjTuAEyhzk2v+W4kYrj7vX+Tp3n2d2lvVD08PwhCG62Yfbzg==",
"ifr.png": "rI5YJypmz1QcULRf9UaOYSqV4tPUSxUdLAycoYzCwywt4Pw4eWzBg9SUr769VyIimoiIyJR+aNuoIA4p5WO2fQ==",

View File

@ -59,7 +59,6 @@ table td p {
font-smooth: auto;
color: white;
background: black;
z-index: 1;
border: 2px solid;
border-color: white;
padding-top: 2px;

View File

@ -3,7 +3,8 @@
{% from "macros.html" import ressource_legitimacy_details %}
{% from "macros.html" import indexed_hash %}
{% from "macros.html" import indexed_cookies %}
{% from "macros.html" import popup_icons %}
{% from "macros.html" import popup_icons_request %}
{% from "macros.html" import popup_icons_response %}
{% from "macros.html" import shorten_string %}
{% from "macros.html" import other_captures_table %}
{% from "macros.html" import get_ressource_button %}
@ -142,7 +143,7 @@
<li class="list-group-item">
<p class="h4">Request</p>
{{ popup_icons(keys_request, url['url_object'], tree_uuid) }}
{{ popup_icons_request(url['url_object'], tree_uuid) }}
{% if url['url_object'].posted_data %}
<a href="{{ url_for('urlnode_post_request', tree_uuid=tree_uuid, node_uuid=url['url_object'].uuid) }}">
@ -179,7 +180,7 @@
<span>Load time: {{ url['url_object'].time.total_seconds() }}s</span>
</small>
</p>
{{ popup_icons(keys_response, url['url_object'], tree_uuid) }}
{{ popup_icons_response(url['url_object'], tree_uuid) }}
{% if url['url_object'].has_dl_file %}
<a href="{{ url_for('data', tree_uuid=tree_uuid)}}">

View File

@ -258,55 +258,62 @@
{% endif %}
{% endmacro %}
{% macro popup_icons(lookup_dict, urlnode, tree_uuid) %}
{% macro popup_icons_request(urlnode, tree_uuid) %}
<div>
{% for key, icon_info in lookup_dict.items() %}
{% if urlnode[key] %}
{% if key == "request_cookie" %}
{% if urlnode.request_cookie %}
{% set icon_info = get_icon("request_cookie") %}
<a href="{{ url_for('urlnode_request_cookies', tree_uuid=tree_uuid, node_uuid=urlnode.uuid) }}"
title="Download all the cookies in the request to the server">
<img src="{{ url_for('static', filename=icon_info['icon']) }}" alt="{{ icon_info['tooltip'] }}"
width="21" height="21"/>
<img src="{{ url_for('static', filename=icon_info['icon']) }}" alt="{{ icon_info['tooltip'] }}"
width="21" height="21"/>
</a>
{% elif key == "response_cookie"%}
{% endif %}
</div>
{% endmacro %}
{% macro popup_icons_response(urlnode, tree_uuid) %}
<div>
{% if urlnode.response_cookie %}
{% set icon_info = get_icon("response_cookie") %}
<a href="{{ url_for('urlnode_response_cookies', tree_uuid=tree_uuid, node_uuid=urlnode.uuid) }}"
title="Download all the cookies in the response from the server">
<img src="{{ url_for('static', filename=icon_info['icon']) }}" alt="{{ icon_info['tooltip'] }}"
width="21" height="21"/>
</a>
{% elif key in ["js", "exe", "css", "font", "html", "json", "image", "video",
"unknown_mimetype", "text", "unset_mimetype", "octet-stream", "livestream"]
and not urlnode.empty_response %}
<a href="{{ url_for('get_ressource', tree_uuid=tree_uuid, node_uuid=urlnode.uuid) }}">
<img src="{{ url_for('static', filename=icon_info['icon']) }}" alt="{{ icon_info['tooltip'] }}"
width="21" height="21"
{% if key == "image" %}
data-bs-toggle="tooltip" data-bs-placement="bottom" data-bs-html="true" title='<img class="ressource_preview" src="{{ url_for('get_ressource_preview', tree_uuid=tree_uuid, node_uuid=urlnode.uuid) }}"/> </br>Click to download.'
{% else %}
data-bs-toggle="tooltip" data-bs-placement="bottom" data-bs-html="true" title="{{icon_info['tooltip']}} <br/>Click to download."
{% endif %}
/>
</a>
{% elif key != "redirect" %}
<img src="{{ url_for('static', filename=icon_info['icon']) }}"
alt="{{ icon_info['tooltip'] }}" title="{{ icon_info['tooltip'] }}" width="21" height="21"/>
{%endif%}
{% endif %}
{% if urlnode.generic_type in ["js", "exe", "css", "font", "html", "json", "image", "video",
"unknown_mimetype", "text", "unset_mimetype", "octet-stream",
"livestream"] %}
{% set icon_info = get_icon(urlnode.generic_type) %}
<a href="{{ url_for('get_ressource', tree_uuid=tree_uuid, node_uuid=urlnode.uuid) }}">
<img src="{{ url_for('static', filename=icon_info['icon']) }}" alt="{{ icon_info['tooltip'] }}"
width="21" height="21"
{% if urlnode.generic_type == "image" %}
data-bs-toggle="tooltip" data-bs-placement="bottom" data-bs-html="true"
title='<img class="ressource_preview" src="{{ url_for('get_ressource_preview', tree_uuid=tree_uuid, node_uuid=urlnode.uuid) }}"/> </br>Click to download.'
{% else %}
data-bs-toggle="tooltip" data-bs-placement="bottom" data-bs-html="true"
title="{{icon_info['tooltip']}} <br/>Click to download."
{% endif %}
/>
</a>
{%endif%}
{% endfor %}
</div>
<div>
{% if "redirect" in lookup_dict and urlnode["redirect"] %}
{% if urlnode["redirect"] %}
{% set icon_info = get_icon('redirect') %}
{% for child in urlnode.children if child.name == urlnode.redirect_url %}
<div title='{{ urlnode.redirect_url }}'>
<b>Redirect to</b>: {{ shorten_string(urlnode.redirect_url, 50) }}
<a href="#/" role="button" onclick="whereAmI('{{ child.hostnode_uuid }}')" title="See the node the URL redirects to.">
<img src="{{ url_for('static', filename=lookup_dict['redirect']['icon']) }}" alt="{{ lookup_dict['redirect']['tooltip'] }}" width="21" height="21"/>
<img src="{{ url_for('static', filename=icon_info['icon']) }}" alt="{{ icon_info['tooltip'] }}" width="21" height="21"/>
</a>
</div>
{% else %}
<img src="{{ url_for('static', filename=lookup_dict['redirect']['icon']) }}"
alt="{{ lookup_dict['redirect']['tooltip'] }}" title="{{ lookup_dict['redirect']['tooltip'] }}"
<img src="{{ url_for('static', filename=icon_info['icon']) }}"
alt="{{ icon_info['tooltip'] }}" title="{{ icon_info['tooltip'] }}"
width="21" height="21"/>
{% endfor %}
{%endif%}

View File

@ -82,6 +82,13 @@
});
</script>
<script>
$('#bodyHashesModal').on('show.bs.modal', function(e) {
var button = $(e.relatedTarget);
var modal = $(this);
modal.find('.modal-body').load(button.data("remote"));
});
</script>
<script>
$('#mispPushModal').on('show.bs.modal', function(e) {
var button = $(e.relatedTarget);
var modal = $(this);
@ -213,6 +220,10 @@
<a href="#statsModal" data-remote="{{ url_for('stats', tree_uuid=tree_uuid) }}"
data-bs-toggle="modal" data-bs-target="#statsModal" role="button">Tree Statistics</a>
</li>
<li>
<a href="#bodyHashesModal" data-remote="{{ url_for('tree_body_hashes', tree_uuid=tree_uuid) }}"
data-bs-toggle="modal" data-bs-target="#bodyHashesModal" role="button">Tree ressources</a>
</li>
<li>
<a href="#modulesModal" data-remote="{{ url_for('trigger_modules', tree_uuid=tree_uuid, force=False) }}"
data-bs-toggle="modal" data-bs-target="#modulesModal" role="button">Third Party Reports</a>
@ -453,6 +464,23 @@
</div>
</div>
<div class="modal fade" id="bodyHashesModal" tabindex="-1" role="dialog">
<div class="modal-dialog modal-xl" role="document">
<div class="modal-content">
<div class="modal-header">
<h5 class="modal-title" id="bodyHashesModalLabel">Ressources in tree</h5>
<button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
</div>
<div class="modal-body">
... loading ressources ...
</div>
<div class="modal-footer">
<button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Close</button>
</div>
</div>
</div>
</div>
<div class="modal fade" id="mispPushModal" tabindex="-1" role="dialog">
<div class="modal-dialog modal-xl" role="document">
<div class="modal-content">

View File

@ -0,0 +1,67 @@
<script type="text/javascript">
$('#bodyHashesTable')
.on('order.dt', () => {
$(function () {
$('[data-bs-toggle="tooltip"]').tooltip()
})
})
.on('search.dt', () => {
$(function () {
$('[data-bs-toggle="tooltip"]').tooltip()
})
})
.on('page.dt', () => {
$(function () {
$('[data-bs-toggle="tooltip"]').tooltip()
})
})
.DataTable( {
"order": [[ 1, "desc" ]],
"pageLength": 50
});
</script>
{% if from_popup %}
<script type="text/javascript">
function openTreeInNewTab(treeUUID) {
window.opener.openTreeInNewTab(treeUUID);
};
</script>
{% endif %}
<div class="table-responsive">
<table id="bodyHashesTable" class="table">
<thead>
<tr>
<th>File type</th>
<th>Captures total</th>
<th>Hash (sha512)</th>
</tr>
</thead>
<tbody>
{% for body_hash, info in body_hashes.items() %}
{% set icon_info = get_icon(info['node'].generic_type) %}
<tr>
<td>
<a href="{{ url_for('get_ressource', tree_uuid=tree_uuid, node_uuid=info['node'].uuid) }}">
<img src="{{ url_for('static', filename=icon_info['icon']) }}" alt="{{ icon_info['tooltip'] }}"
width="21" height="21"
{% if info['node'].generic_type == "image" %}
data-bs-toggle="tooltip" data-bs-placement="left" data-bs-html="true" data-container="#bodyHashesTable"
title='<img class="ressource_preview" src="{{ url_for('get_ressource_preview', tree_uuid=tree_uuid, node_uuid=info['node'].uuid) }}"/> </br>Click to download.'
{% else %}
data-bs-toggle="tooltip" data-bs-placement="left" data-bs-html="true" data-container="#bodyHashesTable"
title="{{icon_info['tooltip']}} <br/>Click to download."
{% endif %}
/>
</a>
</td>
<td>{{ info['total_captures'] }}</td>
<td>
<a href="{{ url_for('body_hash_details', body_hash=body_hash) }}">{{body_hash}}</a>
</td>
</tr>
{% endfor %}
</tbody>
</table>
</div>