new: Modal to find other captures containing the same hostnames

pull/919/head
Raphaël Vinot 2024-05-14 16:08:38 +02:00
parent 7f3fdeb3f6
commit 228432efbb
3 changed files with 136 additions and 40 deletions

View File

@ -351,6 +351,20 @@ def get_all_body_hashes(capture_uuid: str, /) -> dict[str, dict[str, URLNode | i
return to_return
def get_all_hostnames(capture_uuid: str, /) -> dict[str, dict[str, int | list[URLNode]]]:
ct = lookyloo.get_crawled_tree(capture_uuid)
to_return: dict[str, dict[str, list[URLNode] | int]] = defaultdict()
for node in ct.root_hartree.url_tree.traverse():
if not node.hostname:
continue
captures = get_indexing(flask_login.current_user).get_captures_hostname(node.hostname)
# Note for future: mayeb get url, capture title, something better than just the hash to show to the user
if node.hostname not in to_return:
to_return[node.hostname] = {'total_captures': len(captures), 'nodes': []}
to_return[node.hostname]['nodes'].append(node) # type: ignore[union-attr]
return to_return
def get_latest_url_capture(url: str, /) -> CaptureCache | None:
'''Get the most recent capture with this URL'''
captures = lookyloo.sorted_capture_cache(get_indexing(flask_login.current_user).get_captures_url(url))
@ -410,6 +424,11 @@ def get_hostname_occurrences(hostname: str, /, with_urls_occurrences: bool=False
return to_return
def get_hostname_investigator(hostname: str) -> list[tuple[str, str, str, datetime]]:
cached_captures = lookyloo.sorted_capture_cache([uuid for uuid in get_indexing(flask_login.current_user).get_captures_hostname(hostname=hostname)])
return [(cache.uuid, cache.title, cache.redirects[-1], cache.timestamp) for cache in cached_captures]
def get_cookie_name_investigator(cookie_name: str, /) -> tuple[list[tuple[str, str]], list[tuple[str, float, list[tuple[str, float]]]]]:
'''Returns all the captures related to a cookie name entry, used in the web interface.'''
cached_captures = lookyloo.sorted_capture_cache([entry[0] for entry in get_indexing(flask_login.current_user).get_cookies_names_captures(cookie_name)])
@ -1257,6 +1276,12 @@ def tree_body_hashes(tree_uuid: str) -> str:
return render_template('tree_body_hashes.html', tree_uuid=tree_uuid, body_hashes=body_hashes)
@app.route('/tree/<string:tree_uuid>/hostnames', methods=['GET'])
def tree_hostnames(tree_uuid: str) -> str:
hostnames = get_all_hostnames(tree_uuid)
return render_template('tree_hostnames.html', tree_uuid=tree_uuid, hostnames=hostnames)
@app.route('/tree/<string:tree_uuid>/pandora', methods=['GET', 'POST'])
def pandora_submit(tree_uuid: str) -> dict[str, Any] | Response:
node_uuid = None
@ -1724,8 +1749,8 @@ def url_details(url: str) -> str:
@app.route('/hostnames/<string:hostname>', methods=['GET'])
def hostname_details(hostname: str) -> str:
hits = get_hostname_occurrences(hostname.strip(), with_urls_occurrences=True, limit=50)
return render_template('hostname.html', hostname=hostname, hits=hits)
captures = get_hostname_investigator(hostname.strip())
return render_template('hostname.html', hostname=hostname, captures=captures)
@app.route('/stats', methods=['GET'])

View File

@ -1,3 +1,6 @@
{% from "macros.html" import shorten_string %}
{% if from_popup %}
{% extends "main.html" %}
{% from 'bootstrap5/utils.html' import render_messages %}
@ -26,44 +29,57 @@
};
</script>
{% endblock %}
{%endif%}
{% block content %}
<center>
<h4>{{ url }}</h4>
<button onclick="window.history.back();" class="btn btn-primary" type="button">Go Back</button>
</center>
<div class="table-responsive">
<table id="table" class="table" style="width:96%">
<thead>
<tr>
<th>Start timestamp</th>
<th>Captures</th>
</tr>
</thead>
<tbody>
{% for hit in hits %}
<tr>
<td>
{{ hit['start_timestamp'] }}
</td>
<td><a href="{{ url_for('tree', tree_uuid=hit['capture_uuid']) }}">{{ hit['title'] }}</a>
</br>
Nodes:
<ul>
{% for urlnode_uuid, data in hit['urlnodes'].items() %}
<li><a href="{{ url_for('tree', tree_uuid=hit['capture_uuid'], node_uuid=data['hostnode_uuid']) }}">{{ data['start_time'] }}</a></li>
{% endfor %}
</ul>
</td>
</tr>
{% endfor %}
</tbody>
</table>
</div>
<p>The same file was seen in these captures:</p>
<ul>
{% for capture_uuid, title in captures %}
<li><a href="#/" onclick="openTreeInNewTab('{{ capture_uuid }}')">{{ title }}</a></li>
{% endfor %}
</ul>
{% if from_popup %}
<button onclick="window.history.back();" class="btn btn-primary" type="button">Go Back</button>
{%endif%}
<center>
<h4>{{ hostname }}</h4>
</center>
<script type="text/javascript">
new DataTable('#hostnameTable', {
order: [[ 0, "desc" ]],
columnDefs: [{ width: '20%', targets: 0,
render: (data) => {
const date = new Date(data);
return date.getFullYear() + '-' + (date.getMonth() + 1).toString().padStart(2, "0") + '-' + date.getDate().toString().padStart(2, "0") + ' ' + date.toTimeString();
}
},
{ width: '80%', targets: 1 }],
});
</script>
<table id="hostnameTable" class="table table-striped" style="width:100%">
<thead>
<tr>
<th>Capture Time</th>
<th>Capture Title</th>
<th>Landing page</th>
</tr>
</thead>
<tbody>
{% for capture_uuid, title, landing_page, capture_time in captures %}
<tr>
<td>
{{capture_time}}
</td>
<td>
<a href="{{ url_for('tree', tree_uuid=capture_uuid) }}">
{{ title }}
</a>
</td>
<td>
<span class="d-inline-block text-break" style="max-width: 400px;">
{{ landing_page }}
</span>
</td>
</tr>
{% endfor %}
</tbody>
</table>
{% endblock %}

View File

@ -147,6 +147,20 @@
});
</script>
<script>
$('#hostnamesModal').on('show.bs.modal', function(e) {
var button = $(e.relatedTarget);
var modal = $(this);
modal.find('.modal-body').load(button.data("remote"));
});
</script>
<script>
$('#hostnameDetailsModal').on('show.bs.modal', function(e) {
var button = $(e.relatedTarget);
var modal = $(this);
modal.find('.modal-body').load(button.data("remote"));
});
</script>
<script>
$('#mispPushModal').on('show.bs.modal', function(e) {
var button = $(e.relatedTarget);
var modal = $(this);
@ -331,6 +345,10 @@
data-bs-toggle="modal" data-bs-target="#bodyHashesModal" role="button"
title="All ressources contained in the tree">Ressources Capture</a>
<a href="#hostnamesModal" data-remote="{{ url_for('tree_hostnames', tree_uuid=tree_uuid) }}"
data-bs-toggle="modal" data-bs-target="#hostnamesModal" role="button"
title="All hostnames contained in the tree">Hostnames Capture</a>
<a href="#faviconsModal" data-remote="{{ url_for('tree_favicons', tree_uuid=tree_uuid) }}"
data-bs-toggle="modal" data-bs-target="#faviconsModal" role="button"
title="Favicons found on the rendered page">Favicons Capture</a>
@ -743,6 +761,43 @@
</div>
</div>
<div class="modal fade" id="hostnamesModal" tabindex="-1" role="dialog">
<div class="modal-dialog modal-xl" role="document">
<div class="modal-content">
<div class="modal-header">
<h5 class="modal-title" id="hostnamesModalLabel">Hostnames in tree</h5>
<button type="button" class="btn btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
</div>
<div class="modal-body">
... loading hostnames ...
</div>
<div class="modal-footer">
<button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Close</button>
</div>
</div>
</div>
</div>
<div class="modal fade" id="hostnameDetailsModal" tabindex="-1" role="dialog">
<div class="modal-dialog modal-xl" role="document">
<div class="modal-content">
<div class="modal-header">
<h5 class="modal-title" id="hostnameDetailsModalLabel">Other occurrences of the hostname</h5>
<button type="button" class="btn btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
</div>
<div class="modal-body">
... loading hostname details ...
</div>
<div class="modal-footer">
<a class="btn btn-primary" href="#HostnamesModal"
data-remote="{{ url_for('tree_hostnames', tree_uuid=tree_uuid) }}"
data-bs-toggle="modal" data-bs-target="#hostnamesModal" role="button">Back to capture's hostnames</a>
<button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Close</button>
</div>
</div>
</div>
</div>
<div class="modal fade" id="mispPushModal" tabindex="-1" role="dialog">
<div class="modal-dialog modal-xl" role="document">
<div class="modal-content">