new: Modal to find other captures containing the same hostnames

website/web/__init__.py

@@ -351,6 +351,20 @@ def get_all_body_hashes(capture_uuid: str, /) -> dict[str, dict[str, URLNode | i
     return to_return
 
 
+def get_all_hostnames(capture_uuid: str, /) -> dict[str, dict[str, int | list[URLNode]]]:
+    ct = lookyloo.get_crawled_tree(capture_uuid)
+    to_return: dict[str, dict[str, list[URLNode] | int]] = defaultdict()
+    for node in ct.root_hartree.url_tree.traverse():
+        if not node.hostname:
+            continue
+        captures = get_indexing(flask_login.current_user).get_captures_hostname(node.hostname)
+        # Note for future: mayeb get url, capture title, something better than just the hash to show to the user
+        if node.hostname not in to_return:
+            to_return[node.hostname] = {'total_captures': len(captures), 'nodes': []}
+        to_return[node.hostname]['nodes'].append(node)  # type: ignore[union-attr]
+    return to_return
+
+
 def get_latest_url_capture(url: str, /) -> CaptureCache | None:
     '''Get the most recent capture with this URL'''
     captures = lookyloo.sorted_capture_cache(get_indexing(flask_login.current_user).get_captures_url(url))
@@ -410,6 +424,11 @@ def get_hostname_occurrences(hostname: str, /, with_urls_occurrences: bool=False
     return to_return
 
 
+def get_hostname_investigator(hostname: str) -> list[tuple[str, str, str, datetime]]:
+    cached_captures = lookyloo.sorted_capture_cache([uuid for uuid in get_indexing(flask_login.current_user).get_captures_hostname(hostname=hostname)])
+    return [(cache.uuid, cache.title, cache.redirects[-1], cache.timestamp) for cache in cached_captures]
+
+
 def get_cookie_name_investigator(cookie_name: str, /) -> tuple[list[tuple[str, str]], list[tuple[str, float, list[tuple[str, float]]]]]:
     '''Returns all the captures related to a cookie name entry, used in the web interface.'''
     cached_captures = lookyloo.sorted_capture_cache([entry[0] for entry in get_indexing(flask_login.current_user).get_cookies_names_captures(cookie_name)])
@@ -1257,6 +1276,12 @@ def tree_body_hashes(tree_uuid: str) -> str:
     return render_template('tree_body_hashes.html', tree_uuid=tree_uuid, body_hashes=body_hashes)
 
 
+@app.route('/tree/<string:tree_uuid>/hostnames', methods=['GET'])
+def tree_hostnames(tree_uuid: str) -> str:
+    hostnames = get_all_hostnames(tree_uuid)
+    return render_template('tree_hostnames.html', tree_uuid=tree_uuid, hostnames=hostnames)
+
+
 @app.route('/tree/<string:tree_uuid>/pandora', methods=['GET', 'POST'])
 def pandora_submit(tree_uuid: str) -> dict[str, Any] | Response:
     node_uuid = None
@@ -1724,8 +1749,8 @@ def url_details(url: str) -> str:
 
 @app.route('/hostnames/<string:hostname>', methods=['GET'])
 def hostname_details(hostname: str) -> str:
-    hits = get_hostname_occurrences(hostname.strip(), with_urls_occurrences=True, limit=50)
+    captures = get_hostname_investigator(hostname.strip())
-    return render_template('hostname.html', hostname=hostname, hits=hits)
+    return render_template('hostname.html', hostname=hostname, captures=captures)
 
 
 @app.route('/stats', methods=['GET'])

website/web/templates/hostname.html

@@ -1,3 +1,6 @@
+{% from "macros.html" import shorten_string %}
+
+{% if from_popup %}
 {% extends "main.html" %}
 
 {% from 'bootstrap5/utils.html' import render_messages %}
@@ -26,44 +29,57 @@
     };
 </script>
 {% endblock %}
+{%endif%}
 
 {% block content %}
-  <center>
+
-      <h4>{{ url }}</h4>
+{% if from_popup %}
-      <button onclick="window.history.back();" class="btn btn-primary" type="button">Go Back</button>
+<button onclick="window.history.back();" class="btn btn-primary" type="button">Go Back</button>
-  </center>
+{%endif%}
-  <div class="table-responsive">
+
-  <table id="table" class="table" style="width:96%">
+<center>
-    <thead>
+    <h4>{{ hostname }}</h4>
-     <tr>
+</center>
-       <th>Start timestamp</th>
+
-       <th>Captures</th>
+<script type="text/javascript">
-     </tr>
+  new DataTable('#hostnameTable', {
-    </thead>
+    order: [[ 0, "desc" ]],
-    <tbody>
+    columnDefs: [{ width: '20%', targets: 0,
-      {% for hit in hits %}
+                   render: (data) => {
-      <tr>
+                    const date = new Date(data);
-        <td>
+                    return date.getFullYear() + '-' + (date.getMonth() + 1).toString().padStart(2, "0") + '-' + date.getDate().toString().padStart(2, "0") + ' ' + date.toTimeString();
-          {{ hit['start_timestamp'] }}
+                   }
-        </td>
+                 },
-        <td><a href="{{ url_for('tree', tree_uuid=hit['capture_uuid']) }}">{{ hit['title'] }}</a>
+                 { width: '80%', targets: 1 }],
-            </br>
+  });
-            Nodes:
+</script>
-            <ul>
+
-              {% for urlnode_uuid, data in hit['urlnodes'].items() %}
+<table id="hostnameTable" class="table table-striped" style="width:100%">
-              <li><a href="{{ url_for('tree', tree_uuid=hit['capture_uuid'], node_uuid=data['hostnode_uuid']) }}">{{ data['start_time'] }}</a></li>
+  <thead>
-              {% endfor %}
+   <tr>
-            </ul>
+     <th>Capture Time</th>
-        </td>
+     <th>Capture Title</th>
-      </tr>
+     <th>Landing page</th>
-      {% endfor %}
+   </tr>
-    </tbody>
+  </thead>
-  </table>
+  <tbody>
-  </div>
+    {% for capture_uuid, title, landing_page, capture_time in captures %}
-  <p>The same file was seen in these captures:</p>
+    <tr>
-  <ul>
+      <td>
-  {% for capture_uuid, title in captures %}
+        {{capture_time}}
-    <li><a href="#/" onclick="openTreeInNewTab('{{ capture_uuid }}')">{{ title }}</a></li>
+      </td>
-  {% endfor %}
+      <td>
-  </ul>
+        <a href="{{ url_for('tree', tree_uuid=capture_uuid) }}">
+          {{ title }}
+        </a>
+      </td>
+      <td>
+        <span class="d-inline-block text-break" style="max-width: 400px;">
+          {{ landing_page }}
+        </span>
+      </td>
+    </tr>
+    {% endfor %}
+  </tbody>
+</table>
 {% endblock %}

website/web/templates/tree.html

@@ -147,6 +147,20 @@
   });
   </script>
   <script>
+  $('#hostnamesModal').on('show.bs.modal', function(e) {
+      var button = $(e.relatedTarget);
+      var modal = $(this);
+      modal.find('.modal-body').load(button.data("remote"));
+  });
+  </script>
+  <script>
+  $('#hostnameDetailsModal').on('show.bs.modal', function(e) {
+      var button = $(e.relatedTarget);
+      var modal = $(this);
+      modal.find('.modal-body').load(button.data("remote"));
+  });
+  </script>
+  <script>
   $('#mispPushModal').on('show.bs.modal', function(e) {
       var button = $(e.relatedTarget);
       var modal = $(this);
@@ -331,6 +345,10 @@
                   data-bs-toggle="modal" data-bs-target="#bodyHashesModal" role="button"
                   title="All ressources contained in the tree">Ressources Capture</a>
 
+              <a href="#hostnamesModal" data-remote="{{ url_for('tree_hostnames', tree_uuid=tree_uuid) }}"
+                  data-bs-toggle="modal" data-bs-target="#hostnamesModal" role="button"
+                  title="All hostnames contained in the tree">Hostnames Capture</a>
+
               <a href="#faviconsModal" data-remote="{{ url_for('tree_favicons', tree_uuid=tree_uuid) }}"
                   data-bs-toggle="modal" data-bs-target="#faviconsModal" role="button"
                   title="Favicons found on the rendered page">Favicons Capture</a>
@@ -743,6 +761,43 @@
   </div>
 </div>
 
+<div class="modal fade" id="hostnamesModal" tabindex="-1" role="dialog">
+  <div class="modal-dialog modal-xl" role="document">
+      <div class="modal-content">
+          <div class="modal-header">
+              <h5 class="modal-title" id="hostnamesModalLabel">Hostnames in tree</h5>
+              <button type="button" class="btn btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
+          </div>
+          <div class="modal-body">
+              ... loading hostnames ...
+          </div>
+          <div class="modal-footer">
+              <button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Close</button>
+          </div>
+      </div>
+  </div>
+</div>
+
+<div class="modal fade" id="hostnameDetailsModal" tabindex="-1" role="dialog">
+  <div class="modal-dialog modal-xl" role="document">
+      <div class="modal-content">
+          <div class="modal-header">
+              <h5 class="modal-title" id="hostnameDetailsModalLabel">Other occurrences of the hostname</h5>
+              <button type="button" class="btn btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
+          </div>
+          <div class="modal-body">
+              ... loading hostname details ...
+          </div>
+          <div class="modal-footer">
+              <a class="btn btn-primary" href="#HostnamesModal"
+                 data-remote="{{ url_for('tree_hostnames', tree_uuid=tree_uuid) }}"
+                 data-bs-toggle="modal" data-bs-target="#hostnamesModal" role="button">Back to capture's hostnames</a>
+              <button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Close</button>
+          </div>
+      </div>
+  </div>
+</div>
+
 <div class="modal fade" id="mispPushModal" tabindex="-1" role="dialog">
   <div class="modal-dialog modal-xl" role="document">
       <div class="modal-content">