new: find related captures by hostname and URL

.gitignore

@@ -112,11 +112,14 @@ FileSaver.js
 d3.v5.min.js
 d3.v5.js
 
-cache.pid
+*.pid
-dump.rdb
+*.rdb
+*log*
+full_index/db
 
 # Local config files
 config/*.json
+config/users/*.json
 config/*.json.bkp
 config/takedown_filters.ini
 
@@ -126,3 +129,25 @@ known_content_user/
 user_agents/
 
 .DS_Store
+
+.idea
+
+archived_captures
+discarded_captures
+removed_captures
+
+website/web/static/d3.min.js
+website/web/static/datatables.min.css
+website/web/static/datatables.min.js
+website/web/static/jquery.min.js
+
+# Modules
+circl_pypdns
+eupi
+own_user_agents
+phishtank
+riskiq
+sanejs
+urlhaus
+urlscan
+vt_url

website/web/__init__.py

@@ -39,8 +39,7 @@ from pymisp import MISPEvent, MISPServerError  # type: ignore[attr-defined]
 from werkzeug.security import check_password_hash
 from werkzeug.wrappers.response import Response as WerkzeugResponse
 
-from lookyloo import Lookyloo, CaptureSettings, Indexing
+from lookyloo import Lookyloo, CaptureSettings
-from lookyloo.capturecache import CaptureCache
 from lookyloo.default import get_config
 from lookyloo.exceptions import MissingUUID, NoValidHarFile
 from lookyloo.helpers import get_taxonomies, UserAgents, load_cookies
@@ -54,7 +53,7 @@ else:
 from .genericapi import api as generic_api
 from .helpers import (User, build_users_table, get_secret_key,
                       load_user_from_request, src_request_ip, sri_load,
-                      get_lookyloo_instance)
+                      get_lookyloo_instance, get_indexing)
 from .proxied import ReverseProxied
 
 logging.config.dictConfig(get_config('logging'))
@@ -270,23 +269,6 @@ def file_response(func):  # type: ignore[no-untyped-def]
 
 # ##### Methods querying the indexes #####
 
-@functools.cache
-def get_indexing(user: User | None) -> Indexing:
-    '''Depending if we're logged in or not, we (can) get different indexes:
-        if index_everything is enabled, we have an index in kvrocks that contains all
-        the indexes for all the captures.
-        It is only accessible to the admin user.
-    '''
-    if not get_config('generic', 'index_everything'):
-        return Indexing()
-
-    if not user or not user.is_authenticated:
-        # No user or anonymous
-        return Indexing()
-    # Logged in user
-    return Indexing(full_index=True)
-
-
 def _get_body_hash_investigator(body_hash: str, /) -> tuple[list[tuple[str, str, datetime, str, str]], list[tuple[str, float]]]:
     '''Returns all the captures related to a hash (sha512), used in the web interface.'''
     total_captures, details = get_indexing(flask_login.current_user).get_body_hash_captures(body_hash, limit=-1)
@@ -365,70 +347,33 @@ def get_all_hostnames(capture_uuid: str, /) -> dict[str, dict[str, int | list[UR
     return to_return
 
 
-def get_latest_url_capture(url: str, /) -> CaptureCache | None:
+def get_all_urls(capture_uuid: str, /) -> dict[str, dict[str, int | list[URLNode] | str]]:
-    '''Get the most recent capture with this URL'''
+    ct = lookyloo.get_crawled_tree(capture_uuid)
-    captures = lookyloo.sorted_capture_cache(get_indexing(flask_login.current_user).get_captures_url(url))
+    to_return: dict[str, dict[str, list[URLNode] | int | str]] = defaultdict()
-    if captures:
+    for node in ct.root_hartree.url_tree.traverse():
-        return captures[0]
+        if not node.name:
-    return None
+            continue
-
+        captures = get_indexing(flask_login.current_user).get_captures_url(node.name)
-
+        # Note for future: mayeb get url, capture title, something better than just the hash to show to the user
-def get_url_occurrences(url: str, /, limit: int=20, cached_captures_only: bool=True) -> list[dict[str, Any]]:
+        if node.hostname not in to_return:
-    '''Get the most recent captures and URL nodes where the URL has been seen.'''
+            to_return[node.name] = {'total_captures': len(captures), 'nodes': [],
-    captures = lookyloo.sorted_capture_cache(get_indexing(flask_login.current_user).get_captures_url(url), cached_captures_only=cached_captures_only)
+                                    'quoted_url': quote_plus(node.name)}
-
+        to_return[node.name]['nodes'].append(node)  # type: ignore[union-attr]
-    to_return: list[dict[str, Any]] = []
-    for capture in captures[:limit]:
-        ct = lookyloo.get_crawled_tree(capture.uuid)
-        to_append: dict[str, str | dict[str, Any]] = {'capture_uuid': capture.uuid,
-                                                      'start_timestamp': capture.timestamp.isoformat(),
-                                                      'title': capture.title}
-        urlnodes: dict[str, dict[str, str]] = {}
-        for urlnode in ct.root_hartree.url_tree.search_nodes(name=url):
-            urlnodes[urlnode.uuid] = {'start_time': urlnode.start_time.isoformat(),
-                                      'hostnode_uuid': urlnode.hostnode_uuid}
-            if hasattr(urlnode, 'body_hash'):
-                urlnodes[urlnode.uuid]['hash'] = urlnode.body_hash
-        to_append['urlnodes'] = urlnodes
-        to_return.append(to_append)
-    return to_return
-
-
-def get_hostname_occurrences(hostname: str, /, with_urls_occurrences: bool=False, limit: int=20, cached_captures_only: bool=True) -> list[dict[str, Any]]:
-    '''Get the most recent captures and URL nodes where the hostname has been seen.'''
-    captures = lookyloo.sorted_capture_cache(get_indexing(flask_login.current_user).get_captures_hostname(hostname), cached_captures_only=cached_captures_only)
-
-    to_return: list[dict[str, Any]] = []
-    for capture in captures[:limit]:
-        ct = lookyloo.get_crawled_tree(capture.uuid)
-        to_append: dict[str, str | list[Any] | dict[str, Any]] = {
-            'capture_uuid': capture.uuid,
-            'start_timestamp': capture.timestamp.isoformat(),
-            'title': capture.title}
-        hostnodes: list[str] = []
-        if with_urls_occurrences:
-            urlnodes: dict[str, dict[str, str]] = {}
-        for hostnode in ct.root_hartree.hostname_tree.search_nodes(name=hostname):
-            hostnodes.append(hostnode.uuid)
-            if with_urls_occurrences:
-                for urlnode in hostnode.urls:
-                    urlnodes[urlnode.uuid] = {'start_time': urlnode.start_time.isoformat(),
-                                              'url': urlnode.name,
-                                              'hostnode_uuid': urlnode.hostnode_uuid}
-                    if hasattr(urlnode, 'body_hash'):
-                        urlnodes[urlnode.uuid]['hash'] = urlnode.body_hash
-            to_append['hostnodes'] = hostnodes
-            if with_urls_occurrences:
-                to_append['urlnodes'] = urlnodes
-            to_return.append(to_append)
     return to_return
 
 
 def get_hostname_investigator(hostname: str) -> list[tuple[str, str, str, datetime]]:
+    '''Returns all the captures loading content from that hostname, used in the web interface.'''
     cached_captures = lookyloo.sorted_capture_cache([uuid for uuid in get_indexing(flask_login.current_user).get_captures_hostname(hostname=hostname)])
     return [(cache.uuid, cache.title, cache.redirects[-1], cache.timestamp) for cache in cached_captures]
 
 
+def get_url_investigator(url: str) -> list[tuple[str, str, str, datetime]]:
+    '''Returns all the captures loading content from that url, used in the web interface.'''
+    cached_captures = lookyloo.sorted_capture_cache([uuid for uuid in get_indexing(flask_login.current_user).get_captures_url(url=url)])
+    return [(cache.uuid, cache.title, cache.redirects[-1], cache.timestamp) for cache in cached_captures]
+
+
 def get_cookie_name_investigator(cookie_name: str, /) -> tuple[list[tuple[str, str]], list[tuple[str, float, list[tuple[str, float]]]]]:
     '''Returns all the captures related to a cookie name entry, used in the web interface.'''
     cached_captures = lookyloo.sorted_capture_cache([entry[0] for entry in get_indexing(flask_login.current_user).get_cookies_names_captures(cookie_name)])
@@ -1282,6 +1227,12 @@ def tree_hostnames(tree_uuid: str) -> str:
     return render_template('tree_hostnames.html', tree_uuid=tree_uuid, hostnames=hostnames)
 
 
+@app.route('/tree/<string:tree_uuid>/urls', methods=['GET'])
+def tree_urls(tree_uuid: str) -> str:
+    urls = get_all_urls(tree_uuid)
+    return render_template('tree_urls.html', tree_uuid=tree_uuid, urls=urls)
+
+
 @app.route('/tree/<string:tree_uuid>/pandora', methods=['GET', 'POST'])
 def pandora_submit(tree_uuid: str) -> dict[str, Any] | Response:
     node_uuid = None
@@ -1743,8 +1694,8 @@ def body_hash_details(body_hash: str) -> str:
 @app.route('/urls/<string:url>', methods=['GET'])
 def url_details(url: str) -> str:
     url = unquote_plus(url).strip()
-    hits = get_url_occurrences(url, limit=50)
+    captures = get_url_investigator(url)
-    return render_template('url.html', url=url, hits=hits)
+    return render_template('url.html', url=url, captures=captures)
 
 
 @app.route('/hostnames/<string:hostname>', methods=['GET'])

website/web/genericapi.py

@@ -21,7 +21,7 @@ from lookyloo import CaptureSettings, Lookyloo
 from lookyloo.comparator import Comparator
 from lookyloo.exceptions import MissingUUID, NoValidHarFile
 
-from .helpers import build_users_table, load_user_from_request, src_request_ip, get_lookyloo_instance
+from .helpers import build_users_table, load_user_from_request, src_request_ip, get_lookyloo_instance, get_indexing
 
 api = Namespace('GenericAPI', description='Generic Lookyloo API', path='/')
 
@@ -305,6 +305,27 @@ class HashInfo(Resource):  # type: ignore[misc]
         return to_return
 
 
+def get_url_occurrences(url: str, /, limit: int=20, cached_captures_only: bool=True) -> list[dict[str, Any]]:
+    '''Get the most recent captures and URL nodes where the URL has been seen.'''
+    captures = lookyloo.sorted_capture_cache(get_indexing(flask_login.current_user).get_captures_url(url), cached_captures_only=cached_captures_only)
+
+    to_return: list[dict[str, Any]] = []
+    for capture in captures[:limit]:
+        ct = lookyloo.get_crawled_tree(capture.uuid)
+        to_append: dict[str, str | dict[str, Any]] = {'capture_uuid': capture.uuid,
+                                                      'start_timestamp': capture.timestamp.isoformat(),
+                                                      'title': capture.title}
+        urlnodes: dict[str, dict[str, str]] = {}
+        for urlnode in ct.root_hartree.url_tree.search_nodes(name=url):
+            urlnodes[urlnode.uuid] = {'start_time': urlnode.start_time.isoformat(),
+                                      'hostnode_uuid': urlnode.hostnode_uuid}
+            if hasattr(urlnode, 'body_hash'):
+                urlnodes[urlnode.uuid]['hash'] = urlnode.body_hash
+        to_append['urlnodes'] = urlnodes
+        to_return.append(to_append)
+    return to_return
+
+
 url_info_fields = api.model('URLInfoFields', {
     'url': fields.String(description="The URL to search", required=True),
     'limit': fields.Integer(description="The maximal amount of captures to return", example=20),
@@ -318,12 +339,41 @@ class URLInfo(Resource):  # type: ignore[misc]
 
     @api.doc(body=url_info_fields)  # type: ignore[misc]
     def post(self) -> list[dict[str, Any]]:
-        from . import get_url_occurrences
         to_query: dict[str, Any] = request.get_json(force=True)
         occurrences = get_url_occurrences(to_query.pop('url'), **to_query)
         return occurrences
 
 
+def get_hostname_occurrences(hostname: str, /, with_urls_occurrences: bool=False, limit: int=20, cached_captures_only: bool=True) -> list[dict[str, Any]]:
+    '''Get the most recent captures and URL nodes where the hostname has been seen.'''
+    captures = lookyloo.sorted_capture_cache(get_indexing(flask_login.current_user).get_captures_hostname(hostname), cached_captures_only=cached_captures_only)
+
+    to_return: list[dict[str, Any]] = []
+    for capture in captures[:limit]:
+        ct = lookyloo.get_crawled_tree(capture.uuid)
+        to_append: dict[str, str | list[Any] | dict[str, Any]] = {
+            'capture_uuid': capture.uuid,
+            'start_timestamp': capture.timestamp.isoformat(),
+            'title': capture.title}
+        hostnodes: list[str] = []
+        if with_urls_occurrences:
+            urlnodes: dict[str, dict[str, str]] = {}
+        for hostnode in ct.root_hartree.hostname_tree.search_nodes(name=hostname):
+            hostnodes.append(hostnode.uuid)
+            if with_urls_occurrences:
+                for urlnode in hostnode.urls:
+                    urlnodes[urlnode.uuid] = {'start_time': urlnode.start_time.isoformat(),
+                                              'url': urlnode.name,
+                                              'hostnode_uuid': urlnode.hostnode_uuid}
+                    if hasattr(urlnode, 'body_hash'):
+                        urlnodes[urlnode.uuid]['hash'] = urlnode.body_hash
+            to_append['hostnodes'] = hostnodes
+            if with_urls_occurrences:
+                to_append['urlnodes'] = urlnodes
+            to_return.append(to_append)
+    return to_return
+
+
 hostname_info_fields = api.model('HostnameInfoFields', {
     'hostname': fields.String(description="The hostname to search", required=True),
     'limit': fields.Integer(description="The maximal amount of captures to return", example=20),
@@ -337,7 +387,6 @@ class HostnameInfo(Resource):  # type: ignore[misc]
 
     @api.doc(body=hostname_info_fields)  # type: ignore[misc]
     def post(self) -> list[dict[str, Any]]:
-        from . import get_hostname_occurrences
         to_query: dict[str, Any] = request.get_json(force=True)
         return get_hostname_occurrences(to_query.pop('hostname'), **to_query)
 

website/web/helpers.py

@@ -6,14 +6,14 @@ import hashlib
 import json
 import os
 import re
-from functools import lru_cache
+from functools import lru_cache, cache
 from pathlib import Path
 
 import flask_login  # type: ignore[import-untyped]
 from flask import Request
 from werkzeug.security import generate_password_hash
 
-from lookyloo import Lookyloo
+from lookyloo import Lookyloo, Indexing
 from lookyloo.default import get_config, get_homedir
 
 __global_lookyloo_instance = None
@@ -113,3 +113,20 @@ def get_secret_key() -> bytes:
 def sri_load() -> dict[str, dict[str, str]]:
     with (get_homedir() / 'website' / 'web' / 'sri.txt').open() as f:
         return json.load(f)
+
+
+@cache
+def get_indexing(user: User | None) -> Indexing:
+    '''Depending if we're logged in or not, we (can) get different indexes:
+        if index_everything is enabled, we have an index in kvrocks that contains all
+        the indexes for all the captures.
+        It is only accessible to the admin user.
+    '''
+    if not get_config('generic', 'index_everything'):
+        return Indexing()
+
+    if not user or not user.is_authenticated:
+        # No user or anonymous
+        return Indexing()
+    # Logged in user
+    return Indexing(full_index=True)

website/web/templates/body_hash.html

@@ -77,8 +77,8 @@
 <table id="bodyHashDetailsTable" class="table table-striped" style="width:100%">
   <thead>
    <tr>
-     <th>Timestamp</th>
+     <th>Capture Time</th>
-     <th>Title</th>
+     <th>Capture Title</th>
      <th>URL</th>
    </tr>
   </thead>

website/web/templates/hostname.html

@@ -1,42 +1,7 @@
 {% from "macros.html" import shorten_string %}
 
-{% if from_popup %}
-{% extends "main.html" %}
-
-{% from 'bootstrap5/utils.html' import render_messages %}
-
-{% block title %}{{ url }}{% endblock %}
-
-{% block scripts %}
-{{ super() }}
-<script type="text/javascript">
-    $('#table').DataTable( {
-        "order": [[ 0, "desc" ]],
-        "pageLength": 50,
-        "columnDefs": [{
-            "targets": 0,
-            "render": function ( data, type, row, meta ) {
-                let date = new Date(data);
-                return date.getFullYear() + '-' + (date.getMonth() + 1).toString().padStart(2, "0") + '-' + date.getDate().toString().padStart(2, "0") + ' ' + date.toTimeString();
-            }
-        }]
-    });
-</script>
-
-<script type="text/javascript">
-    function openTreeInNewTab(treeUUID) {
-        window.opener.openTreeInNewTab(treeUUID);
-    };
-</script>
-{% endblock %}
-{%endif%}
-
 {% block content %}
 
-{% if from_popup %}
-<button onclick="window.history.back();" class="btn btn-primary" type="button">Go Back</button>
-{%endif%}
-
 <center>
     <h4>{{ hostname }}</h4>
 </center>
@@ -50,7 +15,8 @@
                     return date.getFullYear() + '-' + (date.getMonth() + 1).toString().padStart(2, "0") + '-' + date.getDate().toString().padStart(2, "0") + ' ' + date.toTimeString();
                    }
                  },
-                 { width: '80%', targets: 1 }],
+                 { width: '40%', targets: 1 },
+                 { width: '40%', targets: 2 }],
   });
 </script>
 

website/web/templates/tree.html

@@ -161,6 +161,20 @@
   });
   </script>
   <script>
+  $('#urlsModal').on('show.bs.modal', function(e) {
+      var button = $(e.relatedTarget);
+      var modal = $(this);
+      modal.find('.modal-body').load(button.data("remote"));
+  });
+  </script>
+  <script>
+  $('#urlDetailsModal').on('show.bs.modal', function(e) {
+      var button = $(e.relatedTarget);
+      var modal = $(this);
+      modal.find('.modal-body').load(button.data("remote"));
+  });
+  </script>
+  <script>
   $('#mispPushModal').on('show.bs.modal', function(e) {
       var button = $(e.relatedTarget);
       var modal = $(this);
@@ -343,23 +357,27 @@
 
               <a href="#bodyHashesModal" data-remote="{{ url_for('tree_body_hashes', tree_uuid=tree_uuid) }}"
                   data-bs-toggle="modal" data-bs-target="#bodyHashesModal" role="button"
-                  title="All ressources contained in the tree">Ressources Capture</a>
+                  title="All ressources contained in the tree">Ressources</a>
 
               <a href="#hostnamesModal" data-remote="{{ url_for('tree_hostnames', tree_uuid=tree_uuid) }}"
                   data-bs-toggle="modal" data-bs-target="#hostnamesModal" role="button"
-                  title="All hostnames contained in the tree">Hostnames Capture</a>
+                  title="All hostnames contained in the tree">Hostnames</a>
+
+              <a href="#urlsModal" data-remote="{{ url_for('tree_urls', tree_uuid=tree_uuid) }}"
+                 data-bs-toggle="modal" data-bs-target="#urlsModal" role="button"
+                 title="All URLs contained in the tree">URLs</a>
 
               <a href="#faviconsModal" data-remote="{{ url_for('tree_favicons', tree_uuid=tree_uuid) }}"
                   data-bs-toggle="modal" data-bs-target="#faviconsModal" role="button"
-                  title="Favicons found on the rendered page">Favicons Capture</a>
+                  title="Favicons found on the rendered page">Favicons</a>
 
               <a href="#captureHashesTypesModal" data-remote="{{ url_for('tree_capture_hashes_types', tree_uuid=tree_uuid) }}"
                   data-bs-toggle="modal" data-bs-target="#captureHashesTypesModal" role="button"
-                  title="Compare hashes of the rendered page">Capture hashes types</a>
+                  title="Compare hashes of the rendered page">(Fuzzy)Hashes types</a>
 
               <a href="#identifiersModal" data-remote="{{ url_for('tree_identifiers', tree_uuid=tree_uuid) }}"
                   data-bs-toggle="modal" data-bs-target="#identifiersModal" role="button"
-                  title="Identifiers found on the rendered page">Identifiers Capture</a>
+                  title="Identifiers found on the rendered page">Other Identifiers</a>
             </div>
           </div>
 
@@ -798,6 +816,43 @@
   </div>
 </div>
 
+<div class="modal fade" id="urlsModal" tabindex="-1" role="dialog">
+  <div class="modal-dialog modal-xl" role="document">
+      <div class="modal-content">
+          <div class="modal-header">
+              <h5 class="modal-title" id="urlsModalLabel">URLs in tree</h5>
+              <button type="button" class="btn btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
+          </div>
+          <div class="modal-body">
+              ... loading urls ...
+          </div>
+          <div class="modal-footer">
+              <button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Close</button>
+          </div>
+      </div>
+  </div>
+</div>
+
+<div class="modal fade" id="urlDetailsModal" tabindex="-1" role="dialog">
+  <div class="modal-dialog modal-xl" role="document">
+      <div class="modal-content">
+          <div class="modal-header">
+              <h5 class="modal-title" id="hostnameDetailsModalLabel">Other occurrences of the URL</h5>
+              <button type="button" class="btn btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
+          </div>
+          <div class="modal-body">
+              ... loading url details ...
+          </div>
+          <div class="modal-footer">
+              <a class="btn btn-primary" href="#HostnamesModal"
+                 data-remote="{{ url_for('tree_urls', tree_uuid=tree_uuid) }}"
+                 data-bs-toggle="modal" data-bs-target="#urlsModal" role="button">Back to capture's URLs</a>
+              <button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Close</button>
+          </div>
+      </div>
+  </div>
+</div>
+
 <div class="modal fade" id="mispPushModal" tabindex="-1" role="dialog">
   <div class="modal-dialog modal-xl" role="document">
       <div class="modal-content">

website/web/templates/tree_body_hashes.html

@@ -18,7 +18,7 @@
 <table id="bodyHashesTable" class="table table-striped" style="width:100%">
 <thead>
  <tr>
-   <th>Captures total</th>
+   <th>Number of captures</th>
    <th>File type</th>
    <th>Ressource URL in capture</th>
    <th>Hash (sha512)</th>

website/web/templates/tree_hostnames.html

@@ -0,0 +1,64 @@
+{% from "macros.html" import popup_icons_response %}
+
+<script type="text/javascript">
+  var table = new DataTable('#hostnamesTable', {
+    order: [[ 0, "desc" ]],
+    columnDefs: [{ width: '10%', targets: 0 },
+                 { width: '40%', targets: 1 },
+                 { width: '50%', targets: 2 }],
+    initComplete: function (settings, json) {
+      $('[data-bs-toggle="tooltip"]').tooltip({html: true});
+    }
+  }).on('draw', function() {
+    $('[data-bs-toggle="tooltip"]').tooltip({html: true});
+  });
+</script>
+
+<table id="hostnamesTable" class="table table-striped" style="width:100%">
+<thead>
+ <tr>
+   <th>Number of captures</th>
+   <th>Hostname</th>
+   <th>URLs</th>
+ </tr>
+</thead>
+<tbody>
+  {% for hostname, info in hostnames.items() %}
+  <tr>
+    <td>{{ info['total_captures'] }}</td>
+    <td>
+      <span class="d-inline-block text-break">
+         <a href="#hostnameDetailsModal" data-remote="{{ url_for('hostname_details', hostname=hostname) }}"
+            data-bs-toggle="modal" data-bs-target="#hostnameDetailsModal" role="button">
+            {{hostname}}
+         </a>
+      </span>
+    </td>
+    <td>
+      <p class="d-inline-flex gap-1">
+        <button class="btn btn-primary" type="button"
+                data-bs-toggle="collapse" data-bs-target="#collapseAllNodes_{{loop.index}}"
+                aria-expanded="false" aria-controls="collapseAllNodes_{{loop.index}}">
+          Show
+        </button>
+      </p>
+      <div class="collapse" id="collapseAllNodes_{{loop.index}}">
+          <div class="card card-body">
+            <span class="d-inline-block text-break">
+              <ul>
+              {% for node in info['nodes'] %}
+                <li>
+                  <p class="text-break">{{ node.name }}</p>
+                  <a href="#/" onclick="openTreeInNewTab('{{ tree_uuid }}', '{{ node.uuid }}')">Show on tree</a>
+                </li>
+              {% endfor %}
+              <ul>
+            </span>
+          </div>
+        </div>
+      </div>
+    </td>
+  </tr>
+  {% endfor %}
+</tbody>
+</table>

website/web/templates/tree_urls.html

@@ -0,0 +1,38 @@
+{% from "macros.html" import popup_icons_response %}
+
+<script type="text/javascript">
+  var table = new DataTable('#urlsTable', {
+    order: [[ 0, "desc" ]],
+    columnDefs: [{ width: '10%', targets: 0 },
+                 { width: '90%', targets: 1 }],
+    initComplete: function (settings, json) {
+      $('[data-bs-toggle="tooltip"]').tooltip({html: true});
+    }
+  }).on('draw', function() {
+    $('[data-bs-toggle="tooltip"]').tooltip({html: true});
+  });
+</script>
+
+<table id="urlsTable" class="table table-striped" style="width:100%">
+<thead>
+ <tr>
+   <th>Number of captures</th>
+   <th>URL</th>
+ </tr>
+</thead>
+<tbody>
+  {% for url, info in urls.items() %}
+  <tr>
+    <td>{{ info['total_captures'] }}</td>
+    <td>
+      <span class="d-inline-block text-break">
+         <a href="#urlDetailsModal" data-remote="{{ url_for('url_details', url=info['quoted_url']) }}"
+            data-bs-toggle="modal" data-bs-target="#urlDetailsModal" role="button">
+            {{url}}
+         </a>
+      </span>
+    </td>
+  </tr>
+  {% endfor %}
+</tbody>
+</table>

website/web/templates/url.html

@@ -1,70 +1,51 @@
-{% extends "main.html" %}
+{% from "macros.html" import shorten_string %}
-
-{% from 'bootstrap5/utils.html' import render_messages %}
-
-{% block title %}{{ url }}{% endblock %}
-
-{% block scripts %}
-{{ super() }}
-<script type="text/javascript">
-    $('#table').DataTable( {
-        "order": [[ 0, "desc" ]],
-        "pageLength": 50,
-        "columnDefs": [{
-            "targets": 0,
-            "render": function ( data, type, row, meta ) {
-                let date = new Date(data);
-                return date.getFullYear() + '-' + (date.getMonth() + 1).toString().padStart(2, "0") + '-' + date.toTimeString();
-            }
-        }]
-    });
-</script>
-
-<script type="text/javascript">
-    function openTreeInNewTab(treeUUID) {
-        window.opener.openTreeInNewTab(treeUUID);
-    };
-</script>
-
-{% endblock %}
 
 {% block content %}
-  <center>
+
-      <h4>{{ url }}</h4>
+<center>
-      <button onclick="window.history.back();" class="btn btn-primary" type="button">Go Back</button>
+  <h4>{{ url }}</h4>
-  </center>
+</center>
-  <div class="table-responsive">
+
-  <table id="table" class="table" style="width:96%">
+<script type="text/javascript">
-    <thead>
+  new DataTable('#urlTable', {
-     <tr>
+    order: [[ 0, "desc" ]],
-       <th>Start timestamp</th>
+    columnDefs: [{ width: '20%', targets: 0,
-       <th>Captures</th>
+                   render: (data) => {
-     </tr>
+                    const date = new Date(data);
-    </thead>
+                    return date.getFullYear() + '-' + (date.getMonth() + 1).toString().padStart(2, "0") + '-' + date.getDate().toString().padStart(2, "0") + ' ' + date.toTimeString();
-    <tbody>
+                   }
-      {% for hit in hits %}
+                 },
-      <tr>
+                 { width: '40%', targets: 1 },
-        <td>
+                 { width: '40%', targets: 2 }],
-          {{ hit['start_timestamp'] }}
+  });
-        </td>
+</script>
-        <td><a href="{{ url_for('tree', tree_uuid=hit['capture_uuid']) }}">{{ hit['title'] }}</a>
+
-            </br>
+<table id="urlTable" class="table table-striped" style="width:100%">
-            Nodes:
+  <thead>
-            <ul>
+   <tr>
-              {% for urlnode_uuid, data in hit['urlnodes'].items() %}
+     <th>Capture Time</th>
-              <li><a href="{{ url_for('tree', tree_uuid=hit['capture_uuid'], node_uuid=data['hostnode_uuid']) }}">{{ data['start_time'] }}</a></li>
+     <th>Capture Title</th>
-              {% endfor %}
+     <th>Landing page</th>
-            </ul>
+   </tr>
-        </td>
+  </thead>
-      </tr>
+  <tbody>
-      {% endfor %}
+    {% for capture_uuid, title, landing_page, capture_time in captures %}
-    </tbody>
+    <tr>
-  </table>
+      <td>
-  </div>
+        {{capture_time}}
-  <p>The same file was seen in these captures:</p>
+      </td>
-  <ul>
+      <td>
-  {% for capture_uuid, title in captures %}
+        <a href="{{ url_for('tree', tree_uuid=capture_uuid) }}">
-    <li><a href="#/" onclick="openTreeInNewTab('{{ capture_uuid }}')">{{ title }}</a></li>
+          {{ title }}
-  {% endfor %}
+        </a>
-  </ul>
+      </td>
+      <td>
+        <span class="d-inline-block text-break" style="max-width: 400px;">
+          {{ landing_page }}
+        </span>
+      </td>
+    </tr>
+    {% endfor %}
+  </tbody>
+</table>
 {% endblock %}