new: Paginate hostname table

lookyloo/indexing.py

@@ -512,21 +512,23 @@ class Indexing():
         return self.redis.zcard(f'urls|{md5}|captures')
 
     def get_captures_hostname(self, hostname: str, most_recent_capture: datetime | None = None,
-                              oldest_capture: datetime | None= None) -> list[tuple[str, float]]:
+                              oldest_capture: datetime | None= None,
+                              offset: int | None=None, limit: int | None=None) -> tuple[int, list[tuple[str, float]]]:
         """Get all the captures for a specific hostname, on a time interval starting from the most recent one.
 
         :param url: The URL
         :param most_recent_capture: The capture time of the most recent capture to consider
-        :param oldest_capture: The capture time of the oldest capture to consider, defaults to 15 days ago.
+        :param oldest_capture: The capture time of the oldest capture to consider.
         """
         max_score: str | float = most_recent_capture.timestamp() if most_recent_capture else '+Inf'
-        min_score: str | float = oldest_capture.timestamp() if oldest_capture else (datetime.now() - timedelta(days=15)).timestamp()
+        min_score: str | float = oldest_capture.timestamp() if oldest_capture else '-Inf'
         if self.redis.type(f'hostnames|{hostname}|captures') == 'set':  # type: ignore[no-untyped-call]
             # triggers the re-index soon.
             self.redis.srem('indexed_urls', *self.redis.smembers(f'hostnames|{hostname}|captures'))
             self.redis.delete(f'hostnames|{hostname}|captures')
-            return []
+            return 0, []
-        return self.redis.zrevrangebyscore(f'hostnames|{hostname}|captures', max_score, min_score, withscores=True)
+        total = self.redis.zcard(f'hostnames|{hostname}|captures')
+        return total, self.redis.zrevrangebyscore(f'hostnames|{hostname}|captures', max_score, min_score, withscores=True, start=offset, num=limit)
 
     def get_captures_hostname_count(self, hostname: str) -> int:
         if self.redis.type(f'hostnames|{hostname}|captures') == 'set':  # type: ignore[no-untyped-call]

website/web/__init__.py

@@ -398,14 +398,15 @@ def get_all_urls(capture_uuid: str, /) -> dict[str, dict[str, int | list[URLNode
     return to_return
 
 
-def get_hostname_investigator(hostname: str) -> list[tuple[str, str, str, datetime, set[str]]]:
+def get_hostname_investigator(hostname: str, offset: int | None=None, limit: int | None=None) -> tuple[int, list[tuple[str, str, str, datetime, set[str]]]]:
     '''Returns all the captures loading content from that hostname, used in the web interface.'''
+    total, entries = get_indexing(flask_login.current_user).get_captures_hostname(hostname=hostname, offset=offset, limit=limit)
     cached_captures = lookyloo.sorted_capture_cache(
-        [uuid for uuid, _ in get_indexing(flask_login.current_user).get_captures_hostname(hostname=hostname)],
+        [uuid for uuid, _ in entries],
         cached_captures_only=True)
-    return [(cache.uuid, cache.title, cache.redirects[-1], cache.timestamp,
+    return total, [(cache.uuid, cache.title, cache.redirects[-1], cache.timestamp,
-            get_indexing(flask_login.current_user).get_capture_hostname_nodes(cache.uuid, hostname)
+                   get_indexing(flask_login.current_user).get_capture_hostname_nodes(cache.uuid, hostname)
-             ) for cache in cached_captures]
+                    ) for cache in cached_captures]
 
 
 def get_url_investigator(url: str) -> list[tuple[str, str, str, datetime, set[str]]]:
@@ -1797,8 +1798,8 @@ def url_details(url: str) -> str:
 @app.route('/hostnames/<string:hostname>', methods=['GET'])
 def hostname_details(hostname: str) -> str:
     from_popup = True if (request.args.get('from_popup') and request.args.get('from_popup') == 'True') else False
-    captures = get_hostname_investigator(hostname.strip())
+    # captures = get_hostname_investigator(hostname.strip())
-    return render_template('hostname.html', hostname=hostname, captures=captures, from_popup=from_popup)
+    return render_template('hostname.html', hostname=hostname, from_popup=from_popup)
 
 
 @app.route('/stats', methods=['GET'])
@@ -1977,6 +1978,35 @@ def add_context(tree_uuid: str, node_uuid: str) -> WerkzeugResponse | None:
     return None
 
 
+def __prepare_node_view(capture_uuid: str, nodes: set[str]) -> str:
+    to_return = f'The capture contains this hostname in {len(nodes)} nodes, click below to see them on the tree:'
+    to_return += '<ul>'
+    for node in nodes:
+        to_return += f'<li><a href="{url_for("tree", tree_uuid=capture_uuid, node_uuid=node)}">{node}</a></li>'
+    to_return += '</ul>'
+    return to_return
+
+
+@app.route('/tables/<string:table_name>/<string:value>', methods=['POST'])
+def post_table(table_name: str, value: str) -> Response:
+    draw = request.form.get('draw', type=int)
+    start = request.form.get('start', type=int)
+    length = request.form.get('length', type=int)
+    if table_name == 'hostnameTable':
+        total, captures = get_hostname_investigator(value.strip(), offset=start, limit=length)
+        prepared_captures = []
+        for capture_uuid, title, landing_page, capture_time, nodes in captures:
+            _nodes = __prepare_node_view(capture_uuid, nodes)
+            to_append = {
+                'capture_time': capture_time.isoformat(),
+                'capture_title': f"""<a href="{url_for('tree', tree_uuid=capture_uuid)}">{title}</a></br>{_nodes}""",
+                'landing_page': f"""<span class="d-inline-block text-break" style="max-width: 400px;">{landing_page}</span>"""
+            }
+            prepared_captures.append(to_append)
+        return jsonify({'draw': draw, 'recordsTotal': total, 'recordsFiltered': total, 'data': prepared_captures})
+    return jsonify({})
+
+
 # Query API
 authorizations = {
     'apikey': {

website/web/genericapi.py

@@ -380,8 +380,9 @@ class URLInfo(Resource):  # type: ignore[misc]
 
 def get_hostname_occurrences(hostname: str, /, with_urls_occurrences: bool=False, limit: int=20, cached_captures_only: bool=True) -> list[dict[str, Any]]:
     '''Get the most recent captures and URL nodes where the hostname has been seen.'''
+    _, entries = get_indexing(flask_login.current_user).get_captures_hostname(hostname, offset=0, limit=limit)
     captures = lookyloo.sorted_capture_cache(
-        [uuid for uuid, _ in get_indexing(flask_login.current_user).get_captures_hostname(hostname)],
+        [uuid for uuid, _ in entries],
         cached_captures_only=cached_captures_only)
 
     to_return: list[dict[str, Any]] = []

website/web/sri.txt

@@ -33,7 +33,7 @@
     "loader.gif": "ZZKD5vLSKBWKeUpa2KI9qheUJ49iTI/UULmVU/AX28fBfH00K3lLc2v5pVJZ4qXG1BbB13LTXzRKKU35H2XfNg==",
     "lookyloo.jpeg": "i6wBj8CsIM5YAQLEMQfhs3CNOSKkErF8AMqqM6ZygSwCyQgv9CU8xt94veMZhM/ufBWoz7kAXmR+yywmxsTxug==",
     "redirect.png": "PAjzlPV97rEFvH55mG1ZC9wRl98be3yMeX/nENuFkJcds6/AXgSR2ig/QyPULgobSnNgiYieLVWY/oqsgeywrQ==",
-    "render_tables.js": "Lpte5N1uaEGRFWyn8DlriXQUS51LEMBTj9Ux8CcdkTcg70LuuTo3RPdBrC2b63U/sdaOR9RuDFo9Z4UELK2GTw==",
+    "render_tables.js": "Qv7Mlmqz5f5eu2YUfKPGw8wcJ+F4Iu9MoPaFpDYBTWtGUbszYVbdeiHoL9RYqdGrkWosc3Z8R+C0aIHF11UX1A==",
     "secure.svg": "H8ni7t0d60nCJDVGuZpuxC+RBy/ipAjWT627D12HlZGg6LUmjSwPTQTUekm3UJupEP7TUkhXyq6WHc5gy7QBjg==",
     "stats.css": "/kY943FwWBTne4IIyf7iBROSfbGd82TeBicEXqKkRwawMVRIvM/Pk5MRa7okUyGIxaDjFQGmV/U1vy+PhN6Jbw==",
     "stats_graph.js": "S/sMNQK1UMMLD0xQeEa7sq3ce8o6oPxwxGlyKVtaHOODjair86dbBDm7cu6pa/elMRDJT1j09jEFjWp+5GbhTw==",

website/web/static/render_tables.js

@@ -129,8 +129,19 @@
   }
 
   if (document.getElementById('hostnameTable')) {
+    hostname = document.getElementById('hostnameTable').dataset.hostname;
     new DataTable('#hostnameTable', {
-    retrieve: true,
+    processing: true,
+    serverSide: true,
+    ajax: {
+        url: `/tables/hostnameTable/${hostname}`,
+        type: 'POST'
+    },
+    columns : [
+        { data: 'capture_time' },
+        { data: 'capture_title' },
+        { data: 'landing_page' }
+    ],
     order: [[ 0, "desc" ]],
     columnDefs: [{ width: '20%', targets: 0,
                    render: (data) => {

website/web/templates/hostname.html

@@ -20,7 +20,7 @@
     <h6>Only the most recent captures are listed below, this will change soon.</h6>
 </center>
 
-<table id="hostnameTable" class="table table-striped" style="width:100%">
+<table id="hostnameTable" class="table table-striped" style="width:100%" data-hostname="{{hostname}}">
   <thead>
    <tr>
      <th>Capture Time</th>
@@ -28,35 +28,5 @@
      <th>Landing page</th>
    </tr>
   </thead>
-  <tbody>
-    {% for capture_uuid, title, landing_page, capture_time, nodes in captures %}
-    <tr>
-      <td>
-        {{capture_time}}
-      </td>
-      <td>
-        <a href="{{ url_for('tree', tree_uuid=capture_uuid) }}">
-          {{ title }}
-        </a>
-        <br>
-        The capture contains this hostname in {{ nodes|length }} nodes, click below to see them on the tree:
-        <ul>
-            {% for node in nodes %}
-            <li>
-                <a href="{{ url_for('tree', tree_uuid=capture_uuid, node_uuid=node) }}">
-                {{ shorten_string(node, 50) }}
-                </a>
-            </li>
-            {% endfor %}
-        </ul>
-      </td>
-      <td>
-        <span class="d-inline-block text-break" style="max-width: 400px;">
-          {{ landing_page }}
-        </span>
-      </td>
-    </tr>
-    {% endfor %}
-  </tbody>
 </table>
 {% endblock %}