new: Locate node with url or hostname on tree when pivotting

lookyloo/indexing.py

@@ -427,7 +427,7 @@ class Indexing():
             if md5_url not in already_indexed_global:
                 # The URL hasn't been indexed in that run yet
                 already_indexed_global.add(md5_url)
-                pipeline.sadd(f'{internal_index}|urls', urlnode.name)  # Only used to delete index
+                pipeline.sadd(f'{internal_index}|urls', md5_url)  # Only used to delete index
                 pipeline.sadd(f'{internal_index}|hostnames', urlnode.hostname)  # Only used to delete index
                 pipeline.sadd('urls', urlnode.name)
                 pipeline.sadd('hostnames', urlnode.hostname)

website/web/__init__.py

@@ -435,20 +435,24 @@ def get_all_urls(capture_uuid: str, /) -> dict[str, dict[str, int | list[URLNode
     return to_return
 
 
-def get_hostname_investigator(hostname: str) -> list[tuple[str, str, str, datetime]]:
+def get_hostname_investigator(hostname: str) -> list[tuple[str, str, str, datetime, set[str]]]:
     '''Returns all the captures loading content from that hostname, used in the web interface.'''
     cached_captures = lookyloo.sorted_capture_cache(
         [uuid for uuid, _ in get_indexing(flask_login.current_user).get_captures_hostname(hostname=hostname)],
         cached_captures_only=True)
-    return [(cache.uuid, cache.title, cache.redirects[-1], cache.timestamp) for cache in cached_captures]
+    return [(cache.uuid, cache.title, cache.redirects[-1], cache.timestamp,
+            get_indexing(flask_login.current_user).get_capture_hostname_nodes(cache.uuid, hostname)
+             ) for cache in cached_captures]
 
 
-def get_url_investigator(url: str) -> list[tuple[str, str, str, datetime]]:
+def get_url_investigator(url: str) -> list[tuple[str, str, str, datetime, set[str]]]:
     '''Returns all the captures loading content from that url, used in the web interface.'''
     cached_captures = lookyloo.sorted_capture_cache(
         [uuid for uuid, _ in get_indexing(flask_login.current_user).get_captures_url(url=url)],
         cached_captures_only=True)
-    return [(cache.uuid, cache.title, cache.redirects[-1], cache.timestamp) for cache in cached_captures]
+    return [(cache.uuid, cache.title, cache.redirects[-1], cache.timestamp,
+             get_indexing(flask_login.current_user).get_capture_url_nodes(cache.uuid, url)
+             ) for cache in cached_captures]
 
 
 def get_cookie_name_investigator(cookie_name: str, /) -> tuple[list[tuple[str, str]], list[tuple[str, float, list[tuple[str, float]]]]]:
@@ -1245,7 +1249,6 @@ def tree(tree_uuid: str, node_uuid: str | None=None) -> Response | str | Werkzeu
                         hostnode_to_highlight = hostnode.uuid
                 except IndexError as e:
                     logging.info(f'Invalid uuid ({e}): {node_uuid}')
-                    pass
         if cache.error:
             flash(cache.error, 'warning')
 

website/web/templates/hostname.html

@@ -30,7 +30,7 @@
    </tr>
   </thead>
   <tbody>
-    {% for capture_uuid, title, landing_page, capture_time in captures %}
+    {% for capture_uuid, title, landing_page, capture_time, nodes in captures %}
     <tr>
       <td>
         {{capture_time}}
@@ -39,6 +39,17 @@
         <a href="{{ url_for('tree', tree_uuid=capture_uuid) }}">
           {{ title }}
         </a>
+        <br>
+        The capture contains this hostname in {{ nodes|length }} nodes, click below to see them on the tree:
+        <ul>
+            {% for node in nodes %}
+            <li>
+                <a href="{{ url_for('tree', tree_uuid=capture_uuid, node_uuid=node) }}">
+                {{ shorten_string(node, 50) }}
+                </a>
+            </li>
+            {% endfor %}
+        </ul>
       </td>
       <td>
         <span class="d-inline-block text-break" style="max-width: 400px;">

website/web/templates/url.html

@@ -30,7 +30,7 @@
    </tr>
   </thead>
   <tbody>
-    {% for capture_uuid, title, landing_page, capture_time in captures %}
+    {% for capture_uuid, title, landing_page, capture_time, nodes in captures %}
     <tr>
       <td>
         {{capture_time}}
@@ -39,6 +39,17 @@
         <a href="{{ url_for('tree', tree_uuid=capture_uuid) }}">
           {{ title }}
         </a>
+        <br>
+        The capture contains this URL in {{ nodes|length }} nodes, click below to see them on the tree:
+        <ul>
+            {% for node in nodes %}
+            <li>
+                <a href="{{ url_for('tree', tree_uuid=capture_uuid, node_uuid=node) }}">
+                {{ shorten_string(node, 50) }}
+                </a>
+            </li>
+            {% endfor %}
+        </ul>
       </td>
       <td>
         <span class="d-inline-block text-break" style="max-width: 400px;">