new: Get URLs and Hostnames from captures

lookyloo/lookyloo.py

@@ -1208,6 +1208,38 @@ class Lookyloo():
             container = self.get_crawled_tree(tree_uuid)
         return get_resources_hashes(container)
 
+    def get_hostnames(self, tree_uuid: str, /, hostnode_uuid: Optional[str]=None, urlnode_uuid: Optional[str]=None) -> Set[str]:
+        """Return all the unique hostnames:
+            * of a complete tree if no hostnode_uuid and urlnode_uuid are given
+            * of a HostNode if hostnode_uuid is given
+            * of a URLNode if urlnode_uuid is given
+        """
+        if urlnode_uuid:
+            node = self.get_urlnode_from_tree(tree_uuid, urlnode_uuid)
+            return {node.hostname}
+        elif hostnode_uuid:
+            node = self.get_hostnode_from_tree(tree_uuid, hostnode_uuid)
+            return {node.name}
+        else:
+            ct = self.get_crawled_tree(tree_uuid)
+            return {node.name for node in ct.root_hartree.hostname_tree.traverse()}
+
+    def get_urls(self, tree_uuid: str, /, hostnode_uuid: Optional[str]=None, urlnode_uuid: Optional[str]=None) -> Set[str]:
+        """Return all the unique URLs:
+            * of a complete tree if no hostnode_uuid and urlnode_uuid are given
+            * of a HostNode if hostnode_uuid is given
+            * of a URLNode if urlnode_uuid is given
+        """
+        if urlnode_uuid:
+            node = self.get_urlnode_from_tree(tree_uuid, urlnode_uuid)
+            return {node.name}
+        elif hostnode_uuid:
+            node = self.get_hostnode_from_tree(tree_uuid, hostnode_uuid)
+            return {urlnode.name for urlnode in node.urls}
+        else:
+            ct = self.get_crawled_tree(tree_uuid)
+            return {node.name for node in ct.root_hartree.url_tree.traverse()}
+
     def get_hostnode_investigator(self, capture_uuid: str, /, node_uuid: str) -> Tuple[HostNode, List[Dict[str, Any]]]:
         '''Gather all the informations needed to display the Hostnode investigator popup.'''
         ct = self.get_crawled_tree(capture_uuid)

website/web/genericapi.py

@@ -65,6 +65,30 @@ class CaptureStatusQuery(Resource):
         return {'status_code': lookyloo.get_capture_status(capture_uuid)}
 
 
+@api.route('/json/<string:capture_uuid>/hostnames')
+@api.doc(description='Get all the hostnames of all the resources of a capture',
+         params={'capture_uuid': 'The UUID of the capture'})
+class CaptureHostnames(Resource):
+    def get(self, capture_uuid: str):
+        cache = lookyloo.capture_cache(capture_uuid)
+        if not cache:
+            return {'error': 'UUID missing in cache, try again later.'}
+        to_return: Dict[str, Any] = {'response': {'hostnames': list(lookyloo.get_hostnames(capture_uuid))}}
+        return to_return
+
+
+@api.route('/json/<string:capture_uuid>/urls')
+@api.doc(description='Get all the URLs of all the resources of a capture',
+         params={'capture_uuid': 'The UUID of the capture'})
+class CaptureURLs(Resource):
+    def get(self, capture_uuid: str):
+        cache = lookyloo.capture_cache(capture_uuid)
+        if not cache:
+            return {'error': 'UUID missing in cache, try again later.'}
+        to_return: Dict[str, Any] = {'response': {'urls': list(lookyloo.get_urls(capture_uuid))}}
+        return to_return
+
+
 @api.route('/json/<string:capture_uuid>/hashes')
 @api.doc(description='Get all the hashes of all the resources of a capture',
          params={'capture_uuid': 'The UUID of the capture'})