new: Expose rendered content on hostnode page

lookyloo/lookyloo.py

@@ -565,11 +565,10 @@ class Lookyloo():
         with open((capture_dir / 'uuid'), 'r') as f:
             uuid = f.read()
         har_files = sorted(capture_dir.glob('*.har'))
-        html_files = sorted(capture_dir.glob('*.html'))
         # NOTE: We only index the public captures
         index = True
         try:
-            ct = CrawledTree(zip(har_files, html_files), uuid)
+            ct = CrawledTree(har_files, uuid)
             if self.is_public_instance:
                 cache = self.capture_cache(capture_uuid)
                 if cache.get('no_index') is not None:

poetry.lock

@@ -303,8 +303,8 @@ category = "main"
 description = "HTTP Archive (HAR) to ETE Toolkit generator"
 name = "har2tree"
 optional = false
-python-versions = ">=3.6,<4.0"
-version = "1.2.3"
+python-versions = ">=3.7,<4.0"
+version = "1.2.4"
 
 [package.dependencies]
 beautifulsoup4 = ">=4.8.2,<5.0.0"
@@ -1101,7 +1101,7 @@ test = ["coverage (>=5.0.3)", "zope.event", "zope.testing"]
 testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"]
 
 [metadata]
-content-hash = "620043e3c07ceff2aadca0da8dcf3658dbd249cba424bd9bc34764f437d978a9"
+content-hash = "f652af4a1d5c25a6908fdda42d0336910abada3eb67414e1a0e45f07b3caee6f"
 lock-version = "1.0"
 python-versions = "^3.7"
 
@@ -1299,8 +1299,8 @@ gunicorn = [
     {file = "gunicorn-20.0.4.tar.gz", hash = "sha256:1904bb2b8a43658807108d59c3f3d56c2b6121a701161de0ddf9ad140073c626"},
 ]
 har2tree = [
-    {file = "har2tree-1.2.3-py3-none-any.whl", hash = "sha256:8283742206bd47f0fa17f71677db874ed5c7aa53aa655df6d91fa01384333c91"},
-    {file = "har2tree-1.2.3.tar.gz", hash = "sha256:d748eef74099a87f498d3de6cf8da100b81952d2a309faa517592549da068d3c"},
+    {file = "har2tree-1.2.4-py3-none-any.whl", hash = "sha256:fe601e443707fd98fae0ee3f61346d191b1d86d0cd392cf4b0fbd51464dcd5b1"},
+    {file = "har2tree-1.2.4.tar.gz", hash = "sha256:397e5502be088a927610e6f82103953296cc4a937f48f5effcdb7bd3ee1bd971"},
 ]
 hyperlink = [
     {file = "hyperlink-20.0.1-py2.py3-none-any.whl", hash = "sha256:c528d405766f15a2c536230de7e160b65a08e20264d8891b3eb03307b0df3c63"},

pyproject.toml

@@ -43,9 +43,9 @@ defang = "^0.5.3"
 vt-py = "^0.5.2"
 Flask-HTTPAuth = "^4.1.0"
 pyeupi = "^1.0"
-scrapysplashwrapper = "^1.2"
+scrapysplashwrapper = "^1.2.3"
 pysanejs = "^1.1"
-har2tree = "^1.2"
+har2tree = "^1.2.4"
 pylookyloo = "^1.2"
 
 [tool.poetry.dev-dependencies]

website/web/__init__.py

@@ -228,6 +228,15 @@ def urlnode_response_cookies(tree_uuid: str, node_uuid: str):
                      mimetype='text/plain', as_attachment=True, attachment_filename='response_cookies.txt')
 
 
+@app.route('/tree/<string:tree_uuid>/url/<string:node_uuid>/rendered_content', methods=['GET'])
+def urlnode_rendered_content(tree_uuid: str, node_uuid: str):
+    urlnode = lookyloo.get_urlnode_from_tree(tree_uuid, node_uuid)
+    if not urlnode.rendered_html:
+        return
+    return send_file(BytesIO(urlnode.rendered_html.getvalue()), mimetype='text/plain',
+                     as_attachment=True, attachment_filename='rendered_content.txt')
+
+
 @app.route('/tree/<string:tree_uuid>/url/<string:node_uuid>/posted_data', methods=['GET'])
 def urlnode_post_request(tree_uuid: str, node_uuid: str):
     urlnode = lookyloo.get_urlnode_from_tree(tree_uuid, node_uuid)

website/web/templates/hostname_popup.html

@@ -128,6 +128,12 @@
           </p>
           {{ popup_icons(keys_response, url['url_object'], tree_uuid) }}
 
+          {% if url['url_object'].rendered_html %}
+          <a href="{{ url_for('urlnode_rendered_content', tree_uuid=tree_uuid, node_uuid=url['url_object'].uuid) }}">
+              Download rendered HTML page
+          </a>
+          {% endif %}
+
           <div>
           {% if url['url_object'].empty_response %}
           Empty body.