From b06f02e1018812eb14d49f14692bf42e2d19e7d8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= <raphael@vinot.info>
Date: Wed, 9 Sep 2020 15:04:21 +0200
Subject: [PATCH] new: Expose rendered content on hostnode page

---
 lookyloo/lookyloo.py                      |  3 +--
 poetry.lock                               | 10 +++++-----
 pyproject.toml                            |  4 ++--
 website/web/__init__.py                   |  9 +++++++++
 website/web/templates/hostname_popup.html |  6 ++++++
 5 files changed, 23 insertions(+), 9 deletions(-)

diff --git a/lookyloo/lookyloo.py b/lookyloo/lookyloo.py
index f77a21c..9c750d2 100644
--- a/lookyloo/lookyloo.py
+++ b/lookyloo/lookyloo.py
@@ -565,11 +565,10 @@ class Lookyloo():
         with open((capture_dir / 'uuid'), 'r') as f:
             uuid = f.read()
         har_files = sorted(capture_dir.glob('*.har'))
-        html_files = sorted(capture_dir.glob('*.html'))
         # NOTE: We only index the public captures
         index = True
         try:
-            ct = CrawledTree(zip(har_files, html_files), uuid)
+            ct = CrawledTree(har_files, uuid)
             if self.is_public_instance:
                 cache = self.capture_cache(capture_uuid)
                 if cache.get('no_index') is not None:
diff --git a/poetry.lock b/poetry.lock
index d17a547..3b420a8 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -303,8 +303,8 @@ category = "main"
 description = "HTTP Archive (HAR) to ETE Toolkit generator"
 name = "har2tree"
 optional = false
-python-versions = ">=3.6,<4.0"
-version = "1.2.3"
+python-versions = ">=3.7,<4.0"
+version = "1.2.4"
 
 [package.dependencies]
 beautifulsoup4 = ">=4.8.2,<5.0.0"
@@ -1101,7 +1101,7 @@ test = ["coverage (>=5.0.3)", "zope.event", "zope.testing"]
 testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"]
 
 [metadata]
-content-hash = "620043e3c07ceff2aadca0da8dcf3658dbd249cba424bd9bc34764f437d978a9"
+content-hash = "f652af4a1d5c25a6908fdda42d0336910abada3eb67414e1a0e45f07b3caee6f"
 lock-version = "1.0"
 python-versions = "^3.7"
 
@@ -1299,8 +1299,8 @@ gunicorn = [
     {file = "gunicorn-20.0.4.tar.gz", hash = "sha256:1904bb2b8a43658807108d59c3f3d56c2b6121a701161de0ddf9ad140073c626"},
 ]
 har2tree = [
-    {file = "har2tree-1.2.3-py3-none-any.whl", hash = "sha256:8283742206bd47f0fa17f71677db874ed5c7aa53aa655df6d91fa01384333c91"},
-    {file = "har2tree-1.2.3.tar.gz", hash = "sha256:d748eef74099a87f498d3de6cf8da100b81952d2a309faa517592549da068d3c"},
+    {file = "har2tree-1.2.4-py3-none-any.whl", hash = "sha256:fe601e443707fd98fae0ee3f61346d191b1d86d0cd392cf4b0fbd51464dcd5b1"},
+    {file = "har2tree-1.2.4.tar.gz", hash = "sha256:397e5502be088a927610e6f82103953296cc4a937f48f5effcdb7bd3ee1bd971"},
 ]
 hyperlink = [
     {file = "hyperlink-20.0.1-py2.py3-none-any.whl", hash = "sha256:c528d405766f15a2c536230de7e160b65a08e20264d8891b3eb03307b0df3c63"},
diff --git a/pyproject.toml b/pyproject.toml
index 92e5bad..170e93e 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -43,9 +43,9 @@ defang = "^0.5.3"
 vt-py = "^0.5.2"
 Flask-HTTPAuth = "^4.1.0"
 pyeupi = "^1.0"
-scrapysplashwrapper = "^1.2"
+scrapysplashwrapper = "^1.2.3"
 pysanejs = "^1.1"
-har2tree = "^1.2"
+har2tree = "^1.2.4"
 pylookyloo = "^1.2"
 
 [tool.poetry.dev-dependencies]
diff --git a/website/web/__init__.py b/website/web/__init__.py
index 2168691..c0df15a 100644
--- a/website/web/__init__.py
+++ b/website/web/__init__.py
@@ -228,6 +228,15 @@ def urlnode_response_cookies(tree_uuid: str, node_uuid: str):
                      mimetype='text/plain', as_attachment=True, attachment_filename='response_cookies.txt')
 
 
+@app.route('/tree/<string:tree_uuid>/url/<string:node_uuid>/rendered_content', methods=['GET'])
+def urlnode_rendered_content(tree_uuid: str, node_uuid: str):
+    urlnode = lookyloo.get_urlnode_from_tree(tree_uuid, node_uuid)
+    if not urlnode.rendered_html:
+        return
+    return send_file(BytesIO(urlnode.rendered_html.getvalue()), mimetype='text/plain',
+                     as_attachment=True, attachment_filename='rendered_content.txt')
+
+
 @app.route('/tree/<string:tree_uuid>/url/<string:node_uuid>/posted_data', methods=['GET'])
 def urlnode_post_request(tree_uuid: str, node_uuid: str):
     urlnode = lookyloo.get_urlnode_from_tree(tree_uuid, node_uuid)
diff --git a/website/web/templates/hostname_popup.html b/website/web/templates/hostname_popup.html
index 67503d0..1e05cbf 100644
--- a/website/web/templates/hostname_popup.html
+++ b/website/web/templates/hostname_popup.html
@@ -128,6 +128,12 @@
           </p>
           {{ popup_icons(keys_response, url['url_object'], tree_uuid) }}
 
+          {% if url['url_object'].rendered_html %}
+          <a href="{{ url_for('urlnode_rendered_content', tree_uuid=tree_uuid, node_uuid=url['url_object'].uuid) }}">
+              Download rendered HTML page
+          </a>
+          {% endif %}
+
           <div>
           {% if url['url_object'].empty_response %}
           Empty body.