fix: MISP export when the capture is a file.

lookyloo/lookyloo.py

@@ -1083,6 +1083,12 @@ class Lookyloo():
         if not cache:
             return {'error': 'UUID missing in cache, try again later.'}
 
+        # if the file submitted on lookyloo cannot be displayed (PDF), it willbe downloaded.
+        # In the case, we want to have it as a FileObject in the export
+        filename, pseudofile = self.get_data(capture_uuid)
+        if filename:
+            event = self.misp.export(cache, self.is_public_instance, filename, pseudofile)
+        else:
             event = self.misp.export(cache, self.is_public_instance)
         screenshot: MISPAttribute = event.add_attribute('attachment', 'screenshot_landing_page.png',
                                                         data=self.get_screenshot(cache.uuid),

lookyloo/modules/misp.py

@@ -2,6 +2,8 @@
 
 import logging
 import re
+
+from io import BytesIO
 from collections import defaultdict
 from typing import Any, Dict, List, Optional, Set, Union, TYPE_CHECKING
 
@@ -165,7 +167,9 @@ class MISP():
             if hostnodes and hasattr(hostnodes[0], 'resolved_ips'):
                 obj.add_attributes('ip', *hostnodes[0].resolved_ips)
 
-    def export(self, cache: 'CaptureCache', is_public_instance: bool=False) -> MISPEvent:
+    def export(self, cache: 'CaptureCache', is_public_instance: bool=False,
+               submitted_filename: Optional[str]=None,
+               submitted_file: Optional[BytesIO]=None) -> MISPEvent:
         '''Export a capture in MISP format. You can POST the return of this method
         directly to a MISP instance and it will create an event.'''
         public_domain = get_config('generic', 'public_domain')
@@ -173,15 +177,32 @@ class MISP():
         if cache.url.startswith('file'):
             filename = cache.url.rsplit('/', 1)[-1]
             event.info = f'Lookyloo Capture ({filename})'
+            # Create file object as initial
+            if hasattr(cache.tree.root_hartree.url_tree, 'body'):
+                # The file could be viewed in the browser
+                filename = cache.tree.root_hartree.url_tree.name
+                pseudofile = cache.tree.root_hartree.url_tree.body
+            elif submitted_filename:
+                # Impossible to get the file from the HAR.
+                filename = submitted_filename
+                pseudofile = submitted_file
+            else:
+                raise Exception('We must have a file here.')
+
+            initial_file = FileObject(pseudofile=pseudofile, filename=filename)
+            initial_file.comment = 'This is a capture of a file, rendered in the browser'
+            initial_obj = event.add_object(initial_file)
         else:
             event.info = f'Lookyloo Capture ({cache.url})'
             initial_url = URLObject(cache.url)
             initial_url.comment = 'Submitted URL'
             self.__misp_add_ips_to_URLObject(initial_url, cache.tree.root_hartree.hostname_tree)
+            initial_obj = event.add_object(initial_url)
 
         lookyloo_link: MISPAttribute = event.add_attribute('link', f'https://{public_domain}/tree/{cache.uuid}')  # type: ignore
         if not is_public_instance:
             lookyloo_link.distribution = 0
+        initial_obj.add_reference(lookyloo_link, 'captured-by', 'Capture on lookyloo')
 
         redirects: List[URLObject] = []
         for nb, url in enumerate(cache.redirects):
@@ -201,9 +222,6 @@ class MISP():
                 prec_object.add_reference(u_object, 'redirects-to')
                 prec_object = u_object
 
-        initial_obj = event.add_object(initial_url)
-        initial_obj.add_reference(lookyloo_link, 'captured-by', 'Capture on lookyloo')
-
         for u_object in redirects:
             event.add_object(u_object)
         final_redirect = event.objects[-1]

poetry.lock

@@ -908,13 +908,13 @@ tornado = ["tornado (>=0.2)"]
 
 [[package]]
 name = "har2tree"
-version = "1.21.4"
+version = "1.21.5"
 description = "HTTP Archive (HAR) to ETE Toolkit generator"
 optional = false
 python-versions = ">=3.8,<3.12"
 files = [
-    {file = "har2tree-1.21.4-py3-none-any.whl", hash = "sha256:3f26b06b8e52f811cef35bac02a3266c8ce1354e4674353f884ecb2c01ec56a9"},
+    {file = "har2tree-1.21.5-py3-none-any.whl", hash = "sha256:21681f2bd04a3c7169b5a571dd2ce21c63c80dee4a085bd62043be0d837fa376"},
-    {file = "har2tree-1.21.4.tar.gz", hash = "sha256:c2941315f24a7c207bf79acd1c2da52b57dce31c4685d34cb63b5ce2fbabdcda"},
+    {file = "har2tree-1.21.5.tar.gz", hash = "sha256:62fe710cc0a560ac508c2c598246896a19dfea6eeac41c849dae9eeca7dae757"},
 ]
 
 [package.dependencies]
@@ -926,11 +926,11 @@ numpy = [
     {version = "1.23.3", markers = "python_version < \"3.10\""},
     {version = ">=1.23.4,<2.0.0", markers = "python_version >= \"3.10\""},
 ]
-publicsuffixlist = ">=0.10.0.20230719,<0.11.0.0"
+publicsuffixlist = ">=0.10.0.20230730,<0.11.0.0"
 w3lib = ">=2.1.1,<3.0.0"
 
 [package.extras]
-docs = ["Sphinx (>=7.1.0,<8.0.0)"]
+docs = ["Sphinx (>=7.1.1,<8.0.0)"]
 
 [[package]]
 name = "hiredis"
@@ -1169,21 +1169,21 @@ files = [
 
 [[package]]
 name = "jedi"
-version = "0.18.2"
+version = "0.19.0"
 description = "An autocompletion tool for Python that can be used for text editors."
 optional = false
 python-versions = ">=3.6"
 files = [
-    {file = "jedi-0.18.2-py2.py3-none-any.whl", hash = "sha256:203c1fd9d969ab8f2119ec0a3342e0b49910045abe6af0a3ae83a5764d54639e"},
+    {file = "jedi-0.19.0-py2.py3-none-any.whl", hash = "sha256:cb8ce23fbccff0025e9386b5cf85e892f94c9b822378f8da49970471335ac64e"},
-    {file = "jedi-0.18.2.tar.gz", hash = "sha256:bae794c30d07f6d910d32a7048af09b5a39ed740918da923c6b780790ebac612"},
+    {file = "jedi-0.19.0.tar.gz", hash = "sha256:bcf9894f1753969cbac8022a8c2eaee06bfa3724e4192470aaffe7eb6272b0c4"},
 ]
 
 [package.dependencies]
-parso = ">=0.8.0,<0.9.0"
+parso = ">=0.8.3,<0.9.0"
 
 [package.extras]
 docs = ["Jinja2 (==2.11.3)", "MarkupSafe (==1.1.1)", "Pygments (==2.8.1)", "alabaster (==0.7.12)", "babel (==2.9.1)", "chardet (==4.0.0)", "commonmark (==0.8.1)", "docutils (==0.17.1)", "future (==0.18.2)", "idna (==2.10)", "imagesize (==1.2.0)", "mock (==1.0.1)", "packaging (==20.9)", "pyparsing (==2.4.7)", "pytz (==2021.1)", "readthedocs-sphinx-ext (==2.1.4)", "recommonmark (==0.5.0)", "requests (==2.25.1)", "six (==1.15.0)", "snowballstemmer (==2.1.0)", "sphinx (==1.8.5)", "sphinx-rtd-theme (==0.4.3)", "sphinxcontrib-serializinghtml (==1.1.4)", "sphinxcontrib-websupport (==1.2.4)", "urllib3 (==1.26.4)"]
-qa = ["flake8 (==3.8.3)", "mypy (==0.782)"]
+qa = ["flake8 (==5.0.4)", "mypy (==0.971)", "types-setuptools (==67.2.0.1)"]
 testing = ["Django (<3.1)", "attrs", "colorama", "docopt", "pytest (<7.0.0)"]
 
 [[package]]
@@ -1931,13 +1931,13 @@ files = [
 
 [[package]]
 name = "publicsuffixlist"
-version = "0.10.0.20230719"
+version = "0.10.0.20230730"
 description = "publicsuffixlist implement"
 optional = false
 python-versions = ">=2.6"
 files = [
-    {file = "publicsuffixlist-0.10.0.20230719-py2.py3-none-any.whl", hash = "sha256:62a8d1523726b6a89b1d971b1a687c9b3f1b1daec0ab5ba3bca1ee66bc748254"},
+    {file = "publicsuffixlist-0.10.0.20230730-py2.py3-none-any.whl", hash = "sha256:3a23efc4416474a7ad1db7e85d04f7af24db76a34cec6d82e6e3392b3dd6d8f1"},
-    {file = "publicsuffixlist-0.10.0.20230719.tar.gz", hash = "sha256:9b53fdfe06e07e6da5e6575dabe10698ceb82105057dbcdd98865ee1e5fe0a6d"},
+    {file = "publicsuffixlist-0.10.0.20230730.tar.gz", hash = "sha256:1cd3847ec1ebc02069715630958ad67bdfeb7e7f9f35f699d30fc41298ceedac"},
 ]
 
 [package.extras]
@@ -2458,13 +2458,13 @@ six = "*"
 
 [[package]]
 name = "rich"
-version = "13.4.2"
+version = "13.5.0"
 description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal"
 optional = false
 python-versions = ">=3.7.0"
 files = [
-    {file = "rich-13.4.2-py3-none-any.whl", hash = "sha256:8f87bc7ee54675732fa66a05ebfe489e27264caeeff3728c945d25971b6485ec"},
+    {file = "rich-13.5.0-py3-none-any.whl", hash = "sha256:996670a7618ccce27c55ba6fc0142e6e343773e11d34c96566a17b71b0e6f179"},
-    {file = "rich-13.4.2.tar.gz", hash = "sha256:d653d6bccede5844304c605d5aac802c7cf9621efd700b46c7ec2b51ea914898"},
+    {file = "rich-13.5.0.tar.gz", hash = "sha256:62c81e88dc078d2372858660e3d5566746870133e51321f852ccc20af5c7e7b2"},
 ]
 
 [package.dependencies]
@@ -3146,4 +3146,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p
 [metadata]
 lock-version = "2.0"
 python-versions = ">=3.8,<3.12"
-content-hash = "005fd2ad7533d8f017f529ffd2b8844a7c41b40d7f63e8ce599a42595c59375f"
+content-hash = "525ee1c64910329ddcf9bb9d13eaac3d2241f02355b131b9e2ee599aed770a3a"

pyproject.toml

@@ -53,14 +53,14 @@ pytaxonomies = "^1.5.0"
 pymisp = {version = "^2.4.173", extras = ["url", "fileobjects"]}
 Pillow = "^10.0.0"
 flask-restx = "^1.1.0"
-rich = "^13.4.2"
+rich = "^13.5.0"
 pyphishtanklookup = "^1.3.0"
 Flask-Cors = "^4.0.0"
 pyhashlookup = "^1.2.1"
 lief = "^0.13.2"
 ua-parser = "^0.18.0"
 Flask-Login = "^0.6.2"
-har2tree = "^1.21.4"
+har2tree = "^1.21.5"
 passivetotal = "^2.5.9"
 werkzeug = "^2.3.6"
 filetype = "^1.2.0"
@@ -68,7 +68,7 @@ pypandora = "^1.5.0"
 lacuscore = "^1.6.3"
 pylacus = "^1.6.0"
 pyipasnhistory = "^2.1.2"
-publicsuffixlist = "^0.10.0.20230719"
+publicsuffixlist = "^0.10.0.20230730"
 pyfaup = "^1.2"
 chardet = "^5.1.0"
 pysecuritytxt = "^1.1.1"