CIRCL
/
lookyloo
mirror of https://github.com/CIRCL/lookyloo
2
0
Fork 0
Code Issues Releases Wiki Activity

fix: MISP export when the capture is a file.

pull/746/head
Raphaël Vinot 2023-07-30 17:48:16 +02:00
parent 4be8186cc6
commit 35218900ed
4 changed files with 49 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
lookyloo/lookyloo.py
View File

@ -1083,7 +1083,13 @@ class Lookyloo():
if not cache:
return {'error': 'UUID missing in cache, try again later.'}
event = self.misp.export(cache, self.is_public_instance)
# if the file submitted on lookyloo cannot be displayed (PDF), it willbe downloaded.
# In the case, we want to have it as a FileObject in the export
filename, pseudofile = self.get_data(capture_uuid)
if filename:
event = self.misp.export(cache, self.is_public_instance, filename, pseudofile)
else:
event = self.misp.export(cache, self.is_public_instance)
screenshot: MISPAttribute = event.add_attribute('attachment', 'screenshot_landing_page.png',
data=self.get_screenshot(cache.uuid),
disable_correlation=True) # type: ignore

26
lookyloo/modules/misp.py
View File

@ -2,6 +2,8 @@
import logging
import re
from io import BytesIO
from collections import defaultdict
from typing import Any, Dict, List, Optional, Set, Union, TYPE_CHECKING
@ -165,7 +167,9 @@ class MISP():
if hostnodes and hasattr(hostnodes[0], 'resolved_ips'):
obj.add_attributes('ip', *hostnodes[0].resolved_ips)
def export(self, cache: 'CaptureCache', is_public_instance: bool=False) -> MISPEvent:
def export(self, cache: 'CaptureCache', is_public_instance: bool=False,
submitted_filename: Optional[str]=None,
submitted_file: Optional[BytesIO]=None) -> MISPEvent:
'''Export a capture in MISP format. You can POST the return of this method
directly to a MISP instance and it will create an event.'''
public_domain = get_config('generic', 'public_domain')
@ -173,15 +177,32 @@ class MISP():
if cache.url.startswith('file'):
filename = cache.url.rsplit('/', 1)[-1]
event.info = f'Lookyloo Capture ({filename})'
# Create file object as initial
if hasattr(cache.tree.root_hartree.url_tree, 'body'):
# The file could be viewed in the browser
filename = cache.tree.root_hartree.url_tree.name
pseudofile = cache.tree.root_hartree.url_tree.body
elif submitted_filename:
# Impossible to get the file from the HAR.
filename = submitted_filename
pseudofile = submitted_file
else:
raise Exception('We must have a file here.')
initial_file = FileObject(pseudofile=pseudofile, filename=filename)
initial_file.comment = 'This is a capture of a file, rendered in the browser'
initial_obj = event.add_object(initial_file)
else:
event.info = f'Lookyloo Capture ({cache.url})'
initial_url = URLObject(cache.url)
initial_url.comment = 'Submitted URL'
self.__misp_add_ips_to_URLObject(initial_url, cache.tree.root_hartree.hostname_tree)
initial_obj = event.add_object(initial_url)
lookyloo_link: MISPAttribute = event.add_attribute('link', f'https://{public_domain}/tree/{cache.uuid}') # type: ignore
if not is_public_instance:
lookyloo_link.distribution = 0
initial_obj.add_reference(lookyloo_link, 'captured-by', 'Capture on lookyloo')
redirects: List[URLObject] = []
for nb, url in enumerate(cache.redirects):
@ -201,9 +222,6 @@ class MISP():
prec_object.add_reference(u_object, 'redirects-to')
prec_object = u_object
initial_obj = event.add_object(initial_url)
initial_obj.add_reference(lookyloo_link, 'captured-by', 'Capture on lookyloo')
for u_object in redirects:
event.add_object(u_object)
final_redirect = event.objects[-1]

34
poetry.lock generated
View File

@ -908,13 +908,13 @@ tornado = ["tornado (>=0.2)"]
[[package]]
name = "har2tree"
version = "1.21.4"
version = "1.21.5"
description = "HTTP Archive (HAR) to ETE Toolkit generator"
optional = false
python-versions = ">=3.8,<3.12"
files = [
{file = "har2tree-1.21.4-py3-none-any.whl", hash = "sha256:3f26b06b8e52f811cef35bac02a3266c8ce1354e4674353f884ecb2c01ec56a9"},
{file = "har2tree-1.21.4.tar.gz", hash = "sha256:c2941315f24a7c207bf79acd1c2da52b57dce31c4685d34cb63b5ce2fbabdcda"},
{file = "har2tree-1.21.5-py3-none-any.whl", hash = "sha256:21681f2bd04a3c7169b5a571dd2ce21c63c80dee4a085bd62043be0d837fa376"},
{file = "har2tree-1.21.5.tar.gz", hash = "sha256:62fe710cc0a560ac508c2c598246896a19dfea6eeac41c849dae9eeca7dae757"},
]
[package.dependencies]
@ -926,11 +926,11 @@ numpy = [
{version = "1.23.3", markers = "python_version < \"3.10\""},
{version = ">=1.23.4,<2.0.0", markers = "python_version >= \"3.10\""},
]
publicsuffixlist = ">=0.10.0.20230719,<0.11.0.0"
publicsuffixlist = ">=0.10.0.20230730,<0.11.0.0"
w3lib = ">=2.1.1,<3.0.0"
[package.extras]
docs = ["Sphinx (>=7.1.0,<8.0.0)"]
docs = ["Sphinx (>=7.1.1,<8.0.0)"]
[[package]]
name = "hiredis"
@ -1169,21 +1169,21 @@ files = [
[[package]]
name = "jedi"
version = "0.18.2"
version = "0.19.0"
description = "An autocompletion tool for Python that can be used for text editors."
optional = false
python-versions = ">=3.6"
files = [
{file = "jedi-0.18.2-py2.py3-none-any.whl", hash = "sha256:203c1fd9d969ab8f2119ec0a3342e0b49910045abe6af0a3ae83a5764d54639e"},
{file = "jedi-0.18.2.tar.gz", hash = "sha256:bae794c30d07f6d910d32a7048af09b5a39ed740918da923c6b780790ebac612"},
{file = "jedi-0.19.0-py2.py3-none-any.whl", hash = "sha256:cb8ce23fbccff0025e9386b5cf85e892f94c9b822378f8da49970471335ac64e"},
{file = "jedi-0.19.0.tar.gz", hash = "sha256:bcf9894f1753969cbac8022a8c2eaee06bfa3724e4192470aaffe7eb6272b0c4"},
]
[package.dependencies]
parso = ">=0.8.0,<0.9.0"
parso = ">=0.8.3,<0.9.0"
[package.extras]
docs = ["Jinja2 (==2.11.3)", "MarkupSafe (==1.1.1)", "Pygments (==2.8.1)", "alabaster (==0.7.12)", "babel (==2.9.1)", "chardet (==4.0.0)", "commonmark (==0.8.1)", "docutils (==0.17.1)", "future (==0.18.2)", "idna (==2.10)", "imagesize (==1.2.0)", "mock (==1.0.1)", "packaging (==20.9)", "pyparsing (==2.4.7)", "pytz (==2021.1)", "readthedocs-sphinx-ext (==2.1.4)", "recommonmark (==0.5.0)", "requests (==2.25.1)", "six (==1.15.0)", "snowballstemmer (==2.1.0)", "sphinx (==1.8.5)", "sphinx-rtd-theme (==0.4.3)", "sphinxcontrib-serializinghtml (==1.1.4)", "sphinxcontrib-websupport (==1.2.4)", "urllib3 (==1.26.4)"]
qa = ["flake8 (==3.8.3)", "mypy (==0.782)"]
qa = ["flake8 (==5.0.4)", "mypy (==0.971)", "types-setuptools (==67.2.0.1)"]
testing = ["Django (<3.1)", "attrs", "colorama", "docopt", "pytest (<7.0.0)"]
[[package]]
@ -1931,13 +1931,13 @@ files = [
[[package]]
name = "publicsuffixlist"
version = "0.10.0.20230719"
version = "0.10.0.20230730"
description = "publicsuffixlist implement"
optional = false
python-versions = ">=2.6"
files = [
{file = "publicsuffixlist-0.10.0.20230719-py2.py3-none-any.whl", hash = "sha256:62a8d1523726b6a89b1d971b1a687c9b3f1b1daec0ab5ba3bca1ee66bc748254"},
{file = "publicsuffixlist-0.10.0.20230719.tar.gz", hash = "sha256:9b53fdfe06e07e6da5e6575dabe10698ceb82105057dbcdd98865ee1e5fe0a6d"},
{file = "publicsuffixlist-0.10.0.20230730-py2.py3-none-any.whl", hash = "sha256:3a23efc4416474a7ad1db7e85d04f7af24db76a34cec6d82e6e3392b3dd6d8f1"},
{file = "publicsuffixlist-0.10.0.20230730.tar.gz", hash = "sha256:1cd3847ec1ebc02069715630958ad67bdfeb7e7f9f35f699d30fc41298ceedac"},
]
[package.extras]
@ -2458,13 +2458,13 @@ six = "*"
[[package]]
name = "rich"
version = "13.4.2"
version = "13.5.0"
description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal"
optional = false
python-versions = ">=3.7.0"
files = [
{file = "rich-13.4.2-py3-none-any.whl", hash = "sha256:8f87bc7ee54675732fa66a05ebfe489e27264caeeff3728c945d25971b6485ec"},
{file = "rich-13.4.2.tar.gz", hash = "sha256:d653d6bccede5844304c605d5aac802c7cf9621efd700b46c7ec2b51ea914898"},
{file = "rich-13.5.0-py3-none-any.whl", hash = "sha256:996670a7618ccce27c55ba6fc0142e6e343773e11d34c96566a17b71b0e6f179"},
{file = "rich-13.5.0.tar.gz", hash = "sha256:62c81e88dc078d2372858660e3d5566746870133e51321f852ccc20af5c7e7b2"},
]
[package.dependencies]
@ -3146,4 +3146,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p
[metadata]
lock-version = "2.0"
python-versions = ">=3.8,<3.12"
content-hash = "005fd2ad7533d8f017f529ffd2b8844a7c41b40d7f63e8ce599a42595c59375f"
content-hash = "525ee1c64910329ddcf9bb9d13eaac3d2241f02355b131b9e2ee599aed770a3a"

6
pyproject.toml
View File

@ -53,14 +53,14 @@ pytaxonomies = "^1.5.0"
pymisp = {version = "^2.4.173", extras = ["url", "fileobjects"]}
Pillow = "^10.0.0"
flask-restx = "^1.1.0"
rich = "^13.4.2"
rich = "^13.5.0"
pyphishtanklookup = "^1.3.0"
Flask-Cors = "^4.0.0"
pyhashlookup = "^1.2.1"
lief = "^0.13.2"
ua-parser = "^0.18.0"
Flask-Login = "^0.6.2"
har2tree = "^1.21.4"
har2tree = "^1.21.5"
passivetotal = "^2.5.9"
werkzeug = "^2.3.6"
filetype = "^1.2.0"
@ -68,7 +68,7 @@ pypandora = "^1.5.0"
lacuscore = "^1.6.3"
pylacus = "^1.6.0"
pyipasnhistory = "^2.1.2"
publicsuffixlist = "^0.10.0.20230719"
publicsuffixlist = "^0.10.0.20230730"
pyfaup = "^1.2"
chardet = "^5.1.0"
pysecuritytxt = "^1.1.1"