new: Initial implementation of MISP export

pull/135/head
Raphaël Vinot 2020-12-07 20:50:46 +01:00
parent 9b3d338d3c
commit 8eab287721
5 changed files with 198 additions and 11 deletions

View File

@ -3,6 +3,7 @@
"splash_loglevel": "WARNING", "splash_loglevel": "WARNING",
"only_global_lookups": true, "only_global_lookups": true,
"public_instance": false, "public_instance": false,
"public_domain": "lookyloo.myorg.local",
"website_listen_ip": "0.0.0.0", "website_listen_ip": "0.0.0.0",
"website_listen_port": 5100, "website_listen_port": 5100,
"splash_url": "http://127.0.0.1:8050", "splash_url": "http://127.0.0.1:8050",
@ -25,7 +26,6 @@
"from": "Lookyloo <lookyloo@myorg.local>", "from": "Lookyloo <lookyloo@myorg.local>",
"to": "Investigation Team <investigation_unit@myorg.local>", "to": "Investigation Team <investigation_unit@myorg.local>",
"subject": "Capture from Lookyloo to review", "subject": "Capture from Lookyloo to review",
"domain": "lookyloo.myorg.local",
"smtp_host": "localhost", "smtp_host": "localhost",
"smtp_port": "25" "smtp_port": "25"
}, },

View File

@ -20,10 +20,12 @@ from uuid import uuid4
from zipfile import ZipFile from zipfile import ZipFile
import operator import operator
from defang import refang # type: ignore
import dns.resolver import dns.resolver
import dns.rdatatype import dns.rdatatype
from defang import refang # type: ignore
from har2tree import CrawledTree, Har2TreeError, HarFile, HostNode, URLNode from har2tree import CrawledTree, Har2TreeError, HarFile, HostNode, URLNode
from pymisp import MISPEvent
from pymisp.tools import URLObject
from redis import Redis from redis import Redis
from scrapysplashwrapper import crawl from scrapysplashwrapper import crawl
from werkzeug.useragents import UserAgent from werkzeug.useragents import UserAgent
@ -44,6 +46,7 @@ class Lookyloo():
self.logger.setLevel(get_config('generic', 'loglevel')) self.logger.setLevel(get_config('generic', 'loglevel'))
self.indexing = Indexing() self.indexing = Indexing()
self.is_public_instance = get_config('generic', 'public_instance') self.is_public_instance = get_config('generic', 'public_instance')
self.public_domain = get_config('generic', 'public_domain')
self.taxonomies = get_taxonomies() self.taxonomies = get_taxonomies()
self.redis: Redis = Redis(unix_socket_path=get_socket_path('cache'), decode_responses=True) self.redis: Redis = Redis(unix_socket_path=get_socket_path('cache'), decode_responses=True)
@ -485,6 +488,8 @@ class Lookyloo():
return sorted(all_cache, key=operator.itemgetter('timestamp'), reverse=True) return sorted(all_cache, key=operator.itemgetter('timestamp'), reverse=True)
def capture_cache(self, capture_uuid: str) -> Dict[str, Union[str, Path, List]]: def capture_cache(self, capture_uuid: str) -> Dict[str, Union[str, Path, List]]:
"""Get the cache from redis.
NOTE: Doesn't try to build the pickle"""
capture_dir = self.lookup_capture_dir(capture_uuid) capture_dir = self.lookup_capture_dir(capture_uuid)
if not capture_dir: if not capture_dir:
raise MissingUUID(f'Unable to find UUID {capture_uuid} in the cache') raise MissingUUID(f'Unable to find UUID {capture_uuid} in the cache')
@ -579,7 +584,7 @@ class Lookyloo():
body = get_email_template() body = get_email_template()
body = body.format( body = body.format(
recipient=msg['To'].addresses[0].display_name, recipient=msg['To'].addresses[0].display_name,
domain=email_config['domain'], domain=self.public_domain,
uuid=capture_uuid, uuid=capture_uuid,
initial_url=initial_url, initial_url=initial_url,
redirects=redirects, redirects=redirects,
@ -875,6 +880,29 @@ class Lookyloo():
return 'embedded_ressource.bin', blob return 'embedded_ressource.bin', blob
return None return None
def misp_export(self, capture_uuid: str) -> MISPEvent:
cache = self.capture_cache(capture_uuid)
if not cache:
return {'error': 'UUID missing in cache, try again later.'}
if cache['incomplete_redirects']:
self.cache_tree(capture_uuid)
cache = self.capture_cache(capture_uuid)
event = MISPEvent()
event.info = f'Lookyloo Capture ({cache["url"]})'
event.add_attribute('link', f'https://{self.public_domain}/tree/{capture_uuid}')
initial_url = URLObject(cache["url"])
redirects = [URLObject(url) for url in cache['redirects']]
initial_url.add_reference(redirects[0], 'redirects-to')
prec_object = redirects[0]
for u_object in redirects[1:]:
prec_object.add_reference(u_object, 'redirects-to')
prec_object = u_object
event.add_object(initial_url)
for u_object in redirects:
event.add_object(u_object)
return event
def get_hashes(self, tree_uuid: str, hostnode_uuid: Optional[str]=None, urlnode_uuid: Optional[str]=None) -> Set[str]: def get_hashes(self, tree_uuid: str, hostnode_uuid: Optional[str]=None, urlnode_uuid: Optional[str]=None) -> Set[str]:
"""Return hashes of resources. """Return hashes of resources.
Only tree_uuid: All the hashes Only tree_uuid: All the hashes

166
poetry.lock generated
View File

@ -110,7 +110,7 @@ python-versions = "*"
[[package]] [[package]]
name = "certifi" name = "certifi"
version = "2020.11.8" version = "2020.12.5"
description = "Python package for providing Mozilla's CA Bundle." description = "Python package for providing Mozilla's CA Bundle."
category = "main" category = "main"
optional = false optional = false
@ -215,6 +215,20 @@ category = "main"
optional = false optional = false
python-versions = "*" python-versions = "*"
[[package]]
name = "deprecated"
version = "1.2.10"
description = "Python @deprecated decorator to deprecate old python classes, functions or methods."
category = "main"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
[package.dependencies]
wrapt = ">=1.10,<2"
[package.extras]
dev = ["tox", "bumpversion (<1)", "sphinx (<2)", "PyTest (<5)", "PyTest-Cov (<2.6)", "pytest", "pytest-cov"]
[[package]] [[package]]
name = "dnspython" name = "dnspython"
version = "2.0.0" version = "2.0.0"
@ -326,6 +340,21 @@ category = "main"
optional = false optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
[[package]]
name = "importlib-metadata"
version = "3.1.1"
description = "Read metadata from Python packages"
category = "main"
optional = false
python-versions = ">=3.6"
[package.dependencies]
zipp = ">=0.5"
[package.extras]
docs = ["sphinx", "jaraco.packaging (>=3.2)", "rst.linker (>=1.9)"]
testing = ["pytest (>=3.5,!=3.7.3)", "pytest-checkdocs (>=1.2.3)", "pytest-flake8", "pytest-cov", "jaraco.test (>=3.2.0)", "packaging", "pep517", "pyfakefs", "flufl.flake8", "pytest-black (>=0.3.7)", "pytest-mypy", "importlib-resources (>=1.3)"]
[[package]] [[package]]
name = "incremental" name = "incremental"
version = "17.5.0" version = "17.5.0"
@ -443,6 +472,24 @@ category = "main"
optional = false optional = false
python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*"
[[package]]
name = "jsonschema"
version = "3.2.0"
description = "An implementation of JSON Schema validation for Python"
category = "main"
optional = false
python-versions = "*"
[package.dependencies]
attrs = ">=17.4.0"
importlib-metadata = {version = "*", markers = "python_version < \"3.8\""}
pyrsistent = ">=0.14.0"
six = ">=1.11.0"
[package.extras]
format = ["idna", "jsonpointer (>1.13)", "rfc3987", "strict-rfc3339", "webcolors"]
format_nongpl = ["idna", "jsonpointer (>1.13)", "webcolors", "rfc3986-validator (>0.1.0)", "rfc3339-validator"]
[[package]] [[package]]
name = "lxml" name = "lxml"
version = "4.6.2" version = "4.6.2"
@ -633,9 +680,17 @@ python-versions = ">=3.6,<4.0"
[package.dependencies] [package.dependencies]
requests = ">=2.23.0,<3.0.0" requests = ">=2.23.0,<3.0.0"
[[package]]
name = "pyfaup"
version = "1.2"
description = "Python bindings for the faup library"
category = "main"
optional = false
python-versions = "*"
[[package]] [[package]]
name = "pygments" name = "pygments"
version = "2.7.2" version = "2.7.3"
description = "Pygments is a syntax highlighting package written in Python." description = "Pygments is a syntax highlighting package written in Python."
category = "dev" category = "dev"
optional = false optional = false
@ -660,6 +715,30 @@ python-versions = ">=3.6,<4.0"
[package.dependencies] [package.dependencies]
requests = ">=2.22.0,<3.0.0" requests = ">=2.22.0,<3.0.0"
[[package]]
name = "pymisp"
version = "2.4.135.3"
description = "Python API for MISP."
category = "main"
optional = false
python-versions = ">=3.6,<4.0"
[package.dependencies]
deprecated = ">=1.2.7,<2.0.0"
jsonschema = ">=3.2.0,<4.0.0"
pyfaup = {version = ">=1.2,<2.0", optional = true, markers = "extra == \"url\""}
python-dateutil = ">=2.8.1,<3.0.0"
requests = ">=2.22.0,<3.0.0"
[package.extras]
email = ["mail-parser (>=3.12.0,<4.0.0)"]
fileobjects = ["python-magic (>=0.4.15,<0.5.0)", "pydeep (>=0.4,<0.5)", "lief (>=0.10.1,<0.11.0)"]
openioc = ["beautifulsoup4 (>=4.8.2,<5.0.0)"]
virustotal = ["validators (>=0.14.2,<0.15.0)"]
docs = ["sphinx-autodoc-typehints (>=1.10.3,<2.0.0)", "recommonmark (>=0.6.0,<0.7.0)"]
pdfexport = ["reportlab (>=3.5.34,<4.0.0)"]
url = ["pyfaup (>=1.2,<2.0)"]
[[package]] [[package]]
name = "pyopenssl" name = "pyopenssl"
version = "20.0.0" version = "20.0.0"
@ -692,6 +771,14 @@ category = "main"
optional = false optional = false
python-versions = "*" python-versions = "*"
[[package]]
name = "pyrsistent"
version = "0.17.3"
description = "Persistent/Functional/Immutable data structures"
category = "main"
optional = false
python-versions = ">=3.5"
[[package]] [[package]]
name = "pysanejs" name = "pysanejs"
version = "1.4" version = "1.4"
@ -715,6 +802,17 @@ python-versions = ">=3.6,<4.0"
remote = ["requests (>=2.22.0,<3.0.0)"] remote = ["requests (>=2.22.0,<3.0.0)"]
webui = ["flask-nav (>=0.6,<0.7)", "Flask (>=1.1.1,<2.0.0)", "Flask-Bootstrap (>=3.3.7,<4.0.0)", "Flask-WTF (>=0.14.3,<0.15.0)"] webui = ["flask-nav (>=0.6,<0.7)", "Flask (>=1.1.1,<2.0.0)", "Flask-Bootstrap (>=3.3.7,<4.0.0)", "Flask-WTF (>=0.14.3,<0.15.0)"]
[[package]]
name = "python-dateutil"
version = "2.8.1"
description = "Extensions to the standard Python datetime module"
category = "main"
optional = false
python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7"
[package.dependencies]
six = ">=1.5"
[[package]] [[package]]
name = "queuelib" name = "queuelib"
version = "1.5.0" version = "1.5.0"
@ -961,6 +1059,14 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
dev = ["pytest", "pytest-timeout", "coverage", "tox", "sphinx", "pallets-sphinx-themes", "sphinx-issues"] dev = ["pytest", "pytest-timeout", "coverage", "tox", "sphinx", "pallets-sphinx-themes", "sphinx-issues"]
watchdog = ["watchdog"] watchdog = ["watchdog"]
[[package]]
name = "wrapt"
version = "1.12.1"
description = "Module for decorators, wrappers and monkey patching."
category = "main"
optional = false
python-versions = "*"
[[package]] [[package]]
name = "yarl" name = "yarl"
version = "1.6.3" version = "1.6.3"
@ -974,6 +1080,18 @@ idna = ">=2.0"
multidict = ">=4.0" multidict = ">=4.0"
typing-extensions = {version = ">=3.7.4", markers = "python_version < \"3.8\""} typing-extensions = {version = ">=3.7.4", markers = "python_version < \"3.8\""}
[[package]]
name = "zipp"
version = "3.4.0"
description = "Backport of pathlib-compatible object wrapper for zip files"
category = "main"
optional = false
python-versions = ">=3.6"
[package.extras]
docs = ["sphinx", "jaraco.packaging (>=3.2)", "rst.linker (>=1.9)"]
testing = ["pytest (>=3.5,!=3.7.3)", "pytest-checkdocs (>=1.2.3)", "pytest-flake8", "pytest-cov", "jaraco.test (>=3.2.0)", "jaraco.itertools", "func-timeout", "pytest-black (>=0.3.7)", "pytest-mypy"]
[[package]] [[package]]
name = "zope.interface" name = "zope.interface"
version = "5.2.0" version = "5.2.0"
@ -990,7 +1108,7 @@ testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"]
[metadata] [metadata]
lock-version = "1.1" lock-version = "1.1"
python-versions = "^3.7" python-versions = "^3.7"
content-hash = "25146700e27f76c68facb6ee7d4a745ad58c2b4ecc5757fdcda5f11134bfdb29" content-hash = "2d239c1ccb0516874bfa976bbf238f8674167185c7a513df9aa1f35998eda50b"
[metadata.files] [metadata.files]
aiohttp = [ aiohttp = [
@ -1093,8 +1211,8 @@ cchardet = [
{file = "cchardet-2.1.7.tar.gz", hash = "sha256:c428b6336545053c2589f6caf24ea32276c6664cb86db817e03a94c60afa0eaf"}, {file = "cchardet-2.1.7.tar.gz", hash = "sha256:c428b6336545053c2589f6caf24ea32276c6664cb86db817e03a94c60afa0eaf"},
] ]
certifi = [ certifi = [
{file = "certifi-2020.11.8-py2.py3-none-any.whl", hash = "sha256:1f422849db327d534e3d0c5f02a263458c3955ec0aae4ff09b95f195c59f4edd"}, {file = "certifi-2020.12.5-py2.py3-none-any.whl", hash = "sha256:719a74fb9e33b9bd44cc7f3a8d94bc35e4049deebe19ba7d8e108280cfd59830"},
{file = "certifi-2020.11.8.tar.gz", hash = "sha256:f05def092c44fbf25834a51509ef6e631dc19765ab8a57b4e7ab85531f0a9cf4"}, {file = "certifi-2020.12.5.tar.gz", hash = "sha256:1a4995114262bffbc2413b159f2a1a480c969de6e6eb13ee966d470af86af59c"},
] ]
cffi = [ cffi = [
{file = "cffi-1.14.4-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:ebb253464a5d0482b191274f1c8bf00e33f7e0b9c66405fbffc61ed2c839c775"}, {file = "cffi-1.14.4-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:ebb253464a5d0482b191274f1c8bf00e33f7e0b9c66405fbffc61ed2c839c775"},
@ -1186,6 +1304,10 @@ decorator = [
defang = [ defang = [
{file = "defang-0.5.3.tar.gz", hash = "sha256:86aeff658d7cd4c3b61d16089872e1c1f0a1b7b3c64d4ca9525c017caeb284d7"}, {file = "defang-0.5.3.tar.gz", hash = "sha256:86aeff658d7cd4c3b61d16089872e1c1f0a1b7b3c64d4ca9525c017caeb284d7"},
] ]
deprecated = [
{file = "Deprecated-1.2.10-py2.py3-none-any.whl", hash = "sha256:a766c1dccb30c5f6eb2b203f87edd1d8588847709c78589e1521d769addc8218"},
{file = "Deprecated-1.2.10.tar.gz", hash = "sha256:525ba66fb5f90b07169fdd48b6373c18f1ee12728ca277ca44567a367d9d7f74"},
]
dnspython = [ dnspython = [
{file = "dnspython-2.0.0-py3-none-any.whl", hash = "sha256:40bb3c24b9d4ec12500f0124288a65df232a3aa749bb0c39734b782873a2544d"}, {file = "dnspython-2.0.0-py3-none-any.whl", hash = "sha256:40bb3c24b9d4ec12500f0124288a65df232a3aa749bb0c39734b782873a2544d"},
{file = "dnspython-2.0.0.zip", hash = "sha256:044af09374469c3a39eeea1a146e8cac27daec951f1f1f157b1962fc7cb9d1b7"}, {file = "dnspython-2.0.0.zip", hash = "sha256:044af09374469c3a39eeea1a146e8cac27daec951f1f1f157b1962fc7cb9d1b7"},
@ -1221,6 +1343,10 @@ idna = [
{file = "idna-2.10-py2.py3-none-any.whl", hash = "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0"}, {file = "idna-2.10-py2.py3-none-any.whl", hash = "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0"},
{file = "idna-2.10.tar.gz", hash = "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6"}, {file = "idna-2.10.tar.gz", hash = "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6"},
] ]
importlib-metadata = [
{file = "importlib_metadata-3.1.1-py3-none-any.whl", hash = "sha256:6112e21359ef8f344e7178aa5b72dc6e62b38b0d008e6d3cb212c5b84df72013"},
{file = "importlib_metadata-3.1.1.tar.gz", hash = "sha256:b0c2d3b226157ae4517d9625decf63591461c66b3a808c2666d538946519d170"},
]
incremental = [ incremental = [
{file = "incremental-17.5.0-py2.py3-none-any.whl", hash = "sha256:717e12246dddf231a349175f48d74d93e2897244939173b01974ab6661406b9f"}, {file = "incremental-17.5.0-py2.py3-none-any.whl", hash = "sha256:717e12246dddf231a349175f48d74d93e2897244939173b01974ab6661406b9f"},
{file = "incremental-17.5.0.tar.gz", hash = "sha256:7b751696aaf36eebfab537e458929e194460051ccad279c72b755a167eebd4b3"}, {file = "incremental-17.5.0.tar.gz", hash = "sha256:7b751696aaf36eebfab537e458929e194460051ccad279c72b755a167eebd4b3"},
@ -1257,6 +1383,10 @@ jmespath = [
{file = "jmespath-0.10.0-py2.py3-none-any.whl", hash = "sha256:cdf6525904cc597730141d61b36f2e4b8ecc257c420fa2f4549bac2c2d0cb72f"}, {file = "jmespath-0.10.0-py2.py3-none-any.whl", hash = "sha256:cdf6525904cc597730141d61b36f2e4b8ecc257c420fa2f4549bac2c2d0cb72f"},
{file = "jmespath-0.10.0.tar.gz", hash = "sha256:b85d0567b8666149a93172712e68920734333c0ce7e89b78b3e987f71e5ed4f9"}, {file = "jmespath-0.10.0.tar.gz", hash = "sha256:b85d0567b8666149a93172712e68920734333c0ce7e89b78b3e987f71e5ed4f9"},
] ]
jsonschema = [
{file = "jsonschema-3.2.0-py2.py3-none-any.whl", hash = "sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163"},
{file = "jsonschema-3.2.0.tar.gz", hash = "sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a"},
]
lxml = [ lxml = [
{file = "lxml-4.6.2-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:a9d6bc8642e2c67db33f1247a77c53476f3a166e09067c0474facb045756087f"}, {file = "lxml-4.6.2-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:a9d6bc8642e2c67db33f1247a77c53476f3a166e09067c0474facb045756087f"},
{file = "lxml-4.6.2-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:791394449e98243839fa822a637177dd42a95f4883ad3dec2a0ce6ac99fb0a9d"}, {file = "lxml-4.6.2-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:791394449e98243839fa822a637177dd42a95f4883ad3dec2a0ce6ac99fb0a9d"},
@ -1494,9 +1624,13 @@ pyeupi = [
{file = "pyeupi-1.1-py3-none-any.whl", hash = "sha256:a0798a4a52601b0840339449a1bbf2aa2bc180d8f82a979022954e05fcb5bfba"}, {file = "pyeupi-1.1-py3-none-any.whl", hash = "sha256:a0798a4a52601b0840339449a1bbf2aa2bc180d8f82a979022954e05fcb5bfba"},
{file = "pyeupi-1.1.tar.gz", hash = "sha256:2309c61ac2ef0eafabd6e9f32a0078069ffbba0e113ebc6b51cffc1869094472"}, {file = "pyeupi-1.1.tar.gz", hash = "sha256:2309c61ac2ef0eafabd6e9f32a0078069ffbba0e113ebc6b51cffc1869094472"},
] ]
pyfaup = [
{file = "pyfaup-1.2-py2.py3-none-any.whl", hash = "sha256:75f96f7da86ffb5402d3fcc2dbf98a511e792cf9100c159e34cdba8996ddc7f9"},
{file = "pyfaup-1.2.tar.gz", hash = "sha256:5648bc3ebd80239aec927aedfc218c3a6ff36de636cc53822bfeb70b0869b1e7"},
]
pygments = [ pygments = [
{file = "Pygments-2.7.2-py3-none-any.whl", hash = "sha256:88a0bbcd659fcb9573703957c6b9cff9fab7295e6e76db54c9d00ae42df32773"}, {file = "Pygments-2.7.3-py3-none-any.whl", hash = "sha256:f275b6c0909e5dafd2d6269a656aa90fa58ebf4a74f8fcf9053195d226b24a08"},
{file = "Pygments-2.7.2.tar.gz", hash = "sha256:381985fcc551eb9d37c52088a32914e00517e57f4a21609f48141ba08e193fa0"}, {file = "Pygments-2.7.3.tar.gz", hash = "sha256:ccf3acacf3782cbed4a989426012f1c535c9a90d3a7fc3f16d231b9372d2b716"},
] ]
pyhamcrest = [ pyhamcrest = [
{file = "PyHamcrest-2.0.2-py3-none-any.whl", hash = "sha256:7ead136e03655af85069b6f47b23eb7c3e5c221aa9f022a4fbb499f5b7308f29"}, {file = "PyHamcrest-2.0.2-py3-none-any.whl", hash = "sha256:7ead136e03655af85069b6f47b23eb7c3e5c221aa9f022a4fbb499f5b7308f29"},
@ -1506,6 +1640,10 @@ pylookyloo = [
{file = "pylookyloo-1.2.6-py3-none-any.whl", hash = "sha256:0af17024eab09aa2146c731992e1632276780b4645abfcc6a8d7417c20bf14f9"}, {file = "pylookyloo-1.2.6-py3-none-any.whl", hash = "sha256:0af17024eab09aa2146c731992e1632276780b4645abfcc6a8d7417c20bf14f9"},
{file = "pylookyloo-1.2.6.tar.gz", hash = "sha256:39f1d450c515238f2b68723a9ff9922a1e2b2be2451e7270cbbffc2ad98e8244"}, {file = "pylookyloo-1.2.6.tar.gz", hash = "sha256:39f1d450c515238f2b68723a9ff9922a1e2b2be2451e7270cbbffc2ad98e8244"},
] ]
pymisp = [
{file = "pymisp-2.4.135.3-py3-none-any.whl", hash = "sha256:b7bdf04442f46ddcbbcc55866a7ad90f79aa7330c70b1446448dbed2ccdbda80"},
{file = "pymisp-2.4.135.3.tar.gz", hash = "sha256:f0bbdd77358223ba75c9cc40f192c7a2a7a5838bdd08b28381f71d220151ea8a"},
]
pyopenssl = [ pyopenssl = [
{file = "pyOpenSSL-20.0.0-py2.py3-none-any.whl", hash = "sha256:898aefbde331ba718570244c3b01dcddb1b31a3b336613436a45e52e27d9a82d"}, {file = "pyOpenSSL-20.0.0-py2.py3-none-any.whl", hash = "sha256:898aefbde331ba718570244c3b01dcddb1b31a3b336613436a45e52e27d9a82d"},
{file = "pyOpenSSL-20.0.0.tar.gz", hash = "sha256:92f08eccbd73701cf744e8ffd6989aa7842d48cbe3fea8a7c031c5647f590ac5"}, {file = "pyOpenSSL-20.0.0.tar.gz", hash = "sha256:92f08eccbd73701cf744e8ffd6989aa7842d48cbe3fea8a7c031c5647f590ac5"},
@ -1517,6 +1655,9 @@ pyparsing = [
pypydispatcher = [ pypydispatcher = [
{file = "PyPyDispatcher-2.1.2.tar.gz", hash = "sha256:b6bec5dfcff9d2535bca2b23c80eae367b1ac250a645106948d315fcfa9130f2"}, {file = "PyPyDispatcher-2.1.2.tar.gz", hash = "sha256:b6bec5dfcff9d2535bca2b23c80eae367b1ac250a645106948d315fcfa9130f2"},
] ]
pyrsistent = [
{file = "pyrsistent-0.17.3.tar.gz", hash = "sha256:2e636185d9eb976a18a8a8e96efce62f2905fea90041958d8cc2a189756ebf3e"},
]
pysanejs = [ pysanejs = [
{file = "pysanejs-1.4-py3-none-any.whl", hash = "sha256:55d1c14a4e0a8658cefb622b175c62901e33bea24fbc68f2aa420f592ed687e8"}, {file = "pysanejs-1.4-py3-none-any.whl", hash = "sha256:55d1c14a4e0a8658cefb622b175c62901e33bea24fbc68f2aa420f592ed687e8"},
{file = "pysanejs-1.4.tar.gz", hash = "sha256:c3c767acb6329c8c4ff888fb105e63ebc7825c1aba04b9ed44c03cbebc6e8246"}, {file = "pysanejs-1.4.tar.gz", hash = "sha256:c3c767acb6329c8c4ff888fb105e63ebc7825c1aba04b9ed44c03cbebc6e8246"},
@ -1525,6 +1666,10 @@ pytaxonomies = [
{file = "pytaxonomies-1.3-py3-none-any.whl", hash = "sha256:c3c698d2b4030580a4f4ee251a56e816670d678ad433f059920a15792cdf9850"}, {file = "pytaxonomies-1.3-py3-none-any.whl", hash = "sha256:c3c698d2b4030580a4f4ee251a56e816670d678ad433f059920a15792cdf9850"},
{file = "pytaxonomies-1.3.tar.gz", hash = "sha256:c03658aec15b8783f871634b95d8dfde204990ddd6c548f1fa18a1e1f3e3eb36"}, {file = "pytaxonomies-1.3.tar.gz", hash = "sha256:c03658aec15b8783f871634b95d8dfde204990ddd6c548f1fa18a1e1f3e3eb36"},
] ]
python-dateutil = [
{file = "python-dateutil-2.8.1.tar.gz", hash = "sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c"},
{file = "python_dateutil-2.8.1-py2.py3-none-any.whl", hash = "sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a"},
]
queuelib = [ queuelib = [
{file = "queuelib-1.5.0-py2.py3-none-any.whl", hash = "sha256:ff43b5b74b9266f8df4232a8f768dc4d67281a271905e2ed4a3689d4d304cd02"}, {file = "queuelib-1.5.0-py2.py3-none-any.whl", hash = "sha256:ff43b5b74b9266f8df4232a8f768dc4d67281a271905e2ed4a3689d4d304cd02"},
{file = "queuelib-1.5.0.tar.gz", hash = "sha256:42b413295551bdc24ed9376c1a2cd7d0b1b0fa4746b77b27ca2b797a276a1a17"}, {file = "queuelib-1.5.0.tar.gz", hash = "sha256:42b413295551bdc24ed9376c1a2cd7d0b1b0fa4746b77b27ca2b797a276a1a17"},
@ -1641,6 +1786,9 @@ werkzeug = [
{file = "Werkzeug-1.0.1-py2.py3-none-any.whl", hash = "sha256:2de2a5db0baeae7b2d2664949077c2ac63fbd16d98da0ff71837f7d1dea3fd43"}, {file = "Werkzeug-1.0.1-py2.py3-none-any.whl", hash = "sha256:2de2a5db0baeae7b2d2664949077c2ac63fbd16d98da0ff71837f7d1dea3fd43"},
{file = "Werkzeug-1.0.1.tar.gz", hash = "sha256:6c80b1e5ad3665290ea39320b91e1be1e0d5f60652b964a3070216de83d2e47c"}, {file = "Werkzeug-1.0.1.tar.gz", hash = "sha256:6c80b1e5ad3665290ea39320b91e1be1e0d5f60652b964a3070216de83d2e47c"},
] ]
wrapt = [
{file = "wrapt-1.12.1.tar.gz", hash = "sha256:b62ffa81fb85f4332a4f609cab4ac40709470da05643a082ec1eb88e6d9b97d7"},
]
yarl = [ yarl = [
{file = "yarl-1.6.3-cp36-cp36m-macosx_10_14_x86_64.whl", hash = "sha256:0355a701b3998dcd832d0dc47cc5dedf3874f966ac7f870e0f3a6788d802d434"}, {file = "yarl-1.6.3-cp36-cp36m-macosx_10_14_x86_64.whl", hash = "sha256:0355a701b3998dcd832d0dc47cc5dedf3874f966ac7f870e0f3a6788d802d434"},
{file = "yarl-1.6.3-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:bafb450deef6861815ed579c7a6113a879a6ef58aed4c3a4be54400ae8871478"}, {file = "yarl-1.6.3-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:bafb450deef6861815ed579c7a6113a879a6ef58aed4c3a4be54400ae8871478"},
@ -1680,6 +1828,10 @@ yarl = [
{file = "yarl-1.6.3-cp39-cp39-win_amd64.whl", hash = "sha256:4953fb0b4fdb7e08b2f3b3be80a00d28c5c8a2056bb066169de00e6501b986b6"}, {file = "yarl-1.6.3-cp39-cp39-win_amd64.whl", hash = "sha256:4953fb0b4fdb7e08b2f3b3be80a00d28c5c8a2056bb066169de00e6501b986b6"},
{file = "yarl-1.6.3.tar.gz", hash = "sha256:8a9066529240171b68893d60dca86a763eae2139dd42f42106b03cf4b426bf10"}, {file = "yarl-1.6.3.tar.gz", hash = "sha256:8a9066529240171b68893d60dca86a763eae2139dd42f42106b03cf4b426bf10"},
] ]
zipp = [
{file = "zipp-3.4.0-py3-none-any.whl", hash = "sha256:102c24ef8f171fd729d46599845e95c7ab894a4cf45f5de11a44cc7444fb1108"},
{file = "zipp-3.4.0.tar.gz", hash = "sha256:ed5eee1974372595f9e416cc7bbeeb12335201d8081ca8a0743c954d4446e5cb"},
]
"zope.interface" = [ "zope.interface" = [
{file = "zope.interface-5.2.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:518950fe6a5d56f94ba125107895f938a4f34f704c658986eae8255edb41163b"}, {file = "zope.interface-5.2.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:518950fe6a5d56f94ba125107895f938a4f34f704c658986eae8255edb41163b"},
{file = "zope.interface-5.2.0-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:6278c080d4afffc9016e14325f8734456831124e8c12caa754fd544435c08386"}, {file = "zope.interface-5.2.0-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:6278c080d4afffc9016e14325f8734456831124e8c12caa754fd544435c08386"},

View File

@ -50,6 +50,7 @@ har2tree = "^1.2.10"
pylookyloo = "^1.2" pylookyloo = "^1.2"
dnspython = "^2.0.0" dnspython = "^2.0.0"
pytaxonomies = "^1.3" pytaxonomies = "^1.3"
pymisp = {version = "^2.4.135", extras = ["url"]}
[tool.poetry.dev-dependencies] [tool.poetry.dev-dependencies]
mypy = "^0.790" mypy = "^0.790"

View File

@ -660,7 +660,7 @@ def json_redirects(tree_uuid: str):
return to_return return to_return
if cache['incomplete_redirects']: if cache['incomplete_redirects']:
# Trigger tree build, get all redirects # Trigger tree build, get all redirects
lookyloo.load_tree(tree_uuid) lookyloo.cache_tree(tree_uuid)
cache = lookyloo.capture_cache(tree_uuid) cache = lookyloo.capture_cache(tree_uuid)
if cache: if cache:
to_return['response']['redirects'] = cache['redirects'] to_return['response']['redirects'] = cache['redirects']
@ -670,6 +670,12 @@ def json_redirects(tree_uuid: str):
return jsonify(to_return) return jsonify(to_return)
@app.route('/json/<string:tree_uuid>/misp_export', methods=['GET'])
def misp_export(tree_uuid: str):
event = lookyloo.misp_export(tree_uuid)
return Response(event.to_json(indent=2), mimetype='application/json')
@app.route('/json/hash_info/<h>', methods=['GET']) @app.route('/json/hash_info/<h>', methods=['GET'])
def json_hash_info(h: str): def json_hash_info(h: str):
details, body = lookyloo.get_body_hash_full(h) details, body = lookyloo.get_body_hash_full(h)