new: Initial implementation of MISP export

pull/135/head
Raphaël Vinot 2020-12-07 20:50:46 +01:00
parent 9b3d338d3c
commit 8eab287721
5 changed files with 198 additions and 11 deletions

View File

@ -3,6 +3,7 @@
"splash_loglevel": "WARNING",
"only_global_lookups": true,
"public_instance": false,
"public_domain": "lookyloo.myorg.local",
"website_listen_ip": "0.0.0.0",
"website_listen_port": 5100,
"splash_url": "http://127.0.0.1:8050",
@ -25,7 +26,6 @@
"from": "Lookyloo <lookyloo@myorg.local>",
"to": "Investigation Team <investigation_unit@myorg.local>",
"subject": "Capture from Lookyloo to review",
"domain": "lookyloo.myorg.local",
"smtp_host": "localhost",
"smtp_port": "25"
},

View File

@ -20,10 +20,12 @@ from uuid import uuid4
from zipfile import ZipFile
import operator
from defang import refang # type: ignore
import dns.resolver
import dns.rdatatype
from defang import refang # type: ignore
from har2tree import CrawledTree, Har2TreeError, HarFile, HostNode, URLNode
from pymisp import MISPEvent
from pymisp.tools import URLObject
from redis import Redis
from scrapysplashwrapper import crawl
from werkzeug.useragents import UserAgent
@ -44,6 +46,7 @@ class Lookyloo():
self.logger.setLevel(get_config('generic', 'loglevel'))
self.indexing = Indexing()
self.is_public_instance = get_config('generic', 'public_instance')
self.public_domain = get_config('generic', 'public_domain')
self.taxonomies = get_taxonomies()
self.redis: Redis = Redis(unix_socket_path=get_socket_path('cache'), decode_responses=True)
@ -485,6 +488,8 @@ class Lookyloo():
return sorted(all_cache, key=operator.itemgetter('timestamp'), reverse=True)
def capture_cache(self, capture_uuid: str) -> Dict[str, Union[str, Path, List]]:
"""Get the cache from redis.
NOTE: Doesn't try to build the pickle"""
capture_dir = self.lookup_capture_dir(capture_uuid)
if not capture_dir:
raise MissingUUID(f'Unable to find UUID {capture_uuid} in the cache')
@ -579,7 +584,7 @@ class Lookyloo():
body = get_email_template()
body = body.format(
recipient=msg['To'].addresses[0].display_name,
domain=email_config['domain'],
domain=self.public_domain,
uuid=capture_uuid,
initial_url=initial_url,
redirects=redirects,
@ -875,6 +880,29 @@ class Lookyloo():
return 'embedded_ressource.bin', blob
return None
def misp_export(self, capture_uuid: str) -> MISPEvent:
cache = self.capture_cache(capture_uuid)
if not cache:
return {'error': 'UUID missing in cache, try again later.'}
if cache['incomplete_redirects']:
self.cache_tree(capture_uuid)
cache = self.capture_cache(capture_uuid)
event = MISPEvent()
event.info = f'Lookyloo Capture ({cache["url"]})'
event.add_attribute('link', f'https://{self.public_domain}/tree/{capture_uuid}')
initial_url = URLObject(cache["url"])
redirects = [URLObject(url) for url in cache['redirects']]
initial_url.add_reference(redirects[0], 'redirects-to')
prec_object = redirects[0]
for u_object in redirects[1:]:
prec_object.add_reference(u_object, 'redirects-to')
prec_object = u_object
event.add_object(initial_url)
for u_object in redirects:
event.add_object(u_object)
return event
def get_hashes(self, tree_uuid: str, hostnode_uuid: Optional[str]=None, urlnode_uuid: Optional[str]=None) -> Set[str]:
"""Return hashes of resources.
Only tree_uuid: All the hashes

166
poetry.lock generated
View File

@ -110,7 +110,7 @@ python-versions = "*"
[[package]]
name = "certifi"
version = "2020.11.8"
version = "2020.12.5"
description = "Python package for providing Mozilla's CA Bundle."
category = "main"
optional = false
@ -215,6 +215,20 @@ category = "main"
optional = false
python-versions = "*"
[[package]]
name = "deprecated"
version = "1.2.10"
description = "Python @deprecated decorator to deprecate old python classes, functions or methods."
category = "main"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
[package.dependencies]
wrapt = ">=1.10,<2"
[package.extras]
dev = ["tox", "bumpversion (<1)", "sphinx (<2)", "PyTest (<5)", "PyTest-Cov (<2.6)", "pytest", "pytest-cov"]
[[package]]
name = "dnspython"
version = "2.0.0"
@ -326,6 +340,21 @@ category = "main"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
[[package]]
name = "importlib-metadata"
version = "3.1.1"
description = "Read metadata from Python packages"
category = "main"
optional = false
python-versions = ">=3.6"
[package.dependencies]
zipp = ">=0.5"
[package.extras]
docs = ["sphinx", "jaraco.packaging (>=3.2)", "rst.linker (>=1.9)"]
testing = ["pytest (>=3.5,!=3.7.3)", "pytest-checkdocs (>=1.2.3)", "pytest-flake8", "pytest-cov", "jaraco.test (>=3.2.0)", "packaging", "pep517", "pyfakefs", "flufl.flake8", "pytest-black (>=0.3.7)", "pytest-mypy", "importlib-resources (>=1.3)"]
[[package]]
name = "incremental"
version = "17.5.0"
@ -443,6 +472,24 @@ category = "main"
optional = false
python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*"
[[package]]
name = "jsonschema"
version = "3.2.0"
description = "An implementation of JSON Schema validation for Python"
category = "main"
optional = false
python-versions = "*"
[package.dependencies]
attrs = ">=17.4.0"
importlib-metadata = {version = "*", markers = "python_version < \"3.8\""}
pyrsistent = ">=0.14.0"
six = ">=1.11.0"
[package.extras]
format = ["idna", "jsonpointer (>1.13)", "rfc3987", "strict-rfc3339", "webcolors"]
format_nongpl = ["idna", "jsonpointer (>1.13)", "webcolors", "rfc3986-validator (>0.1.0)", "rfc3339-validator"]
[[package]]
name = "lxml"
version = "4.6.2"
@ -633,9 +680,17 @@ python-versions = ">=3.6,<4.0"
[package.dependencies]
requests = ">=2.23.0,<3.0.0"
[[package]]
name = "pyfaup"
version = "1.2"
description = "Python bindings for the faup library"
category = "main"
optional = false
python-versions = "*"
[[package]]
name = "pygments"
version = "2.7.2"
version = "2.7.3"
description = "Pygments is a syntax highlighting package written in Python."
category = "dev"
optional = false
@ -660,6 +715,30 @@ python-versions = ">=3.6,<4.0"
[package.dependencies]
requests = ">=2.22.0,<3.0.0"
[[package]]
name = "pymisp"
version = "2.4.135.3"
description = "Python API for MISP."
category = "main"
optional = false
python-versions = ">=3.6,<4.0"
[package.dependencies]
deprecated = ">=1.2.7,<2.0.0"
jsonschema = ">=3.2.0,<4.0.0"
pyfaup = {version = ">=1.2,<2.0", optional = true, markers = "extra == \"url\""}
python-dateutil = ">=2.8.1,<3.0.0"
requests = ">=2.22.0,<3.0.0"
[package.extras]
email = ["mail-parser (>=3.12.0,<4.0.0)"]
fileobjects = ["python-magic (>=0.4.15,<0.5.0)", "pydeep (>=0.4,<0.5)", "lief (>=0.10.1,<0.11.0)"]
openioc = ["beautifulsoup4 (>=4.8.2,<5.0.0)"]
virustotal = ["validators (>=0.14.2,<0.15.0)"]
docs = ["sphinx-autodoc-typehints (>=1.10.3,<2.0.0)", "recommonmark (>=0.6.0,<0.7.0)"]
pdfexport = ["reportlab (>=3.5.34,<4.0.0)"]
url = ["pyfaup (>=1.2,<2.0)"]
[[package]]
name = "pyopenssl"
version = "20.0.0"
@ -692,6 +771,14 @@ category = "main"
optional = false
python-versions = "*"
[[package]]
name = "pyrsistent"
version = "0.17.3"
description = "Persistent/Functional/Immutable data structures"
category = "main"
optional = false
python-versions = ">=3.5"
[[package]]
name = "pysanejs"
version = "1.4"
@ -715,6 +802,17 @@ python-versions = ">=3.6,<4.0"
remote = ["requests (>=2.22.0,<3.0.0)"]
webui = ["flask-nav (>=0.6,<0.7)", "Flask (>=1.1.1,<2.0.0)", "Flask-Bootstrap (>=3.3.7,<4.0.0)", "Flask-WTF (>=0.14.3,<0.15.0)"]
[[package]]
name = "python-dateutil"
version = "2.8.1"
description = "Extensions to the standard Python datetime module"
category = "main"
optional = false
python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7"
[package.dependencies]
six = ">=1.5"
[[package]]
name = "queuelib"
version = "1.5.0"
@ -961,6 +1059,14 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
dev = ["pytest", "pytest-timeout", "coverage", "tox", "sphinx", "pallets-sphinx-themes", "sphinx-issues"]
watchdog = ["watchdog"]
[[package]]
name = "wrapt"
version = "1.12.1"
description = "Module for decorators, wrappers and monkey patching."
category = "main"
optional = false
python-versions = "*"
[[package]]
name = "yarl"
version = "1.6.3"
@ -974,6 +1080,18 @@ idna = ">=2.0"
multidict = ">=4.0"
typing-extensions = {version = ">=3.7.4", markers = "python_version < \"3.8\""}
[[package]]
name = "zipp"
version = "3.4.0"
description = "Backport of pathlib-compatible object wrapper for zip files"
category = "main"
optional = false
python-versions = ">=3.6"
[package.extras]
docs = ["sphinx", "jaraco.packaging (>=3.2)", "rst.linker (>=1.9)"]
testing = ["pytest (>=3.5,!=3.7.3)", "pytest-checkdocs (>=1.2.3)", "pytest-flake8", "pytest-cov", "jaraco.test (>=3.2.0)", "jaraco.itertools", "func-timeout", "pytest-black (>=0.3.7)", "pytest-mypy"]
[[package]]
name = "zope.interface"
version = "5.2.0"
@ -990,7 +1108,7 @@ testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"]
[metadata]
lock-version = "1.1"
python-versions = "^3.7"
content-hash = "25146700e27f76c68facb6ee7d4a745ad58c2b4ecc5757fdcda5f11134bfdb29"
content-hash = "2d239c1ccb0516874bfa976bbf238f8674167185c7a513df9aa1f35998eda50b"
[metadata.files]
aiohttp = [
@ -1093,8 +1211,8 @@ cchardet = [
{file = "cchardet-2.1.7.tar.gz", hash = "sha256:c428b6336545053c2589f6caf24ea32276c6664cb86db817e03a94c60afa0eaf"},
]
certifi = [
{file = "certifi-2020.11.8-py2.py3-none-any.whl", hash = "sha256:1f422849db327d534e3d0c5f02a263458c3955ec0aae4ff09b95f195c59f4edd"},
{file = "certifi-2020.11.8.tar.gz", hash = "sha256:f05def092c44fbf25834a51509ef6e631dc19765ab8a57b4e7ab85531f0a9cf4"},
{file = "certifi-2020.12.5-py2.py3-none-any.whl", hash = "sha256:719a74fb9e33b9bd44cc7f3a8d94bc35e4049deebe19ba7d8e108280cfd59830"},
{file = "certifi-2020.12.5.tar.gz", hash = "sha256:1a4995114262bffbc2413b159f2a1a480c969de6e6eb13ee966d470af86af59c"},
]
cffi = [
{file = "cffi-1.14.4-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:ebb253464a5d0482b191274f1c8bf00e33f7e0b9c66405fbffc61ed2c839c775"},
@ -1186,6 +1304,10 @@ decorator = [
defang = [
{file = "defang-0.5.3.tar.gz", hash = "sha256:86aeff658d7cd4c3b61d16089872e1c1f0a1b7b3c64d4ca9525c017caeb284d7"},
]
deprecated = [
{file = "Deprecated-1.2.10-py2.py3-none-any.whl", hash = "sha256:a766c1dccb30c5f6eb2b203f87edd1d8588847709c78589e1521d769addc8218"},
{file = "Deprecated-1.2.10.tar.gz", hash = "sha256:525ba66fb5f90b07169fdd48b6373c18f1ee12728ca277ca44567a367d9d7f74"},
]
dnspython = [
{file = "dnspython-2.0.0-py3-none-any.whl", hash = "sha256:40bb3c24b9d4ec12500f0124288a65df232a3aa749bb0c39734b782873a2544d"},
{file = "dnspython-2.0.0.zip", hash = "sha256:044af09374469c3a39eeea1a146e8cac27daec951f1f1f157b1962fc7cb9d1b7"},
@ -1221,6 +1343,10 @@ idna = [
{file = "idna-2.10-py2.py3-none-any.whl", hash = "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0"},
{file = "idna-2.10.tar.gz", hash = "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6"},
]
importlib-metadata = [
{file = "importlib_metadata-3.1.1-py3-none-any.whl", hash = "sha256:6112e21359ef8f344e7178aa5b72dc6e62b38b0d008e6d3cb212c5b84df72013"},
{file = "importlib_metadata-3.1.1.tar.gz", hash = "sha256:b0c2d3b226157ae4517d9625decf63591461c66b3a808c2666d538946519d170"},
]
incremental = [
{file = "incremental-17.5.0-py2.py3-none-any.whl", hash = "sha256:717e12246dddf231a349175f48d74d93e2897244939173b01974ab6661406b9f"},
{file = "incremental-17.5.0.tar.gz", hash = "sha256:7b751696aaf36eebfab537e458929e194460051ccad279c72b755a167eebd4b3"},
@ -1257,6 +1383,10 @@ jmespath = [
{file = "jmespath-0.10.0-py2.py3-none-any.whl", hash = "sha256:cdf6525904cc597730141d61b36f2e4b8ecc257c420fa2f4549bac2c2d0cb72f"},
{file = "jmespath-0.10.0.tar.gz", hash = "sha256:b85d0567b8666149a93172712e68920734333c0ce7e89b78b3e987f71e5ed4f9"},
]
jsonschema = [
{file = "jsonschema-3.2.0-py2.py3-none-any.whl", hash = "sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163"},
{file = "jsonschema-3.2.0.tar.gz", hash = "sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a"},
]
lxml = [
{file = "lxml-4.6.2-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:a9d6bc8642e2c67db33f1247a77c53476f3a166e09067c0474facb045756087f"},
{file = "lxml-4.6.2-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:791394449e98243839fa822a637177dd42a95f4883ad3dec2a0ce6ac99fb0a9d"},
@ -1494,9 +1624,13 @@ pyeupi = [
{file = "pyeupi-1.1-py3-none-any.whl", hash = "sha256:a0798a4a52601b0840339449a1bbf2aa2bc180d8f82a979022954e05fcb5bfba"},
{file = "pyeupi-1.1.tar.gz", hash = "sha256:2309c61ac2ef0eafabd6e9f32a0078069ffbba0e113ebc6b51cffc1869094472"},
]
pyfaup = [
{file = "pyfaup-1.2-py2.py3-none-any.whl", hash = "sha256:75f96f7da86ffb5402d3fcc2dbf98a511e792cf9100c159e34cdba8996ddc7f9"},
{file = "pyfaup-1.2.tar.gz", hash = "sha256:5648bc3ebd80239aec927aedfc218c3a6ff36de636cc53822bfeb70b0869b1e7"},
]
pygments = [
{file = "Pygments-2.7.2-py3-none-any.whl", hash = "sha256:88a0bbcd659fcb9573703957c6b9cff9fab7295e6e76db54c9d00ae42df32773"},
{file = "Pygments-2.7.2.tar.gz", hash = "sha256:381985fcc551eb9d37c52088a32914e00517e57f4a21609f48141ba08e193fa0"},
{file = "Pygments-2.7.3-py3-none-any.whl", hash = "sha256:f275b6c0909e5dafd2d6269a656aa90fa58ebf4a74f8fcf9053195d226b24a08"},
{file = "Pygments-2.7.3.tar.gz", hash = "sha256:ccf3acacf3782cbed4a989426012f1c535c9a90d3a7fc3f16d231b9372d2b716"},
]
pyhamcrest = [
{file = "PyHamcrest-2.0.2-py3-none-any.whl", hash = "sha256:7ead136e03655af85069b6f47b23eb7c3e5c221aa9f022a4fbb499f5b7308f29"},
@ -1506,6 +1640,10 @@ pylookyloo = [
{file = "pylookyloo-1.2.6-py3-none-any.whl", hash = "sha256:0af17024eab09aa2146c731992e1632276780b4645abfcc6a8d7417c20bf14f9"},
{file = "pylookyloo-1.2.6.tar.gz", hash = "sha256:39f1d450c515238f2b68723a9ff9922a1e2b2be2451e7270cbbffc2ad98e8244"},
]
pymisp = [
{file = "pymisp-2.4.135.3-py3-none-any.whl", hash = "sha256:b7bdf04442f46ddcbbcc55866a7ad90f79aa7330c70b1446448dbed2ccdbda80"},
{file = "pymisp-2.4.135.3.tar.gz", hash = "sha256:f0bbdd77358223ba75c9cc40f192c7a2a7a5838bdd08b28381f71d220151ea8a"},
]
pyopenssl = [
{file = "pyOpenSSL-20.0.0-py2.py3-none-any.whl", hash = "sha256:898aefbde331ba718570244c3b01dcddb1b31a3b336613436a45e52e27d9a82d"},
{file = "pyOpenSSL-20.0.0.tar.gz", hash = "sha256:92f08eccbd73701cf744e8ffd6989aa7842d48cbe3fea8a7c031c5647f590ac5"},
@ -1517,6 +1655,9 @@ pyparsing = [
pypydispatcher = [
{file = "PyPyDispatcher-2.1.2.tar.gz", hash = "sha256:b6bec5dfcff9d2535bca2b23c80eae367b1ac250a645106948d315fcfa9130f2"},
]
pyrsistent = [
{file = "pyrsistent-0.17.3.tar.gz", hash = "sha256:2e636185d9eb976a18a8a8e96efce62f2905fea90041958d8cc2a189756ebf3e"},
]
pysanejs = [
{file = "pysanejs-1.4-py3-none-any.whl", hash = "sha256:55d1c14a4e0a8658cefb622b175c62901e33bea24fbc68f2aa420f592ed687e8"},
{file = "pysanejs-1.4.tar.gz", hash = "sha256:c3c767acb6329c8c4ff888fb105e63ebc7825c1aba04b9ed44c03cbebc6e8246"},
@ -1525,6 +1666,10 @@ pytaxonomies = [
{file = "pytaxonomies-1.3-py3-none-any.whl", hash = "sha256:c3c698d2b4030580a4f4ee251a56e816670d678ad433f059920a15792cdf9850"},
{file = "pytaxonomies-1.3.tar.gz", hash = "sha256:c03658aec15b8783f871634b95d8dfde204990ddd6c548f1fa18a1e1f3e3eb36"},
]
python-dateutil = [
{file = "python-dateutil-2.8.1.tar.gz", hash = "sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c"},
{file = "python_dateutil-2.8.1-py2.py3-none-any.whl", hash = "sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a"},
]
queuelib = [
{file = "queuelib-1.5.0-py2.py3-none-any.whl", hash = "sha256:ff43b5b74b9266f8df4232a8f768dc4d67281a271905e2ed4a3689d4d304cd02"},
{file = "queuelib-1.5.0.tar.gz", hash = "sha256:42b413295551bdc24ed9376c1a2cd7d0b1b0fa4746b77b27ca2b797a276a1a17"},
@ -1641,6 +1786,9 @@ werkzeug = [
{file = "Werkzeug-1.0.1-py2.py3-none-any.whl", hash = "sha256:2de2a5db0baeae7b2d2664949077c2ac63fbd16d98da0ff71837f7d1dea3fd43"},
{file = "Werkzeug-1.0.1.tar.gz", hash = "sha256:6c80b1e5ad3665290ea39320b91e1be1e0d5f60652b964a3070216de83d2e47c"},
]
wrapt = [
{file = "wrapt-1.12.1.tar.gz", hash = "sha256:b62ffa81fb85f4332a4f609cab4ac40709470da05643a082ec1eb88e6d9b97d7"},
]
yarl = [
{file = "yarl-1.6.3-cp36-cp36m-macosx_10_14_x86_64.whl", hash = "sha256:0355a701b3998dcd832d0dc47cc5dedf3874f966ac7f870e0f3a6788d802d434"},
{file = "yarl-1.6.3-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:bafb450deef6861815ed579c7a6113a879a6ef58aed4c3a4be54400ae8871478"},
@ -1680,6 +1828,10 @@ yarl = [
{file = "yarl-1.6.3-cp39-cp39-win_amd64.whl", hash = "sha256:4953fb0b4fdb7e08b2f3b3be80a00d28c5c8a2056bb066169de00e6501b986b6"},
{file = "yarl-1.6.3.tar.gz", hash = "sha256:8a9066529240171b68893d60dca86a763eae2139dd42f42106b03cf4b426bf10"},
]
zipp = [
{file = "zipp-3.4.0-py3-none-any.whl", hash = "sha256:102c24ef8f171fd729d46599845e95c7ab894a4cf45f5de11a44cc7444fb1108"},
{file = "zipp-3.4.0.tar.gz", hash = "sha256:ed5eee1974372595f9e416cc7bbeeb12335201d8081ca8a0743c954d4446e5cb"},
]
"zope.interface" = [
{file = "zope.interface-5.2.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:518950fe6a5d56f94ba125107895f938a4f34f704c658986eae8255edb41163b"},
{file = "zope.interface-5.2.0-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:6278c080d4afffc9016e14325f8734456831124e8c12caa754fd544435c08386"},

View File

@ -50,6 +50,7 @@ har2tree = "^1.2.10"
pylookyloo = "^1.2"
dnspython = "^2.0.0"
pytaxonomies = "^1.3"
pymisp = {version = "^2.4.135", extras = ["url"]}
[tool.poetry.dev-dependencies]
mypy = "^0.790"

View File

@ -660,7 +660,7 @@ def json_redirects(tree_uuid: str):
return to_return
if cache['incomplete_redirects']:
# Trigger tree build, get all redirects
lookyloo.load_tree(tree_uuid)
lookyloo.cache_tree(tree_uuid)
cache = lookyloo.capture_cache(tree_uuid)
if cache:
to_return['response']['redirects'] = cache['redirects']
@ -670,6 +670,12 @@ def json_redirects(tree_uuid: str):
return jsonify(to_return)
@app.route('/json/<string:tree_uuid>/misp_export', methods=['GET'])
def misp_export(tree_uuid: str):
event = lookyloo.misp_export(tree_uuid)
return Response(event.to_json(indent=2), mimetype='application/json')
@app.route('/json/hash_info/<h>', methods=['GET'])
def json_hash_info(h: str):
details, body = lookyloo.get_body_hash_full(h)