From a0c906d3dc2df4254109ad8d0550e42fa1f3d91f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 20 Apr 2020 16:41:42 +0200 Subject: [PATCH] new: Initial view for 3rd party modules --- lookyloo/lookyloo.py | 26 ++++++++++++ lookyloo/modules.py | 64 +++++++++++++++++++++------- poetry.lock | 75 +++++++++++++++++++++++++-------- website/web/__init__.py | 36 ++++++++++++++++ website/web/templates/tree.html | 38 +++++++++++++++++ 5 files changed, 206 insertions(+), 33 deletions(-) diff --git a/lookyloo/lookyloo.py b/lookyloo/lookyloo.py index 76d8846f..eec555fb 100644 --- a/lookyloo/lookyloo.py +++ b/lookyloo/lookyloo.py @@ -97,6 +97,32 @@ class Lookyloo(): sample_config = json.load(_c) return sample_config[entry] + def trigger_modules(self, capture_dir: Path, force: bool=False) -> None: + # We need the pickle + ct = self._load_pickle(capture_dir / 'tree.pickle') + if not ct: + self.logger.warning('Unable to trigger the modules unless the tree ({capture_dir}) is cached.') + return + + if hasattr(self, 'vt') and self.vt.available: + if ct.redirects: + for redirect in ct.redirects: + self.vt.url_lookup(redirect, force) + else: + self.vt.url_lookup(ct.root_hartree.har.first_url, force) + + def get_modules_responses(self, capture_dir: Path) -> Dict: + ct = self._load_pickle(capture_dir / 'tree.pickle') + to_return = {} + if hasattr(self, 'vt') and self.vt.available: + to_return['vt'] = {} + if ct.redirects: + for redirect in ct.redirects: + to_return['vt'][redirect] = self.vt.get_url_lookup(redirect) + else: + to_return['vt'][ct.root_hartree.har.first_url] = self.vt.get_url_lookup(ct.root_hartree.har.first_url) + return to_return + def _set_capture_cache(self, capture_dir: Path, force: bool=False) -> None: if force or not self.redis.exists(str(capture_dir)): # (re)build cache diff --git a/lookyloo/modules.py b/lookyloo/modules.py index 84d7d3b2..22e57231 100644 --- a/lookyloo/modules.py +++ b/lookyloo/modules.py @@ -1,10 +1,12 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -from typing import Dict, Any +from typing import Dict, Any, Optional from datetime import date import hashlib import json +from pathlib import Path +import time from .helpers import get_homedir @@ -32,25 +34,57 @@ class VirusTotal(): if hasattr(self, 'client'): self.client.close() - def url_lookup(self, url: str): + def __get_cache_directory(self, url: str) -> Path: + url_id = vt.url_id(url) + m = hashlib.md5() + m.update(url_id.encode()) + return self.storage_dir_vt / m.hexdigest() + + def get_url_lookup(self, url: str) -> Optional[Dict]: + url_storage_dir = self.__get_cache_directory(url) + if not url_storage_dir.exists(): + return None + cached_entries = sorted(url_storage_dir.glob('*'), reverse=True) + if not cached_entries: + return None + + with cached_entries[0].open() as f: + return json.load(f) + + def url_lookup(self, url: str, force: bool=False): + '''Lookup an URL on VT + Note: force means 2 things: + * (re)scan of the URL + * re fetch the object from VT even if we already did it today + + Note: the URL will only be sent for scan if autosubmit is set to true in the config + ''' if not self.available: raise ConfigError('VirusTotal not available, probably no API key') url_id = vt.url_id(url) - m = hashlib.md5() - m.update(url_id.encode()) - - url_storage_dir = self.storage_dir_vt / m.hexdigest() + url_storage_dir = self.__get_cache_directory(url) url_storage_dir.mkdir(parents=True, exist_ok=True) - vt_file = url_storage_dir / date.today().isoformat() - if vt_file.exists(): + + scan_requested = False + if self.autosubmit and force: + self.client.scan_url(url) + scan_requested = True + + if not force and vt_file.exists(): return - try: - url_information = self.client.get_object(f"/urls/{url_id}") - with vt_file.open('w') as _f: - json.dump(url_information.to_dict(), _f) - except vt.APIError as e: - if self.autosubmit and e.code == 'NotFoundError': - self.client.scan_url(url) + for i in range(3): + try: + url_information = self.client.get_object(f"/urls/{url_id}") + with vt_file.open('w') as _f: + json.dump(url_information.to_dict(), _f) + break + except vt.APIError as e: + if not self.autosubmit: + break + if not scan_requested and e.code == 'NotFoundError': + self.client.scan_url(url) + scan_requested = True + time.sleep(5) diff --git a/poetry.lock b/poetry.lock index 2f302510..cee5d89a 100644 --- a/poetry.lock +++ b/poetry.lock @@ -409,15 +409,15 @@ category = "dev" description = "An autocompletion tool for Python that can be used for text editors." name = "jedi" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -version = "0.16.0" +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +version = "0.17.0" [package.dependencies] -parso = ">=0.5.2" +parso = ">=0.7.0" [package.extras] qa = ["flake8 (3.7.9)"] -testing = ["colorama (0.4.1)", "docopt", "pytest (>=3.9.0,<5.0.0)"] +testing = ["colorama", "docopt", "pytest (>=3.9.0,<5.0.0)"] [[package]] category = "main" @@ -425,7 +425,7 @@ description = "A very fast and expressive template engine." name = "jinja2" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" -version = "2.11.1" +version = "2.11.2" [package.dependencies] MarkupSafe = ">=0.23" @@ -530,7 +530,7 @@ description = "A Python Parser" name = "parso" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -version = "0.6.2" +version = "0.7.0" [package.extras] testing = ["docopt", "pytest (>=3.0.7)"] @@ -979,11 +979,11 @@ description = "HTTP library with thread-safe connection pooling, file post, and name = "urllib3" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4" -version = "1.25.8" +version = "1.25.9" [package.extras] brotli = ["brotlipy (>=0.6.0)"] -secure = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "certifi", "ipaddress"] +secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "pyOpenSSL (>=0.14)", "ipaddress"] socks = ["PySocks (>=1.5.6,<1.5.7 || >1.5.7,<2.0)"] [[package]] @@ -1062,7 +1062,7 @@ description = "Interfaces for Python" name = "zope.interface" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" -version = "5.0.2" +version = "5.1.0" [package.dependencies] setuptools = "*" @@ -1292,12 +1292,12 @@ itsdangerous = [ {file = "itsdangerous-1.1.0.tar.gz", hash = "sha256:321b033d07f2a4136d3ec762eac9f16a10ccd60f53c0c91af90217ace7ba1f19"}, ] jedi = [ - {file = "jedi-0.16.0-py2.py3-none-any.whl", hash = "sha256:b4f4052551025c6b0b0b193b29a6ff7bdb74c52450631206c262aef9f7159ad2"}, - {file = "jedi-0.16.0.tar.gz", hash = "sha256:d5c871cb9360b414f981e7072c52c33258d598305280fef91c6cae34739d65d5"}, + {file = "jedi-0.17.0-py2.py3-none-any.whl", hash = "sha256:cd60c93b71944d628ccac47df9a60fec53150de53d42dc10a7fc4b5ba6aae798"}, + {file = "jedi-0.17.0.tar.gz", hash = "sha256:df40c97641cb943661d2db4c33c2e1ff75d491189423249e989bcea4464f3030"}, ] jinja2 = [ - {file = "Jinja2-2.11.1-py2.py3-none-any.whl", hash = "sha256:b0eaf100007721b5c16c1fc1eecb87409464edc10469ddc9a22a27a99123be49"}, - {file = "Jinja2-2.11.1.tar.gz", hash = "sha256:93187ffbc7808079673ef52771baa950426fd664d3aad1d0fa3e95644360e250"}, + {file = "Jinja2-2.11.2-py2.py3-none-any.whl", hash = "sha256:f0a4641d3cf955324a89c04f3d94663aa4d638abe8f733ecd3582848e1c37035"}, + {file = "Jinja2-2.11.2.tar.gz", hash = "sha256:89aab215427ef59c34ad58735269eb58b1a5808103067f7bb9d5836c651b3bb0"}, ] lxml = [ {file = "lxml-4.5.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:0701f7965903a1c3f6f09328c1278ac0eee8f56f244e66af79cb224b7ef3801c"}, @@ -1410,8 +1410,8 @@ parsel = [ {file = "parsel-1.5.2.tar.gz", hash = "sha256:4da4262ba4605573b6b72a5f557616a2fc9dee7a47a1efad562752a28d366723"}, ] parso = [ - {file = "parso-0.6.2-py2.py3-none-any.whl", hash = "sha256:8515fc12cfca6ee3aa59138741fc5624d62340c97e401c74875769948d4f2995"}, - {file = "parso-0.6.2.tar.gz", hash = "sha256:0c5659e0c6eba20636f99a04f469798dca8da279645ce5c387315b2c23912157"}, + {file = "parso-0.7.0-py2.py3-none-any.whl", hash = "sha256:158c140fc04112dc45bca311633ae5033c2c2a7b732fa33d0955bad8152a8dd0"}, + {file = "parso-0.7.0.tar.gz", hash = "sha256:908e9fae2144a076d72ae4e25539143d40b8e3eafbaeae03c1bfe226f4cdf12c"}, ] pexpect = [ {file = "pexpect-4.8.0-py2.py3-none-any.whl", hash = "sha256:0b48a55dcb3c05f3329815901ea4fc1537514d6ba867a152b581d69ae3710937"}, @@ -1606,8 +1606,8 @@ typing-extensions = [ {file = "typing_extensions-3.7.4.2.tar.gz", hash = "sha256:79ee589a3caca649a9bfd2a8de4709837400dfa00b6cc81962a1e6a1815969ae"}, ] urllib3 = [ - {file = "urllib3-1.25.8-py2.py3-none-any.whl", hash = "sha256:2f3db8b19923a873b3e5256dc9c2dedfa883e33d87c690d9c7913e1f40673cdc"}, - {file = "urllib3-1.25.8.tar.gz", hash = "sha256:87716c2d2a7121198ebcb7ce7cccf6ce5e9ba539041cfbaeecfb641dc0bf6acc"}, + {file = "urllib3-1.25.9-py2.py3-none-any.whl", hash = "sha256:88206b0eb87e6d677d424843ac5209e3fb9d0190d0ee169599165ec25e9d9115"}, + {file = "urllib3-1.25.9.tar.gz", hash = "sha256:3018294ebefce6572a474f0604c2021e33b3fd8006ecd11d62107a5d2a963527"}, ] vt-py = [ {file = "vt-py-0.5.3.tar.gz", hash = "sha256:0a52d58976ec3baf24ade11d0473773d6c7a8ccf862c86f34bc74216ffbe920f"}, @@ -1648,5 +1648,44 @@ zipp = [ {file = "zipp-3.1.0.tar.gz", hash = "sha256:c599e4d75c98f6798c509911d08a22e6c021d074469042177c8c86fb92eefd96"}, ] "zope.interface" = [ - {file = "zope.interface-5.0.2.tar.gz", hash = "sha256:67267aa6764f488833f92d9d6889239af92bd80b4c99cc76e7f847f660e660fa"}, + {file = "zope.interface-5.1.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:645a7092b77fdbc3f68d3cc98f9d3e71510e419f54019d6e282328c0dd140dcd"}, + {file = "zope.interface-5.1.0-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:d1fe9d7d09bb07228650903d6a9dc48ea649e3b8c69b1d263419cc722b3938e8"}, + {file = "zope.interface-5.1.0-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:a744132d0abaa854d1aad50ba9bc64e79c6f835b3e92521db4235a1991176813"}, + {file = "zope.interface-5.1.0-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:461d4339b3b8f3335d7e2c90ce335eb275488c587b61aca4b305196dde2ff086"}, + {file = "zope.interface-5.1.0-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:269b27f60bcf45438e8683269f8ecd1235fa13e5411de93dae3b9ee4fe7f7bc7"}, + {file = "zope.interface-5.1.0-cp27-cp27m-win32.whl", hash = "sha256:6874367586c020705a44eecdad5d6b587c64b892e34305bb6ed87c9bbe22a5e9"}, + {file = "zope.interface-5.1.0-cp27-cp27m-win_amd64.whl", hash = "sha256:8149ded7f90154fdc1a40e0c8975df58041a6f693b8f7edcd9348484e9dc17fe"}, + {file = "zope.interface-5.1.0-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:0103cba5ed09f27d2e3de7e48bb320338592e2fabc5ce1432cf33808eb2dfd8b"}, + {file = "zope.interface-5.1.0-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:b0becb75418f8a130e9d465e718316cd17c7a8acce6fe8fe07adc72762bee425"}, + {file = "zope.interface-5.1.0-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:fb55c182a3f7b84c1a2d6de5fa7b1a05d4660d866b91dbf8d74549c57a1499e8"}, + {file = "zope.interface-5.1.0-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:4f98f70328bc788c86a6a1a8a14b0ea979f81ae6015dd6c72978f1feff70ecda"}, + {file = "zope.interface-5.1.0-cp35-cp35m-macosx_10_6_intel.whl", hash = "sha256:af2c14efc0bb0e91af63d00080ccc067866fb8cbbaca2b0438ab4105f5e0f08d"}, + {file = "zope.interface-5.1.0-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:f68bf937f113b88c866d090fea0bc52a098695173fc613b055a17ff0cf9683b6"}, + {file = "zope.interface-5.1.0-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:d7804f6a71fc2dda888ef2de266727ec2f3915373d5a785ed4ddc603bbc91e08"}, + {file = "zope.interface-5.1.0-cp35-cp35m-manylinux2010_i686.whl", hash = "sha256:74bf0a4f9091131de09286f9a605db449840e313753949fe07c8d0fe7659ad1e"}, + {file = "zope.interface-5.1.0-cp35-cp35m-manylinux2010_x86_64.whl", hash = "sha256:ba4261c8ad00b49d48bbb3b5af388bb7576edfc0ca50a49c11dcb77caa1d897e"}, + {file = "zope.interface-5.1.0-cp35-cp35m-win32.whl", hash = "sha256:ebb4e637a1fb861c34e48a00d03cffa9234f42bef923aec44e5625ffb9a8e8f9"}, + {file = "zope.interface-5.1.0-cp35-cp35m-win_amd64.whl", hash = "sha256:911714b08b63d155f9c948da2b5534b223a1a4fc50bb67139ab68b277c938578"}, + {file = "zope.interface-5.1.0-cp36-cp36m-macosx_10_6_intel.whl", hash = "sha256:e74671e43ed4569fbd7989e5eecc7d06dc134b571872ab1d5a88f4a123814e9f"}, + {file = "zope.interface-5.1.0-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:b1d2ed1cbda2ae107283befd9284e650d840f8f7568cb9060b5466d25dc48975"}, + {file = "zope.interface-5.1.0-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:ef739fe89e7f43fb6494a43b1878a36273e5924869ba1d866f752c5812ae8d58"}, + {file = "zope.interface-5.1.0-cp36-cp36m-manylinux2010_i686.whl", hash = "sha256:eb9b92f456ff3ec746cd4935b73c1117538d6124b8617bc0fe6fda0b3816e345"}, + {file = "zope.interface-5.1.0-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:dcefc97d1daf8d55199420e9162ab584ed0893a109f45e438b9794ced44c9fd0"}, + {file = "zope.interface-5.1.0-cp36-cp36m-win32.whl", hash = "sha256:f40db0e02a8157d2b90857c24d89b6310f9b6c3642369852cdc3b5ac49b92afc"}, + {file = "zope.interface-5.1.0-cp36-cp36m-win_amd64.whl", hash = "sha256:14415d6979356629f1c386c8c4249b4d0082f2ea7f75871ebad2e29584bd16c5"}, + {file = "zope.interface-5.1.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:5e86c66a6dea8ab6152e83b0facc856dc4d435fe0f872f01d66ce0a2131b7f1d"}, + {file = "zope.interface-5.1.0-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:39106649c3082972106f930766ae23d1464a73b7d30b3698c986f74bf1256a34"}, + {file = "zope.interface-5.1.0-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:8cccf7057c7d19064a9e27660f5aec4e5c4001ffcf653a47531bde19b5aa2a8a"}, + {file = "zope.interface-5.1.0-cp37-cp37m-manylinux2010_i686.whl", hash = "sha256:562dccd37acec149458c1791da459f130c6cf8902c94c93b8d47c6337b9fb826"}, + {file = "zope.interface-5.1.0-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:da2844fba024dd58eaa712561da47dcd1e7ad544a257482392472eae1c86d5e5"}, + {file = "zope.interface-5.1.0-cp37-cp37m-win32.whl", hash = "sha256:1ae4693ccee94c6e0c88a4568fb3b34af8871c60f5ba30cf9f94977ed0e53ddd"}, + {file = "zope.interface-5.1.0-cp37-cp37m-win_amd64.whl", hash = "sha256:dd98c436a1fc56f48c70882cc243df89ad036210d871c7427dc164b31500dc11"}, + {file = "zope.interface-5.1.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1b87ed2dc05cb835138f6a6e3595593fea3564d712cb2eb2de963a41fd35758c"}, + {file = "zope.interface-5.1.0-cp38-cp38-manylinux1_i686.whl", hash = "sha256:558a20a0845d1a5dc6ff87cd0f63d7dac982d7c3be05d2ffb6322a87c17fa286"}, + {file = "zope.interface-5.1.0-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7b726194f938791a6691c7592c8b9e805fc6d1b9632a833b9c0640828cd49cbc"}, + {file = "zope.interface-5.1.0-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:60a207efcd8c11d6bbeb7862e33418fba4e4ad79846d88d160d7231fcb42a5ee"}, + {file = "zope.interface-5.1.0-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:b054eb0a8aa712c8e9030065a59b5e6a5cf0746ecdb5f087cca5ec7685690c19"}, + {file = "zope.interface-5.1.0-cp38-cp38-win32.whl", hash = "sha256:27d287e61639d692563d9dab76bafe071fbeb26818dd6a32a0022f3f7ca884b5"}, + {file = "zope.interface-5.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:a5f8f85986197d1dd6444763c4a15c991bfed86d835a1f6f7d476f7198d5f56a"}, + {file = "zope.interface-5.1.0.tar.gz", hash = "sha256:40e4c42bd27ed3c11b2c983fecfb03356fae1209de10686d03c02c8696a1d90e"}, ] diff --git a/website/web/__init__.py b/website/web/__init__.py index 733170ee..b0506ede 100644 --- a/website/web/__init__.py +++ b/website/web/__init__.py @@ -164,6 +164,42 @@ def urlnode_details(node_uuid): as_attachment=True, attachment_filename='file.zip') +@app.route('/tree//trigger_modules/', defaults={'force': False}) +@app.route('/tree//trigger_modules/', methods=['GET']) +def trigger_modules(tree_uuid, force): + capture_dir = lookyloo.lookup_capture_dir(tree_uuid) + if not capture_dir: + return Response('Not available.', mimetype='text/text') + lookyloo.trigger_modules(capture_dir, force) + return redirect(url_for('modules', tree_uuid=tree_uuid)) + + +@app.route('/tree//modules', methods=['GET']) +def modules(tree_uuid): + capture_dir = lookyloo.lookup_capture_dir(tree_uuid) + if not capture_dir: + return Response('Not available.', mimetype='text/text') + modules_responses = lookyloo.get_modules_responses(capture_dir) + if not modules_responses: + return redirect(url_for('tree', tree_uuid=tree_uuid)) + + vt_short_result = {} + if 'vt' in modules_responses: + # VirusTotal cleanup + vt = modules_responses.pop('vt') + # Get malicious entries + for url, full_report in vt.items(): + vt_short_result[url] = { + 'permaurl': f'https://www.virustotal.com/gui/url/{full_report["id"]}/detection', + 'malicious': [] + } + for vendor, result in full_report['attributes']['last_analysis_results'].items(): + if result['category'] == 'malicious': + vt_short_result[url]['malicious'].append((vendor, result['result'])) + + return render_template('modules.html', uuid=tree_uuid, vt=vt_short_result) + + @app.route('/tree//image', methods=['GET']) def image(tree_uuid): capture_dir = lookyloo.lookup_capture_dir(tree_uuid) diff --git a/website/web/templates/tree.html b/website/web/templates/tree.html index 969603c7..24305c82 100644 --- a/website/web/templates/tree.html +++ b/website/web/templates/tree.html @@ -14,6 +14,20 @@ }); }); + + {% endblock %} {% block content %} @@ -122,6 +136,10 @@
+
+ + Show third party reports
@@ -135,4 +153,24 @@
+ + {% endblock content %}