diff --git a/config/modules.json.sample b/config/modules.json.sample index 9b73ab78..506c7cf7 100644 --- a/config/modules.json.sample +++ b/config/modules.json.sample @@ -13,6 +13,7 @@ "MISP": { "apikey": null, "url": "https://misp.url", + "verify_tls_cert": true, "enable_lookup": false, "enable_push": false }, diff --git a/lookyloo/lookyloo.py b/lookyloo/lookyloo.py index 2eb956d4..10d602a2 100644 --- a/lookyloo/lookyloo.py +++ b/lookyloo/lookyloo.py @@ -35,7 +35,7 @@ from .exceptions import NoValidHarFile, MissingUUID, LookylooException from .helpers import (get_homedir, get_socket_path, load_cookies, get_config, safe_create_dir, get_email_template, load_pickle_tree, remove_pickle_tree, get_resources_hashes, get_taxonomies, uniq_domains) -from .modules import VirusTotal, SaneJavaScript, PhishingInitiative +from .modules import VirusTotal, SaneJavaScript, PhishingInitiative, MISP from .capturecache import CaptureCache from .context import Context from .indexing import Indexing @@ -75,6 +75,10 @@ class Lookyloo(): if not self.sanejs.available: self.logger.warning('Unable to setup the SaneJS module') + self.misp = MISP(get_config('modules', 'MISP')) + if not self.misp.available: + self.logger.warning('Unable to setup the MISP module') + self.context = Context(self.sanejs) if not self.redis.exists('cache_loaded'): @@ -912,27 +916,29 @@ class Lookyloo(): event = MISPEvent() event.info = f'Lookyloo Capture ({cache.url})' - event.add_attribute('link', f'https://{self.public_domain}/tree/{capture_uuid}') + lookyloo_link = event.add_attribute('link', f'https://{self.public_domain}/tree/{capture_uuid}') initial_url = URLObject(cache.url) - redirects = [URLObject(url) for url in cache.redirects] + redirects = [URLObject(url) for url in cache.redirects if url != cache.url] if redirects: - initial_url.add_reference(redirects[0], 'redirects-to') - prec_object = redirects[0] - for u_object in redirects[1:]: + prec_object = initial_url + for u_object in redirects: prec_object.add_reference(u_object, 'redirects-to') prec_object = u_object - event.add_object(initial_url) + initial_obj = event.add_object(initial_url) + initial_obj.add_reference(lookyloo_link, 'captured-by', 'Capture on lookyloo') + for u_object in redirects: event.add_object(u_object) - event.add_attribute('attachment', 'screenshot_landing_page.png', data=self.get_screenshot(capture_uuid)) + screenshot = event.add_attribute('attachment', 'screenshot_landing_page.png', data=self.get_screenshot(capture_uuid), disable_correlation=True) try: fo = FileObject(pseudofile=ct.root_hartree.rendered_node.body, filename='body_response.html') fo.comment = 'Content received for the final redirect (before rendering)' fo.add_reference(event.objects[-1], 'loaded-by', 'URL loading that content') + fo.add_reference(screenshot, 'rendered-as', 'Screenshot of the page') event.add_object(fo) except Har2TreeError: pass diff --git a/lookyloo/modules.py b/lookyloo/modules.py index df7819f0..15324c91 100644 --- a/lookyloo/modules.py +++ b/lookyloo/modules.py @@ -16,6 +16,32 @@ from .exceptions import ConfigError import vt # type: ignore from pysanejs import SaneJS from pyeupi import PyEUPI +from pymisp import PyMISP, MISPEvent + + +class MISP(): + + def __init__(self, config: Dict[str, Any]): + if not config.get('apikey'): + self.available = False + return + + self.available = True + self.enable_lookup = False + self.enable_push = False + self.client = PyMISP(url=config['url'], key=config['apikey'], ssl=config['verify_tls_cert']) + if config.get('enable_lookup'): + self.enable_lookup = True + if config.get('enable_push'): + self.enable_push = True + self.storage_dir_misp = get_homedir() / 'misp' + self.storage_dir_misp.mkdir(parents=True, exist_ok=True) + + def push(self, event: MISPEvent) -> Union[MISPEvent, Dict]: + if self.available and self.enable_push: + return self.client.add_event(event, pythonify=True) + else: + return {'error': 'Module not available or push not enabled.'} class SaneJavaScript(): diff --git a/website/web/__init__.py b/website/web/__init__.py index 4a495907..1d140647 100644 --- a/website/web/__init__.py +++ b/website/web/__init__.py @@ -15,6 +15,8 @@ from flask import Flask, render_template, request, send_file, redirect, url_for, from flask_bootstrap import Bootstrap # type: ignore from flask_httpauth import HTTPDigestAuth # type: ignore +from pymisp import MISPEvent + from lookyloo.helpers import get_homedir, update_user_agents, get_user_agents, get_config, get_taxonomies from lookyloo.lookyloo import Lookyloo, Indexing from lookyloo.exceptions import NoValidHarFile, MissingUUID @@ -723,6 +725,27 @@ def add_context(tree_uuid: str, node_uuid: str): return redirect(url_for('ressources')) +@app.route('/tree//misp_push', methods=['GET']) +@auth.login_required +def web_misp_push(tree_uuid: str): + if not lookyloo.misp.available: + flash('MISP module not available.', 'error') + elif not lookyloo.misp.enable_push: + flash('Push not enabled in MISP module.', 'error') + else: + event = lookyloo.misp_export(tree_uuid) + if isinstance(event, dict): + flash(f'Unable to generate the MISP export: {event}', 'error') + else: + event = lookyloo.misp.push(event) + if isinstance(event, MISPEvent): + flash(f'MISP event {event.id} created on {lookyloo.misp.client.root_url}', 'success') + else: + flash(f'Unable to create event: {event}', 'error') + + return redirect(url_for('tree', tree_uuid=tree_uuid)) + + # Query API @app.route('/json//redirects', methods=['GET']) @@ -755,6 +778,30 @@ def misp_export(tree_uuid: str): return Response(event.to_json(indent=2), mimetype='application/json') +@app.route('/json//misp_push', methods=['GET']) +@auth.login_required +def misp_push(tree_uuid: str): + to_return = {} + if not lookyloo.misp.available: + to_return['error'] = 'MISP module not available.' + elif not lookyloo.misp.enable_push: + to_return['error'] = 'Push not enabled in MISP module.' + else: + event = lookyloo.misp_export(tree_uuid) + if isinstance(event, dict): + to_return['error'] = event + else: + event = lookyloo.misp.push(event) + if isinstance(event, MISPEvent): + to_return = event.to_json(indent=2) + else: + to_return['error'] = event + + if isinstance(to_return, dict): + to_return = json.dumps(to_return, indent=2) + return Response(to_return, mimetype='application/json') + + @app.route('/json/hash_info/', methods=['GET']) def json_hash_info(h: str): details, body = lookyloo.get_body_hash_full(h)