new: Add push to MISP feature

config/modules.json.sample

@@ -13,6 +13,7 @@
   "MISP": {
     "apikey": null,
     "url": "https://misp.url",
+    "verify_tls_cert": true,
     "enable_lookup": false,
     "enable_push": false
   },

lookyloo/lookyloo.py

@@ -35,7 +35,7 @@ from .exceptions import NoValidHarFile, MissingUUID, LookylooException
 from .helpers import (get_homedir, get_socket_path, load_cookies, get_config,
                       safe_create_dir, get_email_template, load_pickle_tree,
                       remove_pickle_tree, get_resources_hashes, get_taxonomies, uniq_domains)
-from .modules import VirusTotal, SaneJavaScript, PhishingInitiative
+from .modules import VirusTotal, SaneJavaScript, PhishingInitiative, MISP
 from .capturecache import CaptureCache
 from .context import Context
 from .indexing import Indexing
@@ -75,6 +75,10 @@ class Lookyloo():
         if not self.sanejs.available:
             self.logger.warning('Unable to setup the SaneJS module')
 
+        self.misp = MISP(get_config('modules', 'MISP'))
+        if not self.misp.available:
+            self.logger.warning('Unable to setup the MISP module')
+
         self.context = Context(self.sanejs)
 
         if not self.redis.exists('cache_loaded'):
@@ -912,27 +916,29 @@ class Lookyloo():
 
         event = MISPEvent()
         event.info = f'Lookyloo Capture ({cache.url})'
-        event.add_attribute('link', f'https://{self.public_domain}/tree/{capture_uuid}')
+        lookyloo_link = event.add_attribute('link', f'https://{self.public_domain}/tree/{capture_uuid}')
 
         initial_url = URLObject(cache.url)
-        redirects = [URLObject(url) for url in cache.redirects]
+        redirects = [URLObject(url) for url in cache.redirects if url != cache.url]
 
         if redirects:
-            initial_url.add_reference(redirects[0], 'redirects-to')
+            prec_object = initial_url
-            prec_object = redirects[0]
+            for u_object in redirects:
-            for u_object in redirects[1:]:
                 prec_object.add_reference(u_object, 'redirects-to')
                 prec_object = u_object
 
-        event.add_object(initial_url)
+        initial_obj = event.add_object(initial_url)
+        initial_obj.add_reference(lookyloo_link, 'captured-by', 'Capture on lookyloo')
+
         for u_object in redirects:
             event.add_object(u_object)
 
-        event.add_attribute('attachment', 'screenshot_landing_page.png', data=self.get_screenshot(capture_uuid))
+        screenshot = event.add_attribute('attachment', 'screenshot_landing_page.png', data=self.get_screenshot(capture_uuid), disable_correlation=True)
         try:
             fo = FileObject(pseudofile=ct.root_hartree.rendered_node.body, filename='body_response.html')
             fo.comment = 'Content received for the final redirect (before rendering)'
             fo.add_reference(event.objects[-1], 'loaded-by', 'URL loading that content')
+            fo.add_reference(screenshot, 'rendered-as', 'Screenshot of the page')
             event.add_object(fo)
         except Har2TreeError:
             pass

lookyloo/modules.py

@@ -16,6 +16,32 @@ from .exceptions import ConfigError
 import vt  # type: ignore
 from pysanejs import SaneJS
 from pyeupi import PyEUPI
+from pymisp import PyMISP, MISPEvent
+
+
+class MISP():
+
+    def __init__(self, config: Dict[str, Any]):
+        if not config.get('apikey'):
+            self.available = False
+            return
+
+        self.available = True
+        self.enable_lookup = False
+        self.enable_push = False
+        self.client = PyMISP(url=config['url'], key=config['apikey'], ssl=config['verify_tls_cert'])
+        if config.get('enable_lookup'):
+            self.enable_lookup = True
+        if config.get('enable_push'):
+            self.enable_push = True
+        self.storage_dir_misp = get_homedir() / 'misp'
+        self.storage_dir_misp.mkdir(parents=True, exist_ok=True)
+
+    def push(self, event: MISPEvent) -> Union[MISPEvent, Dict]:
+        if self.available and self.enable_push:
+            return self.client.add_event(event, pythonify=True)
+        else:
+            return {'error': 'Module not available or push not enabled.'}
 
 
 class SaneJavaScript():

website/web/__init__.py

@@ -15,6 +15,8 @@ from flask import Flask, render_template, request, send_file, redirect, url_for,
 from flask_bootstrap import Bootstrap  # type: ignore
 from flask_httpauth import HTTPDigestAuth  # type: ignore
 
+from pymisp import MISPEvent
+
 from lookyloo.helpers import get_homedir, update_user_agents, get_user_agents, get_config, get_taxonomies
 from lookyloo.lookyloo import Lookyloo, Indexing
 from lookyloo.exceptions import NoValidHarFile, MissingUUID
@@ -723,6 +725,27 @@ def add_context(tree_uuid: str, node_uuid: str):
         return redirect(url_for('ressources'))
 
 
+@app.route('/tree/<string:tree_uuid>/misp_push', methods=['GET'])
+@auth.login_required
+def web_misp_push(tree_uuid: str):
+    if not lookyloo.misp.available:
+        flash('MISP module not available.', 'error')
+    elif not lookyloo.misp.enable_push:
+        flash('Push not enabled in MISP module.', 'error')
+    else:
+        event = lookyloo.misp_export(tree_uuid)
+        if isinstance(event, dict):
+            flash(f'Unable to generate the MISP export: {event}', 'error')
+        else:
+            event = lookyloo.misp.push(event)
+            if isinstance(event, MISPEvent):
+                flash(f'MISP event {event.id} created on {lookyloo.misp.client.root_url}', 'success')
+            else:
+                flash(f'Unable to create event: {event}', 'error')
+
+    return redirect(url_for('tree', tree_uuid=tree_uuid))
+
+
 # Query API
 
 @app.route('/json/<string:tree_uuid>/redirects', methods=['GET'])
@@ -755,6 +778,30 @@ def misp_export(tree_uuid: str):
     return Response(event.to_json(indent=2), mimetype='application/json')
 
 
+@app.route('/json/<string:tree_uuid>/misp_push', methods=['GET'])
+@auth.login_required
+def misp_push(tree_uuid: str):
+    to_return = {}
+    if not lookyloo.misp.available:
+        to_return['error'] = 'MISP module not available.'
+    elif not lookyloo.misp.enable_push:
+        to_return['error'] = 'Push not enabled in MISP module.'
+    else:
+        event = lookyloo.misp_export(tree_uuid)
+        if isinstance(event, dict):
+            to_return['error'] = event
+        else:
+            event = lookyloo.misp.push(event)
+            if isinstance(event, MISPEvent):
+                to_return = event.to_json(indent=2)
+            else:
+                to_return['error'] = event
+
+    if isinstance(to_return, dict):
+        to_return = json.dumps(to_return, indent=2)
+    return Response(to_return, mimetype='application/json')
+
+
 @app.route('/json/hash_info/<h>', methods=['GET'])
 def json_hash_info(h: str):
     details, body = lookyloo.get_body_hash_full(h)