mirror of https://github.com/CIRCL/lookyloo
new: (WiP) Add support for multiple MISPs
parent
f456947b62
commit
6034d10834
|
@ -23,16 +23,23 @@
|
||||||
"enabled": true,
|
"enabled": true,
|
||||||
"allow_auto_trigger": true
|
"allow_auto_trigger": true
|
||||||
},
|
},
|
||||||
"MISP": {
|
"MultipleMISPs": {
|
||||||
"apikey": null,
|
"default": "MISP",
|
||||||
"url": "https://misp.url",
|
"instances": {
|
||||||
"verify_tls_cert": true,
|
"MISP": {
|
||||||
"timeout": 10,
|
"apikey": null,
|
||||||
"enable_lookup": false,
|
"url": "https://misp.url",
|
||||||
"enable_push": false,
|
"verify_tls_cert": true,
|
||||||
"default_tags": [],
|
"timeout": 10,
|
||||||
"auto_publish": false,
|
"enable_lookup": false,
|
||||||
"allow_auto_trigger": false
|
"enable_push": false,
|
||||||
|
"default_tags": [
|
||||||
|
"source:lookyloo"
|
||||||
|
],
|
||||||
|
"auto_publish": false,
|
||||||
|
"allow_auto_trigger": false
|
||||||
|
}
|
||||||
|
}
|
||||||
},
|
},
|
||||||
"UniversalWhois": {
|
"UniversalWhois": {
|
||||||
"enabled": false,
|
"enabled": false,
|
||||||
|
|
|
@ -52,7 +52,7 @@ from .helpers import (get_captures_dir, get_email_template,
|
||||||
uniq_domains, ParsedUserAgent, load_cookies, UserAgents,
|
uniq_domains, ParsedUserAgent, load_cookies, UserAgents,
|
||||||
get_useragent_for_requests)
|
get_useragent_for_requests)
|
||||||
from .indexing import Indexing
|
from .indexing import Indexing
|
||||||
from .modules import (MISP, PhishingInitiative, UniversalWhois,
|
from .modules import (MISPs, MISP, PhishingInitiative, UniversalWhois,
|
||||||
UrlScan, VirusTotal, Phishtank, Hashlookup,
|
UrlScan, VirusTotal, Phishtank, Hashlookup,
|
||||||
RiskIQ, RiskIQError, Pandora, URLhaus)
|
RiskIQ, RiskIQError, Pandora, URLhaus)
|
||||||
|
|
||||||
|
@ -106,9 +106,33 @@ class Lookyloo():
|
||||||
if not self.vt.available:
|
if not self.vt.available:
|
||||||
self.logger.warning('Unable to setup the VirusTotal module')
|
self.logger.warning('Unable to setup the VirusTotal module')
|
||||||
|
|
||||||
self.misp = MISP(get_config('modules', 'MISP'))
|
# ## Initialize MISP(s)
|
||||||
if not self.misp.available:
|
try_old_config = False
|
||||||
self.logger.warning('Unable to setup the MISP module')
|
if misps_config := get_config('modules', 'MultipleMISPs'):
|
||||||
|
# New config
|
||||||
|
self.misps = MISPs(misps_config)
|
||||||
|
if not self.misps.available:
|
||||||
|
self.logger.warning('Unable to setup the MISP module')
|
||||||
|
try_old_config = True
|
||||||
|
|
||||||
|
if try_old_config:
|
||||||
|
# Legacy MISP config, now use MultipleMISPs key to support more than one MISP instance
|
||||||
|
try:
|
||||||
|
if misp_config := get_config('modules', 'MISP'):
|
||||||
|
misps_config = {'default': 'MISP', 'instances': {'MISP': misp_config}}
|
||||||
|
self.misps = MISPs(misps_config)
|
||||||
|
if self.misps.available:
|
||||||
|
self.logger.warning('Please migrate the MISP config to the "MultipleMISPs" key in the config, and remove the "MISP" key')
|
||||||
|
else:
|
||||||
|
self.logger.warning('Unable to setup the MISP module')
|
||||||
|
except Exception:
|
||||||
|
# The key was removed from the config, and the sample config
|
||||||
|
pass
|
||||||
|
|
||||||
|
if not self.misps.available:
|
||||||
|
self.logger.info('The MISP module is not configured')
|
||||||
|
|
||||||
|
# ## Done with MISP(s)
|
||||||
|
|
||||||
self.uwhois = UniversalWhois(get_config('modules', 'UniversalWhois'))
|
self.uwhois = UniversalWhois(get_config('modules', 'UniversalWhois'))
|
||||||
if not self.uwhois.available:
|
if not self.uwhois.available:
|
||||||
|
@ -1129,9 +1153,9 @@ class Lookyloo():
|
||||||
# In the case, we want to have it as a FileObject in the export
|
# In the case, we want to have it as a FileObject in the export
|
||||||
filename, pseudofile = self.get_data(capture_uuid)
|
filename, pseudofile = self.get_data(capture_uuid)
|
||||||
if filename:
|
if filename:
|
||||||
event = self.misp.export(cache, self.is_public_instance, filename, pseudofile)
|
event = self.misps.export(cache, self.is_public_instance, filename, pseudofile)
|
||||||
else:
|
else:
|
||||||
event = self.misp.export(cache, self.is_public_instance)
|
event = self.misps.export(cache, self.is_public_instance)
|
||||||
screenshot: MISPAttribute = event.add_attribute('attachment', 'screenshot_landing_page.png',
|
screenshot: MISPAttribute = event.add_attribute('attachment', 'screenshot_landing_page.png',
|
||||||
data=self.get_screenshot(cache.uuid),
|
data=self.get_screenshot(cache.uuid),
|
||||||
disable_correlation=True) # type: ignore
|
disable_correlation=True) # type: ignore
|
||||||
|
@ -1179,8 +1203,18 @@ class Lookyloo():
|
||||||
|
|
||||||
return [event]
|
return [event]
|
||||||
|
|
||||||
def get_misp_occurrences(self, capture_uuid: str, /) -> Optional[Dict[str, Set[str]]]:
|
def get_misp_instance(self, instance_name: Optional[str]=None) -> MISP:
|
||||||
if not self.misp.available:
|
if instance_name:
|
||||||
|
if misp := self.misps.get(instance_name):
|
||||||
|
return misp
|
||||||
|
self.logger.warning(f'Unable to connect to MISP Instance {instance_name}, falling back to default.')
|
||||||
|
|
||||||
|
return self.misps.default_misp
|
||||||
|
|
||||||
|
def get_misp_occurrences(self, capture_uuid: str, /, *, instance_name: Optional[str]=None) -> Optional[Tuple[Dict[str, Set[str]], str]]:
|
||||||
|
misp = self.get_misp_instance(instance_name)
|
||||||
|
|
||||||
|
if not misp.available:
|
||||||
return None
|
return None
|
||||||
try:
|
try:
|
||||||
ct = self.get_crawled_tree(capture_uuid)
|
ct = self.get_crawled_tree(capture_uuid)
|
||||||
|
@ -1190,12 +1224,12 @@ class Lookyloo():
|
||||||
nodes_to_lookup = ct.root_hartree.rendered_node.get_ancestors() + [ct.root_hartree.rendered_node]
|
nodes_to_lookup = ct.root_hartree.rendered_node.get_ancestors() + [ct.root_hartree.rendered_node]
|
||||||
to_return: Dict[str, Set[str]] = defaultdict(set)
|
to_return: Dict[str, Set[str]] = defaultdict(set)
|
||||||
for node in nodes_to_lookup:
|
for node in nodes_to_lookup:
|
||||||
hits = self.misp.lookup(node, ct.root_hartree.get_host_node_by_uuid(node.hostnode_uuid))
|
hits = misp.lookup(node, ct.root_hartree.get_host_node_by_uuid(node.hostnode_uuid))
|
||||||
for event_id, values in hits.items():
|
for event_id, values in hits.items():
|
||||||
if not isinstance(values, set):
|
if not isinstance(values, set):
|
||||||
continue
|
continue
|
||||||
to_return[event_id].update(values)
|
to_return[event_id].update(values)
|
||||||
return to_return
|
return to_return, misp.client.root_url
|
||||||
|
|
||||||
def get_hashes_with_context(self, tree_uuid: str, /, algorithm: str, *, urls_only: bool=False) -> Union[Dict[str, Set[str]], Dict[str, List[URLNode]]]:
|
def get_hashes_with_context(self, tree_uuid: str, /, algorithm: str, *, urls_only: bool=False) -> Union[Dict[str, Set[str]], Dict[str, List[URLNode]]]:
|
||||||
"""Build (on demand) hashes for all the ressources of the tree, using the alorighm provided by the user.
|
"""Build (on demand) hashes for all the ressources of the tree, using the alorighm provided by the user.
|
||||||
|
|
|
@ -1,7 +1,7 @@
|
||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
from .fox import FOX # noqa
|
from .fox import FOX # noqa
|
||||||
from .misp import MISP # noqa
|
from .misp import MISPs, MISP # noqa
|
||||||
from .pi import PhishingInitiative # noqa
|
from .pi import PhishingInitiative # noqa
|
||||||
from .sanejs import SaneJavaScript # noqa
|
from .sanejs import SaneJavaScript # noqa
|
||||||
from .urlscan import UrlScan # noqa
|
from .urlscan import UrlScan # noqa
|
||||||
|
|
|
@ -5,6 +5,7 @@ import re
|
||||||
|
|
||||||
from io import BytesIO
|
from io import BytesIO
|
||||||
from collections import defaultdict
|
from collections import defaultdict
|
||||||
|
from collections.abc import Mapping
|
||||||
from typing import Any, Dict, List, Optional, Set, Union, TYPE_CHECKING
|
from typing import Any, Dict, List, Optional, Set, Union, TYPE_CHECKING
|
||||||
|
|
||||||
import requests
|
import requests
|
||||||
|
@ -19,6 +20,137 @@ if TYPE_CHECKING:
|
||||||
from ..capturecache import CaptureCache
|
from ..capturecache import CaptureCache
|
||||||
|
|
||||||
|
|
||||||
|
class MISPs(Mapping):
|
||||||
|
|
||||||
|
def __init__(self, config: Dict[str, Any]):
|
||||||
|
self.logger = logging.getLogger(f'{self.__class__.__name__}')
|
||||||
|
self.logger.setLevel(get_config('generic', 'loglevel'))
|
||||||
|
|
||||||
|
if not config.get('default'):
|
||||||
|
self.available = False
|
||||||
|
self.logger.info('No default instance configured, disabling MISP.')
|
||||||
|
return
|
||||||
|
if not config.get('instances'):
|
||||||
|
self.available = False
|
||||||
|
self.logger.warning('No MISP instances configured, disabling MISP.')
|
||||||
|
return
|
||||||
|
|
||||||
|
self.default_instance = config['default']
|
||||||
|
|
||||||
|
if self.default_instance not in config['instances']:
|
||||||
|
self.available = False
|
||||||
|
self.logger.warning(f"The default MISP instance ({self.default_instance}) is missing in the instances ({', '.join(config['instances'].keys())}), disabling MISP.")
|
||||||
|
return
|
||||||
|
|
||||||
|
self.__misps: Dict[str, 'MISP'] = {}
|
||||||
|
for instance_name, instance_config in config['instances'].items():
|
||||||
|
if misp_connector := MISP(instance_config):
|
||||||
|
if misp_connector.available:
|
||||||
|
self.__misps[instance_name] = misp_connector
|
||||||
|
else:
|
||||||
|
self.logger.warning(f"MISP '{instance_name}' isn't available.")
|
||||||
|
else:
|
||||||
|
self.logger.warning(f"Unable to initialize the connector to '{instance_name}'. It won't be available.")
|
||||||
|
|
||||||
|
if not self.__misps.get(self.default_instance) or not self.__misps[self.default_instance].available:
|
||||||
|
self.available = False
|
||||||
|
self.logger.warning("Unable to initialize the connector to the default MISP instance, disabling MISP.")
|
||||||
|
return
|
||||||
|
|
||||||
|
self.available = True
|
||||||
|
|
||||||
|
def __getitem__(self, name: str) -> 'MISP':
|
||||||
|
return self.__misps[name]
|
||||||
|
|
||||||
|
def __iter__(self):
|
||||||
|
return iter(self.__misps)
|
||||||
|
|
||||||
|
def __len__(self):
|
||||||
|
return len(self.__misps)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def default_misp(self) -> 'MISP':
|
||||||
|
return self.__misps[self.default_instance]
|
||||||
|
|
||||||
|
def export(self, cache: 'CaptureCache', is_public_instance: bool=False,
|
||||||
|
submitted_filename: Optional[str]=None,
|
||||||
|
submitted_file: Optional[BytesIO]=None) -> MISPEvent:
|
||||||
|
'''Export a capture in MISP format. You can POST the return of this method
|
||||||
|
directly to a MISP instance and it will create an event.'''
|
||||||
|
public_domain = get_config('generic', 'public_domain')
|
||||||
|
event = MISPEvent()
|
||||||
|
if cache.url.startswith('file'):
|
||||||
|
filename = cache.url.rsplit('/', 1)[-1]
|
||||||
|
event.info = f'Lookyloo Capture ({filename})'
|
||||||
|
# Create file object as initial
|
||||||
|
if hasattr(cache.tree.root_hartree.url_tree, 'body'):
|
||||||
|
# The file could be viewed in the browser
|
||||||
|
filename = cache.tree.root_hartree.url_tree.name
|
||||||
|
pseudofile = cache.tree.root_hartree.url_tree.body
|
||||||
|
elif submitted_filename:
|
||||||
|
# Impossible to get the file from the HAR.
|
||||||
|
filename = submitted_filename
|
||||||
|
pseudofile = submitted_file
|
||||||
|
else:
|
||||||
|
raise Exception('We must have a file here.')
|
||||||
|
|
||||||
|
initial_file = FileObject(pseudofile=pseudofile, filename=filename)
|
||||||
|
initial_file.comment = 'This is a capture of a file, rendered in the browser'
|
||||||
|
initial_obj = event.add_object(initial_file)
|
||||||
|
else:
|
||||||
|
event.info = f'Lookyloo Capture ({cache.url})'
|
||||||
|
initial_url = URLObject(cache.url)
|
||||||
|
initial_url.comment = 'Submitted URL'
|
||||||
|
self.__misp_add_ips_to_URLObject(initial_url, cache.tree.root_hartree.hostname_tree)
|
||||||
|
initial_obj = event.add_object(initial_url)
|
||||||
|
|
||||||
|
lookyloo_link: MISPAttribute = event.add_attribute('link', f'https://{public_domain}/tree/{cache.uuid}') # type: ignore
|
||||||
|
if not is_public_instance:
|
||||||
|
lookyloo_link.distribution = 0
|
||||||
|
initial_obj.add_reference(lookyloo_link, 'captured-by', 'Capture on lookyloo')
|
||||||
|
|
||||||
|
redirects: List[URLObject] = []
|
||||||
|
for nb, url in enumerate(cache.redirects):
|
||||||
|
if url == cache.url:
|
||||||
|
continue
|
||||||
|
obj = URLObject(url)
|
||||||
|
obj.comment = f'Redirect {nb}'
|
||||||
|
self.__misp_add_ips_to_URLObject(obj, cache.tree.root_hartree.hostname_tree)
|
||||||
|
redirects.append(obj)
|
||||||
|
|
||||||
|
if redirects:
|
||||||
|
redirects[-1].comment = f'Last redirect ({nb})'
|
||||||
|
|
||||||
|
if redirects:
|
||||||
|
prec_object = initial_url
|
||||||
|
for u_object in redirects:
|
||||||
|
prec_object.add_reference(u_object, 'redirects-to')
|
||||||
|
prec_object = u_object
|
||||||
|
|
||||||
|
for u_object in redirects:
|
||||||
|
event.add_object(u_object)
|
||||||
|
final_redirect = event.objects[-1]
|
||||||
|
|
||||||
|
try:
|
||||||
|
fo = FileObject(pseudofile=cache.tree.root_hartree.rendered_node.body, filename=cache.tree.root_hartree.rendered_node.filename)
|
||||||
|
fo.comment = 'Content received for the final redirect (before rendering)'
|
||||||
|
fo.add_reference(final_redirect, 'loaded-by', 'URL loading that content')
|
||||||
|
event.add_object(fo)
|
||||||
|
except Har2TreeError:
|
||||||
|
pass
|
||||||
|
except AttributeError:
|
||||||
|
# No `body` in rendered node
|
||||||
|
pass
|
||||||
|
return event
|
||||||
|
|
||||||
|
def __misp_add_ips_to_URLObject(self, obj: URLObject, hostname_tree: HostNode) -> None:
|
||||||
|
hosts = obj.get_attributes_by_relation('host')
|
||||||
|
if hosts:
|
||||||
|
hostnodes = hostname_tree.search_nodes(name=hosts[0].value)
|
||||||
|
if hostnodes and hasattr(hostnodes[0], 'resolved_ips'):
|
||||||
|
obj.add_attributes('ip', *hostnodes[0].resolved_ips)
|
||||||
|
|
||||||
|
|
||||||
class MISP():
|
class MISP():
|
||||||
|
|
||||||
def __init__(self, config: Dict[str, Any]):
|
def __init__(self, config: Dict[str, Any]):
|
||||||
|
@ -159,81 +291,3 @@ class MISP():
|
||||||
return {'info': 'No hits.'}
|
return {'info': 'No hits.'}
|
||||||
else:
|
else:
|
||||||
return {'error': 'Module not available or lookup not enabled.'}
|
return {'error': 'Module not available or lookup not enabled.'}
|
||||||
|
|
||||||
def __misp_add_ips_to_URLObject(self, obj: URLObject, hostname_tree: HostNode) -> None:
|
|
||||||
hosts = obj.get_attributes_by_relation('host')
|
|
||||||
if hosts:
|
|
||||||
hostnodes = hostname_tree.search_nodes(name=hosts[0].value)
|
|
||||||
if hostnodes and hasattr(hostnodes[0], 'resolved_ips'):
|
|
||||||
obj.add_attributes('ip', *hostnodes[0].resolved_ips)
|
|
||||||
|
|
||||||
def export(self, cache: 'CaptureCache', is_public_instance: bool=False,
|
|
||||||
submitted_filename: Optional[str]=None,
|
|
||||||
submitted_file: Optional[BytesIO]=None) -> MISPEvent:
|
|
||||||
'''Export a capture in MISP format. You can POST the return of this method
|
|
||||||
directly to a MISP instance and it will create an event.'''
|
|
||||||
public_domain = get_config('generic', 'public_domain')
|
|
||||||
event = MISPEvent()
|
|
||||||
if cache.url.startswith('file'):
|
|
||||||
filename = cache.url.rsplit('/', 1)[-1]
|
|
||||||
event.info = f'Lookyloo Capture ({filename})'
|
|
||||||
# Create file object as initial
|
|
||||||
if hasattr(cache.tree.root_hartree.url_tree, 'body'):
|
|
||||||
# The file could be viewed in the browser
|
|
||||||
filename = cache.tree.root_hartree.url_tree.name
|
|
||||||
pseudofile = cache.tree.root_hartree.url_tree.body
|
|
||||||
elif submitted_filename:
|
|
||||||
# Impossible to get the file from the HAR.
|
|
||||||
filename = submitted_filename
|
|
||||||
pseudofile = submitted_file
|
|
||||||
else:
|
|
||||||
raise Exception('We must have a file here.')
|
|
||||||
|
|
||||||
initial_file = FileObject(pseudofile=pseudofile, filename=filename)
|
|
||||||
initial_file.comment = 'This is a capture of a file, rendered in the browser'
|
|
||||||
initial_obj = event.add_object(initial_file)
|
|
||||||
else:
|
|
||||||
event.info = f'Lookyloo Capture ({cache.url})'
|
|
||||||
initial_url = URLObject(cache.url)
|
|
||||||
initial_url.comment = 'Submitted URL'
|
|
||||||
self.__misp_add_ips_to_URLObject(initial_url, cache.tree.root_hartree.hostname_tree)
|
|
||||||
initial_obj = event.add_object(initial_url)
|
|
||||||
|
|
||||||
lookyloo_link: MISPAttribute = event.add_attribute('link', f'https://{public_domain}/tree/{cache.uuid}') # type: ignore
|
|
||||||
if not is_public_instance:
|
|
||||||
lookyloo_link.distribution = 0
|
|
||||||
initial_obj.add_reference(lookyloo_link, 'captured-by', 'Capture on lookyloo')
|
|
||||||
|
|
||||||
redirects: List[URLObject] = []
|
|
||||||
for nb, url in enumerate(cache.redirects):
|
|
||||||
if url == cache.url:
|
|
||||||
continue
|
|
||||||
obj = URLObject(url)
|
|
||||||
obj.comment = f'Redirect {nb}'
|
|
||||||
self.__misp_add_ips_to_URLObject(obj, cache.tree.root_hartree.hostname_tree)
|
|
||||||
redirects.append(obj)
|
|
||||||
|
|
||||||
if redirects:
|
|
||||||
redirects[-1].comment = f'Last redirect ({nb})'
|
|
||||||
|
|
||||||
if redirects:
|
|
||||||
prec_object = initial_url
|
|
||||||
for u_object in redirects:
|
|
||||||
prec_object.add_reference(u_object, 'redirects-to')
|
|
||||||
prec_object = u_object
|
|
||||||
|
|
||||||
for u_object in redirects:
|
|
||||||
event.add_object(u_object)
|
|
||||||
final_redirect = event.objects[-1]
|
|
||||||
|
|
||||||
try:
|
|
||||||
fo = FileObject(pseudofile=cache.tree.root_hartree.rendered_node.body, filename=cache.tree.root_hartree.rendered_node.filename)
|
|
||||||
fo.comment = 'Content received for the final redirect (before rendering)'
|
|
||||||
fo.add_reference(final_redirect, 'loaded-by', 'URL loading that content')
|
|
||||||
event.add_object(fo)
|
|
||||||
except Har2TreeError:
|
|
||||||
pass
|
|
||||||
except AttributeError:
|
|
||||||
# No `body` in rendered node
|
|
||||||
pass
|
|
||||||
return event
|
|
||||||
|
|
|
@ -354,32 +354,61 @@ def stats(tree_uuid: str):
|
||||||
@app.route('/tree/<string:tree_uuid>/misp_lookup', methods=['GET'])
|
@app.route('/tree/<string:tree_uuid>/misp_lookup', methods=['GET'])
|
||||||
@flask_login.login_required
|
@flask_login.login_required
|
||||||
def web_misp_lookup_view(tree_uuid: str):
|
def web_misp_lookup_view(tree_uuid: str):
|
||||||
hits = lookyloo.get_misp_occurrences(tree_uuid)
|
if not lookyloo.misps.available:
|
||||||
if hits:
|
flash('There are no MISP instances available.', 'error')
|
||||||
misp_root_url = lookyloo.misp.client.root_url
|
return redirect(url_for('tree', tree_uuid=tree_uuid))
|
||||||
else:
|
misps_occurrences = {}
|
||||||
misp_root_url = ''
|
for instance_name in lookyloo.misps:
|
||||||
return render_template('misp_lookup.html', uuid=tree_uuid, hits=hits, misp_root_url=misp_root_url)
|
if occurrences := lookyloo.get_misp_occurrences(tree_uuid, instance_name=instance_name):
|
||||||
|
misps_occurrences[instance_name] = occurrences
|
||||||
|
return render_template('misp_lookup.html', uuid=tree_uuid,
|
||||||
|
current_misp=lookyloo.misps.default_instance,
|
||||||
|
misps_occurrences=misps_occurrences)
|
||||||
|
|
||||||
|
|
||||||
@app.route('/tree/<string:tree_uuid>/misp_push', methods=['GET', 'POST'])
|
@app.route('/tree/<string:tree_uuid>/misp_push', methods=['GET', 'POST'])
|
||||||
@flask_login.login_required
|
@flask_login.login_required
|
||||||
def web_misp_push_view(tree_uuid: str):
|
def web_misp_push_view(tree_uuid: str):
|
||||||
error = False
|
if not lookyloo.misps.available:
|
||||||
if not lookyloo.misp.available:
|
flash('There are no MISP instances available.', 'error')
|
||||||
flash('MISP module not available.', 'error')
|
|
||||||
return redirect(url_for('tree', tree_uuid=tree_uuid))
|
return redirect(url_for('tree', tree_uuid=tree_uuid))
|
||||||
elif not lookyloo.misp.enable_push:
|
|
||||||
flash('Push not enabled in MISP module.', 'error')
|
|
||||||
return redirect(url_for('tree', tree_uuid=tree_uuid))
|
|
||||||
else:
|
|
||||||
event = lookyloo.misp_export(tree_uuid)
|
|
||||||
if isinstance(event, dict):
|
|
||||||
flash(f'Unable to generate the MISP export: {event}', 'error')
|
|
||||||
return redirect(url_for('tree', tree_uuid=tree_uuid))
|
|
||||||
|
|
||||||
if request.method == 'POST':
|
event = lookyloo.misp_export(tree_uuid)
|
||||||
|
if isinstance(event, dict):
|
||||||
|
flash(f'Unable to generate the MISP export: {event}', 'error')
|
||||||
|
return redirect(url_for('tree', tree_uuid=tree_uuid))
|
||||||
|
|
||||||
|
if request.method == 'GET':
|
||||||
|
# Initialize settings that will be displayed on the template
|
||||||
|
misp_instances_settings = {}
|
||||||
|
for name, instance in lookyloo.misps.items():
|
||||||
|
# the 1st attribute in the event is the link to lookyloo
|
||||||
|
misp_instances_settings[name] = {
|
||||||
|
'default_tags': instance.default_tags,
|
||||||
|
'fav_tags': [tag.name for tag in instance.get_fav_tags()],
|
||||||
|
'auto_publish': instance.auto_publish
|
||||||
|
}
|
||||||
|
if existing_misp_url := instance.get_existing_event_url(event[-1].attributes[0].value):
|
||||||
|
misp_instances_settings[name]['existing_event'] = existing_misp_url
|
||||||
|
|
||||||
|
cache = lookyloo.capture_cache(tree_uuid)
|
||||||
|
return render_template('misp_push_view.html',
|
||||||
|
current_misp=lookyloo.misps.default_instance,
|
||||||
|
tree_uuid=tree_uuid,
|
||||||
|
event=event[0],
|
||||||
|
misp_instances_settings=misp_instances_settings,
|
||||||
|
has_parent=True if cache and cache.parent else False)
|
||||||
|
|
||||||
|
elif request.method == 'POST':
|
||||||
# event is a MISPEvent at this point
|
# event is a MISPEvent at this point
|
||||||
|
misp_instance_name = request.form.get('misp_instance_name')
|
||||||
|
if not misp_instance_name or misp_instance_name not in lookyloo.misps:
|
||||||
|
flash(f'MISP instance {misp_instance_name} is unknown.', 'error')
|
||||||
|
return redirect(url_for('tree', tree_uuid=tree_uuid))
|
||||||
|
misp = lookyloo.misps[misp_instance_name]
|
||||||
|
if not misp.enable_push:
|
||||||
|
flash('Push not enabled in MISP module.', 'error')
|
||||||
|
return redirect(url_for('tree', tree_uuid=tree_uuid))
|
||||||
# Submit the event
|
# Submit the event
|
||||||
tags = request.form.getlist('tags')
|
tags = request.form.getlist('tags')
|
||||||
error = False
|
error = False
|
||||||
|
@ -406,30 +435,17 @@ def web_misp_push_view(tree_uuid: str):
|
||||||
events[-1].info = request.form.get('event_info')
|
events[-1].info = request.form.get('event_info')
|
||||||
|
|
||||||
try:
|
try:
|
||||||
new_events = lookyloo.misp.push(events, True if request.form.get('force_push') else False,
|
new_events = misp.push(events, True if request.form.get('force_push') else False,
|
||||||
True if request.form.get('auto_publish') else False)
|
True if request.form.get('auto_publish') else False)
|
||||||
except MISPServerError:
|
except MISPServerError:
|
||||||
flash(f'MISP returned an error, the event(s) might still have been created on {lookyloo.misp.client.root_url}', 'error')
|
flash(f'MISP returned an error, the event(s) might still have been created on {misp.client.root_url}', 'error')
|
||||||
else:
|
else:
|
||||||
if isinstance(new_events, dict):
|
if isinstance(new_events, dict):
|
||||||
flash(f'Unable to create event(s): {new_events}', 'error')
|
flash(f'Unable to create event(s): {new_events}', 'error')
|
||||||
else:
|
else:
|
||||||
for e in new_events:
|
for e in new_events:
|
||||||
flash(f'MISP event {e.id} created on {lookyloo.misp.client.root_url}', 'success')
|
flash(f'MISP event {e.id} created on {misp.client.root_url}', 'success')
|
||||||
return redirect(url_for('tree', tree_uuid=tree_uuid))
|
return redirect(url_for('tree', tree_uuid=tree_uuid))
|
||||||
else:
|
|
||||||
# the 1st attribute in the event is the link to lookyloo
|
|
||||||
existing_misp_url = lookyloo.misp.get_existing_event_url(event[-1].attributes[0].value)
|
|
||||||
|
|
||||||
fav_tags = lookyloo.misp.get_fav_tags()
|
|
||||||
cache = lookyloo.capture_cache(tree_uuid)
|
|
||||||
|
|
||||||
return render_template('misp_push_view.html', tree_uuid=tree_uuid,
|
|
||||||
event=event[0], fav_tags=fav_tags,
|
|
||||||
existing_event=existing_misp_url,
|
|
||||||
auto_publish=lookyloo.misp.auto_publish,
|
|
||||||
has_parent=True if cache and cache.parent else False,
|
|
||||||
default_tags=lookyloo.misp.default_tags)
|
|
||||||
|
|
||||||
|
|
||||||
@app.route('/tree/<string:tree_uuid>/modules', methods=['GET'])
|
@app.route('/tree/<string:tree_uuid>/modules', methods=['GET'])
|
||||||
|
@ -786,8 +802,8 @@ def tree(tree_uuid: str, node_uuid: Optional[str]=None):
|
||||||
enable_context_by_users=enable_context_by_users,
|
enable_context_by_users=enable_context_by_users,
|
||||||
enable_categorization=enable_categorization,
|
enable_categorization=enable_categorization,
|
||||||
enable_bookmark=enable_bookmark,
|
enable_bookmark=enable_bookmark,
|
||||||
misp_push=lookyloo.misp.available and lookyloo.misp.enable_push,
|
misp_push=lookyloo.misps.available and lookyloo.misps.default_misp.enable_push,
|
||||||
misp_lookup=lookyloo.misp.available and lookyloo.misp.enable_lookup,
|
misp_lookup=lookyloo.misps.available and lookyloo.misps.default_misp.enable_lookup,
|
||||||
blur_screenshot=blur_screenshot, urlnode_uuid=hostnode_to_highlight,
|
blur_screenshot=blur_screenshot, urlnode_uuid=hostnode_to_highlight,
|
||||||
auto_trigger_modules=auto_trigger_modules,
|
auto_trigger_modules=auto_trigger_modules,
|
||||||
confirm_message=confirm_message if confirm_message else 'Tick to confirm.',
|
confirm_message=confirm_message if confirm_message else 'Tick to confirm.',
|
||||||
|
|
|
@ -180,6 +180,7 @@ misp_push_fields = api.model('MISPPushFields', {
|
||||||
|
|
||||||
|
|
||||||
@api.route('/json/<string:capture_uuid>/misp_push')
|
@api.route('/json/<string:capture_uuid>/misp_push')
|
||||||
|
@api.route('/json/<string:capture_uuid>/misp_push/<string:instance_name>')
|
||||||
@api.doc(description='Push an event to a pre-configured MISP instance',
|
@api.doc(description='Push an event to a pre-configured MISP instance',
|
||||||
params={'capture_uuid': 'The UUID of the capture'},
|
params={'capture_uuid': 'The UUID of the capture'},
|
||||||
security='apikey')
|
security='apikey')
|
||||||
|
@ -188,20 +189,21 @@ class MISPPush(Resource):
|
||||||
|
|
||||||
@api.param('with_parents', 'Also push the parents of the capture (if any)')
|
@api.param('with_parents', 'Also push the parents of the capture (if any)')
|
||||||
@api.param('allow_duplicates', 'Push the event even if it is already present on the MISP instance')
|
@api.param('allow_duplicates', 'Push the event even if it is already present on the MISP instance')
|
||||||
def get(self, capture_uuid: str):
|
def get(self, capture_uuid: str, instance_name: Optional[str]=None):
|
||||||
with_parents = True if request.args.get('with_parents') else False
|
with_parents = True if request.args.get('with_parents') else False
|
||||||
allow_duplicates = True if request.args.get('allow_duplicates') else False
|
allow_duplicates = True if request.args.get('allow_duplicates') else False
|
||||||
to_return: Dict = {}
|
to_return: Dict = {}
|
||||||
if not lookyloo.misp.available:
|
misp = self.get_misp_instance(instance_name)
|
||||||
|
if not misp.available:
|
||||||
to_return['error'] = 'MISP module not available.'
|
to_return['error'] = 'MISP module not available.'
|
||||||
elif not lookyloo.misp.enable_push:
|
elif not misp.enable_push:
|
||||||
to_return['error'] = 'Push not enabled in MISP module.'
|
to_return['error'] = 'Push not enabled in MISP module.'
|
||||||
else:
|
else:
|
||||||
event = lookyloo.misp_export(capture_uuid, with_parents)
|
event = lookyloo.misp_export(capture_uuid, with_parents)
|
||||||
if isinstance(event, dict):
|
if isinstance(event, dict):
|
||||||
to_return['error'] = event
|
to_return['error'] = event
|
||||||
else:
|
else:
|
||||||
new_events = lookyloo.misp.push(event, allow_duplicates)
|
new_events = misp.push(event, allow_duplicates)
|
||||||
if isinstance(new_events, dict):
|
if isinstance(new_events, dict):
|
||||||
to_return['error'] = new_events
|
to_return['error'] = new_events
|
||||||
else:
|
else:
|
||||||
|
@ -213,22 +215,23 @@ class MISPPush(Resource):
|
||||||
return to_return
|
return to_return
|
||||||
|
|
||||||
@api.doc(body=misp_push_fields)
|
@api.doc(body=misp_push_fields)
|
||||||
def post(self, capture_uuid: str):
|
def post(self, capture_uuid: str, instance_name: Optional[str]=None):
|
||||||
parameters: Dict = request.get_json(force=True)
|
parameters: Dict = request.get_json(force=True)
|
||||||
with_parents = True if parameters.get('with_parents') else False
|
with_parents = True if parameters.get('with_parents') else False
|
||||||
allow_duplicates = True if parameters.get('allow_duplicates') else False
|
allow_duplicates = True if parameters.get('allow_duplicates') else False
|
||||||
|
|
||||||
to_return: Dict = {}
|
to_return: Dict = {}
|
||||||
if not lookyloo.misp.available:
|
misp = self.get_misp_instance(instance_name)
|
||||||
|
if not misp.available:
|
||||||
to_return['error'] = 'MISP module not available.'
|
to_return['error'] = 'MISP module not available.'
|
||||||
elif not lookyloo.misp.enable_push:
|
elif not misp.enable_push:
|
||||||
to_return['error'] = 'Push not enabled in MISP module.'
|
to_return['error'] = 'Push not enabled in MISP module.'
|
||||||
else:
|
else:
|
||||||
event = lookyloo.misp_export(capture_uuid, with_parents)
|
event = lookyloo.misp_export(capture_uuid, with_parents)
|
||||||
if isinstance(event, dict):
|
if isinstance(event, dict):
|
||||||
to_return['error'] = event
|
to_return['error'] = event
|
||||||
else:
|
else:
|
||||||
new_events = lookyloo.misp.push(event, allow_duplicates)
|
new_events = misp.push(event, allow_duplicates)
|
||||||
if isinstance(new_events, dict):
|
if isinstance(new_events, dict):
|
||||||
to_return['error'] = new_events
|
to_return['error'] = new_events
|
||||||
else:
|
else:
|
||||||
|
|
|
@ -1,11 +1,24 @@
|
||||||
{% from "macros.html" import shorten_string %}
|
<center>
|
||||||
|
<h1 class="display-4">MISP hits</h1>
|
||||||
|
<h6>Searching on URL, domain, IPs, and CNAMEs for all the nodes up to the rendered page.</h6>
|
||||||
|
<h6>Skips the entries in warnings lists enabled on your MISP instance.</h6>
|
||||||
|
{% if misps_occurrences|length > 1 %}
|
||||||
|
</br>
|
||||||
|
<hr/>
|
||||||
|
<label for="mispSelector">Select the MISP instance to search in</label>
|
||||||
|
</br>
|
||||||
|
<div class="btn-group" role="group" aria-label="MISP Selector" id="mispSelector">
|
||||||
|
{%for name in misps_occurrences %}
|
||||||
|
<button type="button" value="{{name}}" class="btn btn-outline-primary {%if name == current_misp%}active{%endif%}" href="#">{{name}}</a></li>
|
||||||
|
{%endfor%}
|
||||||
|
</div>
|
||||||
|
{%endif%}
|
||||||
|
</center>
|
||||||
|
|
||||||
<div>
|
<div id="allInstances">
|
||||||
<center>
|
{% for name, occurrences in misps_occurrences.items() %}
|
||||||
<h1 class="display-4">MISP hits</h1>
|
<div id="{{name}}" {%if name != current_misp%}style="display:none"{%endif%}>
|
||||||
<h6>Searching on URL, domain, IPs, and CNAMEs for all the nodes up to the rendered page.</h6>
|
{% set hits, root_url = occurrences %}
|
||||||
<h6>Skips the entries in warnings lists enabled on your MISP instance.</h6>
|
|
||||||
</center>
|
|
||||||
{% if hits %}
|
{% if hits %}
|
||||||
<ul>
|
<ul>
|
||||||
{% for event_id, values in hits.items() %}
|
{% for event_id, values in hits.items() %}
|
||||||
|
@ -21,3 +34,6 @@
|
||||||
{% else %}
|
{% else %}
|
||||||
No hits
|
No hits
|
||||||
{% endif %}
|
{% endif %}
|
||||||
|
</div>
|
||||||
|
{% endfor %}
|
||||||
|
</div>
|
||||||
|
|
|
@ -1,6 +1,24 @@
|
||||||
<div>
|
{% if misp_instances_settings|length > 1 %}
|
||||||
<p>Default tags: {{', '.join(default_tags)}}</p>
|
<center>
|
||||||
<form role="form" action="{{ url_for('web_misp_push_view', tree_uuid=tree_uuid) }}" method=post enctype=multipart/form-data>
|
<label for="mispSelector">Select the MISP instance to push to</label>
|
||||||
|
</br>
|
||||||
|
<div class="btn-group" role="group" aria-label="MISP Selector" id="mispSelector">
|
||||||
|
{%for name in misp_instances_settings %}
|
||||||
|
<button type="button" value="{{name}}" class="btn btn-outline-primary {%if name == current_misp%}active{%endif%}" href="#">{{name}}</a></li>
|
||||||
|
{%endfor%}
|
||||||
|
</div>
|
||||||
|
</center>
|
||||||
|
{%endif%}
|
||||||
|
|
||||||
|
<div id="allInstances">
|
||||||
|
{%for name, misp_settings in misp_instances_settings.items() %}
|
||||||
|
<div id="{{name}}" {%if name != current_misp%}style="display:none"{%endif%}>
|
||||||
|
<form role="form" action="{{ url_for('web_misp_push_view', tree_uuid=tree_uuid) }}"
|
||||||
|
method=post enctype=multipart/form-data>
|
||||||
|
<label for="misp_instance_name" class="col-sm-2 col-form-label">Submit event to:</label>
|
||||||
|
<input type="text" class="form-control" name="misp_instance_name" value="{{name}}" readonly>
|
||||||
|
<label for="defaultTags" class="col-sm-6 col-form-label">Tags attached to the event by default</label>
|
||||||
|
<input type="text" class="form-control" name="defaultTags" value="{{', '.join(misp_settings['default_tags'])}}" disabled readonly>
|
||||||
<div class="row mb-3">
|
<div class="row mb-3">
|
||||||
<div class="col-sm-10">
|
<div class="col-sm-10">
|
||||||
<label for="url" class="col-sm-2 col-form-label">Event info:</label>
|
<label for="url" class="col-sm-2 col-form-label">Event info:</label>
|
||||||
|
@ -12,20 +30,21 @@
|
||||||
<div class="col-sm-10">
|
<div class="col-sm-10">
|
||||||
<label for="tags" class="col-sm-2 col-form-label">Available tags:</label>
|
<label for="tags" class="col-sm-2 col-form-label">Available tags:</label>
|
||||||
<select class="form-control" name="tags" id="tags" multiple>
|
<select class="form-control" name="tags" id="tags" multiple>
|
||||||
{% for tag in fav_tags %}
|
{% for tag_name in misp_settings['fav_tags'] %}
|
||||||
<option value="{{ tag.name }}">{{ tag.name }}</option>
|
<option value="{{ tag_name }}">{{ tag_name }}</option>
|
||||||
{% endfor %}
|
{% endfor %}
|
||||||
</select>
|
</select>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
<div class="form-check">
|
<div class="form-check">
|
||||||
<input class="form-check-input" type="checkbox" name="auto_publish" {%if auto_publish %} checked {% endif %}></input>
|
<input class="form-check-input" type="checkbox" name="auto_publish"
|
||||||
|
{%if misp_settings.auto_publish %} checked {% endif %}></input>
|
||||||
<label for="auto_publish" class="form-check-label">Publish the event automatically</label>
|
<label for="auto_publish" class="form-check-label">Publish the event automatically</label>
|
||||||
</div>
|
</div>
|
||||||
{% if existing_event %}
|
{% if misp_settings.existing_event %}
|
||||||
<p>There is already an <a href="{{existing_event}}">event on your MISP instance</a> with this lookyloo capture.</p>
|
<p>There is already an <a href="{{misp_settings.existing_event}}">event on your MISP instance</a> with this lookyloo capture.</p>
|
||||||
<div class="form-check">
|
<div class="form-check">
|
||||||
<input class="form-check-input" type="checkbox" name="force_push" onchange="document.getElementById('btn-misp-push').disabled = !this.checked;"></input>
|
<input class="form-check-input" type="checkbox" name="force_push" onchange="document.getElementById('btn-misp-push-{{name}}').disabled = !this.checked;"></input>
|
||||||
<label for="force_push" class="form-check-label">Tick this box if you want to push anyway</label>
|
<label for="force_push" class="form-check-label">Tick this box if you want to push anyway</label>
|
||||||
</div>
|
</div>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
|
@ -35,6 +54,9 @@
|
||||||
<label for="with_parents" class="form-check-label">Also push the parents</label>
|
<label for="with_parents" class="form-check-label">Also push the parents</label>
|
||||||
</div>
|
</div>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
<button type="submit" class="btn btn-primary" id="btn-misp-push" {% if existing_event %}disabled=true{% endif %}>Push to MISP</button>
|
<button type="submit" class="btn btn-primary" id="btn-misp-push-{{name}}"
|
||||||
|
{% if misp_settings.existing_event %}disabled=true{% endif %}>Push to {{name}}</button>
|
||||||
</form>
|
</form>
|
||||||
</div>
|
</div>
|
||||||
|
{%endfor%}
|
||||||
|
</div>
|
||||||
|
|
|
@ -94,14 +94,26 @@
|
||||||
$('#mispPushModal').on('show.bs.modal', function(e) {
|
$('#mispPushModal').on('show.bs.modal', function(e) {
|
||||||
var button = $(e.relatedTarget);
|
var button = $(e.relatedTarget);
|
||||||
var modal = $(this);
|
var modal = $(this);
|
||||||
modal.find('.modal-body').load(button.data("remote"));
|
modal.find('.modal-body').load(button.data("remote"), function(result){
|
||||||
|
$('#mispSelector button').on('click', function(e){
|
||||||
|
var thisBtn = $(this);
|
||||||
|
thisBtn.addClass('active').siblings().removeClass('active');
|
||||||
|
$('#'+thisBtn.val()).show().siblings().hide()
|
||||||
|
});
|
||||||
|
});
|
||||||
});
|
});
|
||||||
</script>
|
</script>
|
||||||
<script>
|
<script>
|
||||||
$('#mispLookupModal').on('show.bs.modal', function(e) {
|
$('#mispLookupModal').on('show.bs.modal', function(e) {
|
||||||
var button = $(e.relatedTarget);
|
var button = $(e.relatedTarget);
|
||||||
var modal = $(this);
|
var modal = $(this);
|
||||||
modal.find('.modal-body').load(button.data("remote"));
|
modal.find('.modal-body').load(button.data("remote"), function(result){
|
||||||
|
$('#mispSelector button').on('click', function(e){
|
||||||
|
var thisBtn = $(this);
|
||||||
|
thisBtn.addClass('active').siblings().removeClass('active');
|
||||||
|
$('#'+thisBtn.val()).show().siblings().hide()
|
||||||
|
});
|
||||||
|
});
|
||||||
});
|
});
|
||||||
</script>
|
</script>
|
||||||
<script>
|
<script>
|
||||||
|
|
Loading…
Reference in New Issue