From b8342cfa69e1edc7247338a518adeabda2bd8679 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Tue, 18 Feb 2020 17:02:00 +0100 Subject: [PATCH] chg: [UI MISP export] export to MISP instance --- OVERVIEW.md | 1 + bin/export/AILObjects.py | 21 +++ bin/export/MispExport.py | 73 ++++++++- bin/export/MispImport.py | 8 +- configs/core.cfg.sample | 5 + var/www/blueprints/import_export.py | 33 +++- .../block_create_misp_event.html | 46 ++++++ .../import_export/export_object.html | 150 +++++++++++------- 8 files changed, 264 insertions(+), 73 deletions(-) create mode 100755 bin/export/AILObjects.py create mode 100644 var/www/templates/import_export/block_create_misp_event.html diff --git a/OVERVIEW.md b/OVERVIEW.md index f677da42..e634f31b 100644 --- a/OVERVIEW.md +++ b/OVERVIEW.md @@ -23,6 +23,7 @@ Redis and ARDB overview DB 7 - Metadata DB 8 - Statistics DB 9 - Crawler + DB 10 - Objects * ARDB on TCP port - DB 0 - Lines duplicate diff --git a/bin/export/AILObjects.py b/bin/export/AILObjects.py new file mode 100755 index 00000000..36218dbb --- /dev/null +++ b/bin/export/AILObjects.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +import os +import sys +import uuid +import redis + +sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib')) +sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages')) +import ConfigLoader + +config_loader = ConfigLoader.ConfigLoader() +r_serv_objects = config_loader.get_redis_conn("ARDB_Objects") +config_loader = None + +def create_map_obj_uuid_golbal_id(obj_uuid, global_id): + r_serv_objects.sadd('all_object:uuid', obj_uuid) + r_serv_objects.sadd('all_object:global_id', global_id) + r_serv_objects.sadd('object:map:uuid_id:{}'.format(obj_uuid), global_id) + r_serv_objects.sadd('object:map:id_uuid:{}'.format(global_id), obj_uuid) diff --git a/bin/export/MispExport.py b/bin/export/MispExport.py index 6f4dbe3c..343509f1 100755 --- a/bin/export/MispExport.py +++ b/bin/export/MispExport.py @@ -18,6 +18,10 @@ import Screenshot import Correlate_object +# # TODO: # FIXME: REFRACTOR ME => use UI/Global config +sys.path.append('../../configs/keys') +from mispKEYS import misp_url, misp_key, misp_verifycert + # MISP from pymisp import MISPEvent, MISPObject, PyMISP @@ -38,7 +42,6 @@ def sanitize_obj_export_lvl(lvl): return lvl def get_export_filename(json_content): - print(json_content) return 'ail_export.json' def create_in_memory_file(json_content): @@ -211,7 +214,7 @@ def add_obj_to_create_by_lvl(all_obj_to_export, set_relationship, dict_obj, lvl) add_obj_to_create_by_lvl(all_obj_to_export, set_relationship, dict_obj, lvl) -def create_list_of_objs_to_export(l_obj): +def create_list_of_objs_to_export(l_obj, r_type='json'): all_obj_to_export = set() set_relationship = set() for obj in l_obj: @@ -236,10 +239,10 @@ def create_list_of_objs_to_export(l_obj): # add object to event event.add_object(dict_misp_obj[obj_global_id]) - #misp = PyMISP('https://127.0.0.1:8443/', 'uXgcN42b7xuL88XqK5hubwD8Q8596VrrBvkHQzB0', False) - #misp.add_event(event, pythonify=True) - return event.to_json() - + if r_type == 'json': + return event.to_json() + else: + return event def create_all_misp_obj(all_obj_to_export, set_relationship): dict_misp_obj = {} @@ -322,9 +325,65 @@ def get_relationship_between_global_obj(obj_global_id_1, obj_global_id_2): return {'relation': 'extracted-from', 'src': src, 'dest': dest} # replave by crawled-from return None +def sanitize_event_distribution(distribution): + try: + int(distribution) + if (0 <= distribution <= 3): + return distribution + else: + return 0 + except: + return 0 + +def sanitize_event_threat_level_id(threat_level_id): + try: + int(threat_level_id) + if (1 <= threat_level_id <= 4): + return threat_level_id + else: + return 4 + except: + return 4 + +def sanitize_event_analysis(analysis): + try: + int(analysis) + if (0 <= analysis <= 2): + return analysis + else: + return 0 + except: + return 0 + +def create_misp_event(event, distribution=0, threat_level_id=4, publish=False, analysis=0, event_info=None): + if event_info: + event.info = event_info + event.distribution = sanitize_event_distribution(distribution) + event.threat_level_id = sanitize_event_threat_level_id(threat_level_id) + event.analysis = sanitize_event_analysis(analysis) + if publish: + event.publish() + + # # TODO: handle multiple MISP instance + misp = PyMISP(misp_url, misp_key, misp_verifycert) + misp_event = misp.add_event(event, pythonify=True) + # # TODO: handle error + event_metadata = extract_event_metadata(misp_event) + return event_metadata + +def extract_event_metadata(event): + event_metadata = {} + event_metadata['uuid'] = event.uuid + event_metadata['id'] = event.id + if misp_url[-1] == '/': + event_metadata['url'] = misp_url + 'events/view/' + str(event_metadata['id']) + else: + event_metadata['url'] = misp_url + '/events/view/' + str(event_metadata['id']) + return event_metadata + ###### # -# EXPORT LVL DEFINITION: +# EXPORT LVL DEFINITION: (== Correl PARTIAL Only add core item Correlation # LVL 1 => DETAILED Also add correlated_items correlation diff --git a/bin/export/MispImport.py b/bin/export/MispImport.py index 6365b6d8..9811d53b 100755 --- a/bin/export/MispImport.py +++ b/bin/export/MispImport.py @@ -16,7 +16,7 @@ import Domain import Screenshot import Correlate_object -import Import +import AILObjects # MISP from pymisp import MISPEvent, MISPObject, PyMISP @@ -209,11 +209,7 @@ def create_obj_relationships(map_uuid_global_id, misp_obj): def create_map_all_obj_uuid_golbal_id(map_uuid_global_id): for obj_uuid in map_uuid_global_id: - create_map_objuuid_golbal_id(obj_uuid, map_uuid_global_id[obj_uuid]) - -def create_map_objuuid_golbal_id(obj_uuid, global_id): - print(obj_uuid) - print(global_id) + AILObjects.create_map_obj_uuid_golbal_id(obj_uuid, map_uuid_global_id[obj_uuid]) def import_objs_from_file(filepath): map_uuid_global_id = {} diff --git a/configs/core.cfg.sample b/configs/core.cfg.sample index 632e1d07..9cea213f 100644 --- a/configs/core.cfg.sample +++ b/configs/core.cfg.sample @@ -211,6 +211,11 @@ host = localhost port = 6382 db = 9 +[ARDB_Objects] +host = localhost +port = 6382 +db = 10 + [Url] cc_critical = DE diff --git a/var/www/blueprints/import_export.py b/var/www/blueprints/import_export.py index 60801bad..332f7180 100644 --- a/var/www/blueprints/import_export.py +++ b/var/www/blueprints/import_export.py @@ -90,6 +90,10 @@ def export_object(): def export_object_file(): l_obj_to_export = [] l_obj_invalid = [] + + export_to_misp = False + dict_misp_event_export = {} + for obj_tuple in list(request.form): l_input = request.form.getlist(obj_tuple) if len(l_input) == 3: @@ -114,6 +118,13 @@ def export_object_file(): else: if obj_id: l_obj_invalid.append(obj_dict) + else: + dict_misp_event_export[str(obj_tuple)] = request.form.get(obj_tuple) + + if dict_misp_event_export.get('export_to_misp', None): + export_to_misp = True + else: + dict_misp_event_export = None if l_obj_invalid: for obj_dict in l_obj_to_export: @@ -124,10 +135,20 @@ def export_object_file(): obj_dict['type'] = Correlate_object.get_obj_str_type_subtype(obj_dict['type'], obj_dict.get('subtype', None)) return render_template("export_object.html", l_obj_to_export=l_obj_to_export, - l_obj_invalid=l_obj_invalid) + l_obj_invalid=l_obj_invalid, dict_misp_event_export=dict_misp_event_export) else: - - json_export = MispExport.create_list_of_objs_to_export(l_obj_to_export) - export_filename = MispExport.get_export_filename(json_export) - json_export = MispExport.create_in_memory_file(json_export) - return send_file(json_export, as_attachment=True, attachment_filename=export_filename) + if export_to_misp: + event = MispExport.create_list_of_objs_to_export(l_obj_to_export, r_type='event') + event_metadata = MispExport.create_misp_event(event, distribution=dict_misp_event_export.get('export_to_misp', None), + threat_level_id=dict_misp_event_export.get('misp_threat_level_id', None), + publish=dict_misp_event_export.get('misp_publish', None), + analysis=dict_misp_event_export.get('misp_event_analysis', None), + event_info=dict_misp_event_export.get('misp_event_info', None)) + return render_template("export_object.html", l_obj_to_export=l_obj_to_export, + event_metadata=event_metadata, + l_obj_invalid=[], dict_misp_event_export=[]) + else: + json_export = MispExport.create_list_of_objs_to_export(l_obj_to_export) + export_filename = MispExport.get_export_filename(json_export) + json_export = MispExport.create_in_memory_file(json_export) + return send_file(json_export, as_attachment=True, attachment_filename=export_filename) diff --git a/var/www/templates/import_export/block_create_misp_event.html b/var/www/templates/import_export/block_create_misp_event.html new file mode 100644 index 00000000..f6cdfb6a --- /dev/null +++ b/var/www/templates/import_export/block_create_misp_event.html @@ -0,0 +1,46 @@ +
+ +
+ +
+
+ +
+ +
+ +
+
+ +
+ +
+ +
+
+ +
+ +
+ +
+
+ +
+ + +
diff --git a/var/www/templates/import_export/export_object.html b/var/www/templates/import_export/export_object.html index d53cd8ee..f819b4ef 100644 --- a/var/www/templates/import_export/export_object.html +++ b/var/www/templates/import_export/export_object.html @@ -28,71 +28,100 @@
-

MISP exporter

+
+
+
MISP MISP Exporter
+
+
-
+ -
+
-

Select a list of objects to export

+

Select a list of objects to export

+ +
+
Object Type
+
Object ID
+
Lvl
+
+ +
+
+
+
+
+ + + + +
+ {% for obj_dict in l_obj_to_export %} + {% with obj_type=obj_dict['type'], obj_id=obj_dict['id'], obj_lvl=obj_dict['lvl'], input_uuid=obj_dict['uuid'], obj_error=False%} + {% include 'import_export/block_to_export_input.html' %} + {% endwith %} + {% endfor %} + +
+ + {% for obj_dict in l_obj_invalid %} + {% with obj_type=obj_dict['type'], obj_id=obj_dict['id'], obj_lvl=obj_dict['lvl'], input_uuid=obj_dict['uuid'], obj_error=True%} + {% include 'import_export/block_to_export_input.html' %} + {% endwith %} + {% endfor %} + +
+
+
+
-
-
Object Type
-
Object ID
-
Lvl
-
-
-
-
-
- - - - -
- {% for obj_dict in l_obj_to_export %} - {% with obj_type=obj_dict['type'], obj_id=obj_dict['id'], obj_lvl=obj_dict['lvl'], input_uuid=obj_dict['uuid'], obj_error=False%} - {% include 'import_export/block_to_export_input.html' %} - {% endwith %} - {% endfor %} +
+ JSON Export  +
+ + +
+
-
+
+ {% include 'import_export/block_create_misp_event.html' %} +
+ +
+ +
+ + - {% for obj_dict in l_obj_invalid %} - {% with obj_type=obj_dict['type'], obj_id=obj_dict['id'], obj_lvl=obj_dict['lvl'], input_uuid=obj_dict['uuid'], obj_error=True%} - {% include 'import_export/block_to_export_input.html' %} - {% endwith %} - {% endfor %} - -
-
-
-
-
- -
- + {% if event_metadata %} + MISP Event Created: + + {{ event_metadata['url'] }} + + {% endif %}
@@ -101,6 +130,11 @@ $(document).ready(function(){ $("#page-Decoded").addClass("active"); $("#nav_misp_export").addClass("active"); + export_to_misp_input_controler(); + + $('#export_to_misp').change(function () { + export_to_misp_input_controler(); + }); }); function toggle_sidebar(){ @@ -141,6 +175,14 @@ function uuidv4() { ); } +function export_to_misp_input_controler() { + if($('#export_to_misp').is(':checked')){ + $("#export_to_misp_div").show(); + }else{ + $("#export_to_misp_div").hide(); + } +} +