From e1e9609ad99d5b2d1e705dccfc8c12ea82dba834 Mon Sep 17 00:00:00 2001 From: terrtia Date: Thu, 29 Feb 2024 14:56:45 +0100 Subject: [PATCH] chg: [api] get object + get investigation --- bin/lib/Investigations.py | 30 ++++++++++++++++++---- bin/lib/ail_api.py | 2 +- bin/lib/ail_core.py | 4 ++- bin/lib/objects/Favicons.py | 15 ++++++++--- bin/lib/objects/ail_objects.py | 38 ++++++++++++++++++++++++++- var/www/Flask_server.py | 11 ++++++-- var/www/blueprints/api_rest.py | 47 +++++++++++++++++++++++++++++----- 7 files changed, 127 insertions(+), 20 deletions(-) diff --git a/bin/lib/Investigations.py b/bin/lib/Investigations.py index 855beb49..ede905c8 100755 --- a/bin/lib/Investigations.py +++ b/bin/lib/Investigations.py @@ -152,25 +152,30 @@ class Investigation(object): return r_tracking.smembers(f'investigations:misp:{self.uuid}') # # TODO: DATE FORMAT - def get_metadata(self, r_str=False): + def get_metadata(self, options=set(), r_str=False): if r_str: analysis = self.get_analysis_str() threat_level = self.get_threat_level_str() else: analysis = self.get_analysis() threat_level = self.get_threat_level() - return {'uuid': self.uuid, - 'name': self.get_name(), + + # 'name': self.get_name(), + meta = {'uuid': self.uuid, 'threat_level': threat_level, 'analysis': analysis, - 'tags': self.get_tags(), + 'tags': list(self.get_tags()), 'user_creator': self.get_creator_user(), 'date': self.get_date(), 'timestamp': self.get_timestamp(r_str=r_str), 'last_change': self.get_last_change(r_str=r_str), 'info': self.get_info(), 'nb_objects': self.get_nb_objects(), - 'misp_events': self.get_misp_events()} + 'misp_events': list(self.get_misp_events()) + } + if 'objects' in options: + meta['objects'] = self.get_objects() + return meta def set_name(self, name): r_tracking.hset(f'investigations:data:{self.uuid}', 'name', name) @@ -368,6 +373,21 @@ def get_investigations_selector(): #### API #### +def api_get_investigation(investigation_uuid): # TODO check if is UUIDv4 + investigation = Investigation(investigation_uuid) + if not investigation.exists(): + return {'status': 'error', 'reason': 'Investigation Not Found'}, 404 + + meta = investigation.get_metadata(options={'objects'}, r_str=False) + # objs = [] + # for obj in investigation.get_objects(): + # obj_meta = ail_objects.get_object_meta(obj["type"], obj["subtype"], obj["id"], flask_context=True) + # comment = investigation.get_objects_comment(f'{obj["type"]}:{obj["subtype"]}:{obj["id"]}') + # if comment: + # obj_meta['comment'] = comment + # objs.append(obj_meta) + return meta, 200 + # # TODO: CHECK Mandatory Fields # # TODO: SANITYZE Fields # # TODO: Name ????? diff --git a/bin/lib/ail_api.py b/bin/lib/ail_api.py index 1b91816f..1e05b1cc 100755 --- a/bin/lib/ail_api.py +++ b/bin/lib/ail_api.py @@ -80,4 +80,4 @@ def authenticate_user(token, ip_address): return {'status': 'error', 'reason': 'Authentication failed'}, 401 except Exception as e: print(e) # TODO Logs - return {'status': 'error', 'reason': 'Malformed Authentication String'}, 400 \ No newline at end of file + return {'status': 'error', 'reason': 'Malformed Authentication String'}, 400 diff --git a/bin/lib/ail_core.py b/bin/lib/ail_core.py index 34416c19..bbe17969 100755 --- a/bin/lib/ail_core.py +++ b/bin/lib/ail_core.py @@ -20,6 +20,8 @@ AIL_OBJECTS = sorted({'chat', 'chat-subchannel', 'chat-thread', 'cookie-name', ' 'domain', 'etag', 'favicon', 'file-name', 'hhhash', 'item', 'image', 'message', 'pgp', 'screenshot', 'title', 'user-account', 'username'}) +AIL_OBJECTS_WITH_SUBTYPES = {'chat', 'chat-subchannel', 'cryptocurrency', 'pgp', 'username', 'user-account'} + def get_ail_uuid(): ail_uuid = r_serv_db.get('ail:uuid') if not ail_uuid: @@ -48,7 +50,7 @@ def get_all_objects(): return AIL_OBJECTS def get_objects_with_subtypes(): - return ['chat', 'cryptocurrency', 'pgp', 'username', 'user-account'] + return AIL_OBJECTS_WITH_SUBTYPES def get_object_all_subtypes(obj_type): # TODO Dynamic subtype if obj_type == 'chat': diff --git a/bin/lib/objects/Favicons.py b/bin/lib/objects/Favicons.py index c810864e..6dc7d5e5 100755 --- a/bin/lib/objects/Favicons.py +++ b/bin/lib/objects/Favicons.py @@ -62,11 +62,18 @@ class Favicon(AbstractDaterangeObject): filename = os.path.join(FAVICON_FOLDER, self.get_rel_path()) return os.path.realpath(filename) - def get_file_content(self): + def get_file_content(self, r_type='str'): filepath = self.get_filepath() - with open(filepath, 'rb') as f: - file_content = BytesIO(f.read()) - return file_content + if r_type == 'str': + with open(filepath, 'rb') as f: + file_content = f.read() + b64 = base64.b64encode(file_content) + # b64 = base64.encodebytes(file_content) + return b64.decode() + elif r_type == 'io': + with open(filepath, 'rb') as f: + file_content = BytesIO(f.read()) + return file_content def get_content(self, r_type='str'): return self.get_file_content() diff --git a/bin/lib/objects/ail_objects.py b/bin/lib/objects/ail_objects.py index c521093e..1e5a3024 100755 --- a/bin/lib/objects/ail_objects.py +++ b/bin/lib/objects/ail_objects.py @@ -12,7 +12,7 @@ from lib.exceptions import AILObjectUnknown from lib.ConfigLoader import ConfigLoader -from lib.ail_core import get_all_objects, get_object_all_subtypes +from lib.ail_core import get_all_objects, get_object_all_subtypes, get_objects_with_subtypes from lib import correlations_engine from lib import relationships_engine from lib import btc_ail @@ -47,6 +47,9 @@ config_loader = None def is_valid_object_type(obj_type): return obj_type in get_all_objects() +def is_object_subtype(obj_type): + return obj_type in get_objects_with_subtypes() + def is_valid_object_subtype(obj_type, subtype): return subtype in get_object_all_subtypes(obj_type) @@ -117,6 +120,39 @@ def exists_obj(obj_type, subtype, obj_id): else: return False +#### API #### + +def api_get_object(obj_type, obj_subtype, obj_id): + if not obj_id: + return {'status': 'error', 'reason': 'Invalid object id'}, 400 + if not is_valid_object_type(obj_type): + return {'status': 'error', 'reason': 'Invalid object type'}, 400 + if obj_subtype: + if not is_valid_object_subtype(obj_type, subtype): + return {'status': 'error', 'reason': 'Invalid object subtype'}, 400 + obj = get_object(obj_type, obj_subtype, obj_id) + if not obj.exists(): + return {'status': 'error', 'reason': 'Object Not Found'}, 404 + options = {'chat', 'content', 'files-names', 'images', 'parent', 'parent_meta', 'reactions', 'thread', 'user-account'} + return obj.get_meta(options=options), 200 + + +def api_get_object_type_id(obj_type, obj_id): + if not is_valid_object_type(obj_type): + return {'status': 'error', 'reason': 'Invalid object type'}, 400 + if is_object_subtype(obj_type): + subtype, obj_id = obj_type.split('/', 1) + else: + subtype = None + return api_get_object(obj_type, subtype, obj_id) + + +def api_get_object_global_id(global_id): + obj_type, subtype, obj_id = global_id.split(':', 2) + return api_get_object(obj_type, subtype, obj_id) + +#### --API-- #### + ######################################################################################### ######################################################################################### ######################################################################################### diff --git a/var/www/Flask_server.py b/var/www/Flask_server.py index 5c23b1fb..fe6d6d1d 100755 --- a/var/www/Flask_server.py +++ b/var/www/Flask_server.py @@ -234,18 +234,25 @@ def _handle_client_error(e): anchor_id = anchor_id.replace('/', '_') api_doc_url = 'https://github.com/ail-project/ail-framework/tree/master/doc#{}'.format(anchor_id) res_dict['documentation'] = api_doc_url - return Response(json.dumps(res_dict, indent=2, sort_keys=True), mimetype='application/json'), 405 + return Response(json.dumps(res_dict) + '\n', mimetype='application/json'), 405 else: return e @app.errorhandler(404) def error_page_not_found(e): if request.path.startswith('/api/'): ## # TODO: add baseUrl - return Response(json.dumps({"status": "error", "reason": "404 Not Found"}, indent=2, sort_keys=True), mimetype='application/json'), 404 + return Response(json.dumps({"status": "error", "reason": "404 Not Found"}) + '\n', mimetype='application/json'), 404 else: # avoid endpoint enumeration return page_not_found(e) +@app.errorhandler(500) +def _handle_client_error(e): + if request.path.startswith('/api/'): + return Response(json.dumps({"status": "error", "reason": "Server Error"}) + '\n', mimetype='application/json'), 500 + else: + return e + @login_required def page_not_found(e): # avoid endpoint enumeration diff --git a/var/www/blueprints/api_rest.py b/var/www/blueprints/api_rest.py index 5b19af4b..f0f91fc8 100644 --- a/var/www/blueprints/api_rest.py +++ b/var/www/blueprints/api_rest.py @@ -21,14 +21,14 @@ from lib import ail_core from lib import ail_updates from lib import crawlers +from lib import Investigations from lib import Tag from lib.objects import ail_objects -from importer.FeederImporter import api_add_json_feeder_to_queue - from lib.objects import Domains from lib.objects import Titles +from importer.FeederImporter import api_add_json_feeder_to_queue # ============ BLUEPRINT ============ @@ -75,8 +75,8 @@ def token_required(user_role): # ============ FUNCTIONS ============ -def create_json_response(data, status_code): # TODO REMOVE INDENT ???????????????????? - return Response(json.dumps(data, indent=2, sort_keys=True), mimetype='application/json'), status_code +def create_json_response(data, status_code): + return Response(json.dumps(data) + "\n", mimetype='application/json'), status_code # ============= ROUTES ============== @@ -150,12 +150,38 @@ def import_json_item(): return Response(json.dumps(res[0]), mimetype='application/json'), res[1] # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # -# # # # # # # # # # # # # # # OBJECTS # # # # # # # # # # # # # # # # # # +# # # # # # # # # # # # # # # OBJECTS # # # # # # # # # # # # # # # # # # # TODO LIST OBJ TYPES + SUBTYPES # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # +@api_rest.route("api/v1/object", methods=['GET']) # TODO options +@token_required('read_only') +def v1_object(): + obj_gid = request.args.get('gid') + if obj_gid: + r = ail_objects.api_get_object_global_id(obj_gid) + else: + obj_type = request.args.get('type') + obj_subtype = request.args.get('subtype') + obj_id = request.args.get('id') + r = ail_objects.api_get_object(obj_type, obj_subtype, obj_id) + print(r[0]) + return create_json_response(r[0], r[1]) +@api_rest.route("api/v1/obj/gid/", methods=['GET']) # TODO REMOVE ME ???? +@token_required('read_only') +def v1_object_global_id(object_global_id): + r = ail_objects.api_get_object_global_id(object_global_id) + return create_json_response(r[0], r[1]) + +# @api_rest.route("api/v1/object///", methods=['GET']) +@api_rest.route("api/v1/obj//", methods=['GET']) # TODO REMOVE ME ???? +@token_required('read_only') +def v1_object_type_id(object_type, object_id): + r = ail_objects.api_get_object_type_id(object_type, object_id) + return create_json_response(r[0], r[1]) + # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # -# # # # # # # # # # # # # # # TITLES # # # # # # # # # # # # # # # # # # +# # # # # # # # # # # # # # # TITLES # # # # # # # # # # # # # # # # # # # TODO TO REVIEW # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # @api_rest.route("api/v1/titles/download", methods=['GET']) @@ -184,4 +210,13 @@ def objects_titles_download_unsafe(): all_titles[title_content].append(domain.get_id()) return Response(json.dumps(all_titles), mimetype='application/json'), 200 +# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # +# # # # # # # # # # # # # # # INVESTIGATIONS # # # # # # # # # # # # # # # +# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # +@api_rest.route("api/v1/investigation/", methods=['GET']) # TODO options +@token_required('read_only') +def v1_investigation(investigation_uuid): + r = Investigations.api_get_investigation(investigation_uuid) + return create_json_response(r[0], r[1]) +# TODO CATCH REDIRECT