diff --git a/bin/Decoder.py b/bin/Decoder.py index c309a035..ef2cfc9a 100755 --- a/bin/Decoder.py +++ b/bin/Decoder.py @@ -20,6 +20,8 @@ from Helper import Process from packages import Paste from packages import Item +from lib import Decoded + import re import signal @@ -124,8 +126,7 @@ def save_hash(decoder_name, message, date, decoded): # Domain Object if Item.is_crawled(message): domain = Item.get_item_domain(message) - serv_metadata.sadd('hash_domain:{}'.format(domain), hash) # domain - hash map - serv_metadata.sadd('domain_hash:{}'.format(hash), domain) # hash - domain map + Decoded.save_domain_decoded(domain, hash) def save_hash_on_disk(decode, type, hash, json_data): diff --git a/bin/export/MispExport.py b/bin/export/MispExport.py index abefa86a..33740984 100755 --- a/bin/export/MispExport.py +++ b/bin/export/MispExport.py @@ -321,6 +321,7 @@ if __name__ == '__main__': {'id': 'bfd5f1d89e55b10a8b122a9d7ce31667ec1d086a', 'type': 'decoded', 'lvl': 2}, #{'id': 'a92d459f70c4dea8a14688f585a5e2364be8b91fbf924290ead361d9b909dcf1', 'type': 'image', 'lvl': 3}, {'id': 'archive/pastebin.com_pro/2020/01/27/iHjcWhkD.gz', 'type': 'item', 'lvl': 1}, + {'id': '0xA4BB02A75E6AF448', 'type': 'pgp', 'subtype': 'key', 'lvl': 1}, {'id': '15efuhpw5V9B1opHAgNXKPBPqdYALXP4hc', 'type': 'cryptocurrency', 'subtype': 'bitcoin', 'lvl': 1} ] create_list_of_objs_to_export(l_obj, mode='union') diff --git a/bin/export/MispImport.py b/bin/export/MispImport.py index 521e5f1c..4c40be86 100755 --- a/bin/export/MispImport.py +++ b/bin/export/MispImport.py @@ -8,11 +8,11 @@ import redis sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib')) sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages')) +import Item import Cryptocurrency import Pgp import Decoded import Domain -import Item import Screenshot import Correlate_object @@ -31,8 +31,18 @@ def get_global_id(obj_type, obj_id, obj_subtype=None): # sub type # obj type # obj value -def get_global_id_from_misp_obj(misp_obj): - pass +def get_global_id_from_id(global_id): + obj_meta = {} + global_id = global_id.split(':', 3) + if len(global_id) > 2: + obj_meta['type'] = global_id[0] + obj_meta['subtype'] = global_id[1] + obj_meta['id'] = global_id[2] + else: + obj_meta['type'] = global_id[0] + obj_meta['subtype'] = None + obj_meta['id'] = global_id[1] + return obj_meta def get_misp_obj_tag(misp_obj): if misp_obj.attributes: @@ -49,7 +59,7 @@ def get_object_metadata(misp_obj): if 'first_seen' in misp_obj.keys(): obj_meta['first_seen'] = misp_obj.first_seen if 'last_seen' in misp_obj.keys(): - obj_meta['last_seen'] = misp_obj.first_seen + obj_meta['last_seen'] = misp_obj.last_seen obj_meta['tags'] = get_misp_obj_tag(misp_obj) return obj_meta @@ -65,9 +75,9 @@ def unpack_item_obj(map_uuid_global_id, misp_obj): if obj_id and io_content: res = Item.create_item(obj_id, obj_meta, io_content) - print(res) + #print(res) - map_uuid_global_id[misp_obj.uuid] = get_global_id('item', obj_id) + map_uuid_global_id[misp_obj.uuid] = get_global_id('item', obj_id) @@ -85,12 +95,12 @@ def unpack_obj_pgp(map_uuid_global_id, misp_obj): if obj_id and obj_subtype: obj_meta = get_object_metadata(misp_obj) + print(obj_id) + print(obj_meta) res = Pgp.pgp.create_correlation(obj_subtype, obj_id, obj_meta) - print(res) map_uuid_global_id[misp_obj.uuid] = get_global_id('pgp', obj_id, obj_subtype=obj_subtype) - #get_obj_relationship(misp_obj) def unpack_obj_cryptocurrency(map_uuid_global_id, misp_obj): obj_id = None @@ -103,19 +113,11 @@ def unpack_obj_cryptocurrency(map_uuid_global_id, misp_obj): # valid cryptocurrency type if obj_subtype and obj_id: - print('crypto') - print(obj_id) - print(obj_subtype) - obj_meta = get_object_metadata(misp_obj) - print(obj_meta) res = Cryptocurrency.cryptocurrency.create_correlation(obj_subtype, obj_id, obj_meta) - print(res) map_uuid_global_id[misp_obj.uuid] = get_global_id('pgp', obj_id, obj_subtype=obj_subtype) - #get_obj_relationship(misp_obj) - def get_obj_type_from_relationship(misp_obj): obj_uuid = misp_obj.uuid obj_type = None @@ -128,10 +130,6 @@ def get_obj_type_from_relationship(misp_obj): obj_type = 'decoded' return obj_type -def get_obj_relationship(misp_obj): - for item in misp_obj.ObjectReference: - print(item.to_json()) - # # TODO: covert md5 and sha1 to expected def unpack_file(map_uuid_global_id, misp_obj): @@ -149,7 +147,8 @@ def unpack_file(map_uuid_global_id, misp_obj): # # TODO: use/verify specified mimetype elif attribute.object_relation == 'mimetype': - print(attribute.value) + #print(attribute.value) + pass # # TODO: support more elif attribute.object_relation == 'sha1' and obj_type == 'decoded': @@ -158,7 +157,6 @@ def unpack_file(map_uuid_global_id, misp_obj): obj_id = attribute.value if obj_id and io_content: - print(obj_type) obj_meta = get_object_metadata(misp_obj) if obj_type == 'screenshot': #Screenshot.create_screenshot(obj_id, obj_meta, io_content) @@ -166,34 +164,35 @@ def unpack_file(map_uuid_global_id, misp_obj): else: #decoded Decoded.create_decoded(obj_id, obj_meta, io_content) + map_uuid_global_id[misp_obj.uuid] = get_global_id('item', obj_id) + def get_misp_import_fct(map_uuid_global_id, misp_obj): - #print(misp_obj.ObjectReference) - #for item in misp_obj.ObjectReference: - # print(item.to_json()) - #obj_meta = get_object_metadata(misp_obj) - - #print(misp_obj.name) - if misp_obj.name == 'ail-leak': - #unpack_item_obj(map_uuid_global_id, misp_obj) - #print(misp_obj.to_json()) + unpack_item_obj(map_uuid_global_id, misp_obj) pass elif misp_obj.name == 'domain-ip': pass elif misp_obj.name == 'pgp-meta': - #unpack_obj_pgp(map_uuid_global_id, misp_obj) + unpack_obj_pgp(map_uuid_global_id, misp_obj) pass elif misp_obj.name == 'coin-address': - #unpack_obj_cryptocurrency(map_uuid_global_id, misp_obj) + unpack_obj_cryptocurrency(map_uuid_global_id, misp_obj) pass elif misp_obj.name == 'file': unpack_file(map_uuid_global_id, misp_obj) - print() - print('---') - print() - #unpack_item_obj(map_uuid_global_id, misp_obj) pass +# import relationship between objects +def create_obj_relationships(map_uuid_global_id, misp_obj): + if misp_obj.uuid in map_uuid_global_id: + for relationship in misp_obj.ObjectReference: + if relationship.referenced_uuid in map_uuid_global_id: + obj_meta_src = get_global_id_from_id(map_uuid_global_id[relationship.object_uuid]) + obj_meta_target = get_global_id_from_id(map_uuid_global_id[relationship.referenced_uuid]) + Correlate_object.create_obj_relationship(obj_meta_src['type'], obj_meta_src['id'], obj_meta_target['type'], obj_meta_target['id'], + obj1_subtype=obj_meta_src['subtype'], obj2_subtype=obj_meta_target['subtype']) + + def import_objs_from_file(filepath): event_to_import = MISPEvent() event_to_import.load_file(filepath) @@ -203,12 +202,17 @@ def import_objs_from_file(filepath): for misp_obj in event_to_import.objects: get_misp_import_fct(map_uuid_global_id, misp_obj) - print(map_uuid_global_id) + for misp_obj in event_to_import.objects: + create_obj_relationships(map_uuid_global_id, misp_obj) + + #print(map_uuid_global_id) if __name__ == '__main__': # misp = PyMISP('https://127.0.0.1:8443/', 'uXgcN42b7xuL88XqK5hubwD8Q8596VrrBvkHQzB0', False) - #import_objs_from_file('test_import_item.json') import_objs_from_file('test_import_item.json') + + #Decoded.delete_correlation('23a44cc266880d26386a0a77318afbe09696f935') + #Pgp.pgp.delete_correlation('key', '0xA4BB02A75E6AF448') diff --git a/bin/lib/Correlate_object.py b/bin/lib/Correlate_object.py index fe5599d3..45706d6f 100755 --- a/bin/lib/Correlate_object.py +++ b/bin/lib/Correlate_object.py @@ -52,9 +52,9 @@ def exist_object(object_type, correlation_id, type_id=None): # => work on object elif object_type == 'decoded': return Decoded.exist_decoded(correlation_id) elif object_type == 'pgp': - return Pgp.pgp._exist_corelation_field(type_id, correlation_id) + return Pgp.pgp.exist_correlation(type_id, correlation_id) elif object_type == 'cryptocurrency': - return Cryptocurrency.cryptocurrency._exist_corelation_field(type_id, correlation_id) + return Cryptocurrency.cryptocurrency.exist_correlation(type_id, correlation_id) elif object_type == 'screenshot' or object_type == 'image': return Screenshot.exist_screenshot(correlation_id) else: @@ -203,6 +203,20 @@ def get_obj_tag_table_keys(object_type): if object_type=="domain": return ['id', 'first_seen', 'last_check', 'status'] # # TODO: add root screenshot +def create_obj_relationship(obj1_type, obj1_id, obj2_type, obj2_id, obj1_subtype=None, obj2_subtype=None): + if obj1_type == 'domain': + pass + elif obj1_type == 'item': + pass + elif obj1_type == 'pgp': + Pgp.pgp.save_obj_relationship(obj1_subtype, obj1_id, obj2_type, obj2_id) + elif obj1_type == 'cryptocurrency': + Cryptocurrency.cryptocurrency.save_obj_relationship(obj1_subtype, obj1_type, obj2_type, obj2_id) + elif obj1_type == 'decoded': + pass + elif obj1_type == 'image': + pass + def create_graph_links(links_set): graph_links_list = [] diff --git a/bin/lib/Decoded.py b/bin/lib/Decoded.py index dbfcdb78..1d86426a 100755 --- a/bin/lib/Decoded.py +++ b/bin/lib/Decoded.py @@ -170,10 +170,26 @@ def get_decoded_correlated_object(sha1_string, correlation_objects=[]): decoded_correlation[correlation_object] = res return decoded_correlation +# # # TODO: check if item and decoded exist +def save_decoded_item_correlation(sha1_string, item_id, decoder_type): + item_date = Item.get_item_date(item_id) + + + # domain + if Item.is_crawled(item_id): + domain = Item.get_item_domain(item_id) + save_domain_decoded(domain, sha1_string) + pass + def save_domain_decoded(domain, sha1_string): r_serv_metadata.sadd('hash_domain:{}'.format(domain), sha1_string) # domain - hash map r_serv_metadata.sadd('domain_hash:{}'.format(sha1_string), domain) # hash - domain ma +def save_decoded_correlation(sha1_string, referenced_obj_type, referenced_obj_id): + if referenced_obj_type=='domain': + save_domain_decoded(referenced_obj_type, sha1_string) + elif referenced_obj_type=='item': + pass def get_decoded_file_content(sha1_string, mimetype=None): filepath = get_decoded_filepath(sha1_string, mimetype=mimetype) @@ -189,7 +205,7 @@ def save_decoded_file_content(sha1_string, io_content, date_range, mimetype=None else: mimetype = get_file_mimetype(io_content.getvalue()) - + filepath = get_decoded_filepath(sha1_string, mimetype=mimetype) if os.path.isfile(filepath): @@ -205,6 +221,7 @@ def save_decoded_file_content(sha1_string, io_content, date_range, mimetype=None f.write(io_content.getvalue()) # create hash metadata + # # TODO: save estimated type r_serv_metadata.hset('metadata_hash:{}'.format(sha1_string), 'size', os.path.getsize(filepath)) r_serv_metadata.hset('metadata_hash:{}'.format(sha1_string), 'first_seen', date_range['date_from']) diff --git a/bin/packages/Correlation.py b/bin/packages/Correlation.py index 26558ff0..5067df5b 100755 --- a/bin/packages/Correlation.py +++ b/bin/packages/Correlation.py @@ -10,6 +10,7 @@ import ConfigLoader sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/')) import Date +import Item #import Tag config_loader = ConfigLoader.ConfigLoader() @@ -36,7 +37,7 @@ class Correlation(object): def exist_correlation(self, subtype, obj_id): res = r_serv_metadata.zscore('{}_all:{}'.format(self.correlation_name, subtype), obj_id) - if res: + if res is not None: return True else: return False @@ -283,7 +284,7 @@ class Correlation(object): def get_correlation_all_object(self, correlation_type, correlation_value, correlation_objects=[]): - if correlation_objects is None: + if not correlation_objects: correlation_objects = get_all_correlation_objects() correlation_obj = {} for correlation_object in correlation_objects: @@ -311,8 +312,8 @@ class Correlation(object): if date > last_seen: r_serv_metadata.hset('{}_metadata_{}:{}'.format(self.correlation_name, subtype, obj_id), 'last_seen', date) - def save_item_correlation(self, subtype, date, obj_id, item_id, item_date): - update_correlation_daterange(subtype, obj_id, item_date) + def save_item_correlation(self, subtype, obj_id, item_id, item_date): + self.update_correlation_daterange(subtype, obj_id, item_date) # global set r_serv_metadata.sadd('set_{}_{}:{}'.format(self.correlation_name, subtype, obj_id), item_id) @@ -344,6 +345,12 @@ class Correlation(object): self.update_correlation_daterange(subtype, obj_id, date_range['date_to']) return True + def save_obj_relationship(self, subtype, obj_id, obj2_type, obj2_id): + if obj2_type == 'domain': + self.save_domain_correlation(obj2_id, subtype, obj_id) + elif obj2_type == 'item': + self.save_item_correlation(subtype, obj_id, obj2_id, Item.get_item_date(obj2_id)) + def create_correlation(self, subtype, obj_id, obj_meta): res = self.sanythise_correlation_types([subtype], r_boolean=True) if not res: @@ -359,8 +366,40 @@ class Correlation(object): #Tag.api_add_obj_tags(tags=obj_meta['tags'], object_id=obj_id, object_type=self.get_correlation_obj_type()) return True + # # TODO: handle tags def delete_correlation(self, subtype, obj_id): - pass + res = self.sanythise_correlation_types([subtype], r_boolean=True) + if not res: + print('invalid subtype') + return False + if not self.exist_correlation(subtype, obj_id): + return False + + obj_correlations = self.get_correlation_all_object(subtype, obj_id) + if 'domain' in obj_correlations: + for domain in obj_correlations['domain']: + r_serv_metadata.srem('domain_{}_{}:{}'.format(self.correlation_name, subtype, domain), obj_id) + r_serv_metadata.delete('set_domain_{}_{}:{}'.format(self.correlation_name, subtype, obj_id)) + + + if 'paste' in obj_correlations: # TODO: handle item + for item_id in obj_correlations['paste']: + + r_serv_metadata.srem('item_{}_{}:{}'.format(self.correlation_name, subtype, item_id), obj_id) + r_serv_metadata.delete('set_{}_{}:{}'.format(self.correlation_name, subtype, obj_id)) + + # delete daily correlation + first_seen = self.get_correlation_first_seen(subtype, obj_id) + last_seen = self.get_correlation_last_seen(subtype, obj_id) + meta_date = Date.sanitise_date_range(first_seen, last_seen) + date_range = Date.substract_date(meta_date['date_from'], meta_date['date_to']) + for date_day in date_range: + r_serv_metadata.hdel('{}:{}:{}'.format(self.correlation_name, subtype, date_day), obj_id) + + r_serv_metadata.delete('{}_metadata_{}:{}'.format(self.correlation_name, subtype, obj_id)) + r_serv_metadata.zrem('{}_all:{}'.format(self.correlation_name, subtype), obj_id) + + return True ######## API EXPOSED ########