mirror of https://github.com/CIRCL/AIL-framework
chg: [MISP import-export] pgp and cryptocurrency with relationships
Terrtia
parent
c042a2a66f
commit
ac114814fd
|
|
@ -20,6 +20,8 @@ from Helper import Process
|
|||
from packages import Paste
|
||||
from packages import Item
|
||||
|
||||
from lib import Decoded
|
||||
|
||||
import re
|
||||
import signal
|
||||
|
||||
|
|
@ -124,8 +126,7 @@ def save_hash(decoder_name, message, date, decoded):
|
|||
# Domain Object
|
||||
if Item.is_crawled(message):
|
||||
domain = Item.get_item_domain(message)
|
||||
serv_metadata.sadd('hash_domain:{}'.format(domain), hash) # domain - hash map
|
||||
serv_metadata.sadd('domain_hash:{}'.format(hash), domain) # hash - domain map
|
||||
Decoded.save_domain_decoded(domain, hash)
|
||||
|
||||
|
||||
def save_hash_on_disk(decode, type, hash, json_data):
|
||||
|
|
|
|||
|
|
@ -321,6 +321,7 @@ if __name__ == '__main__':
|
|||
{'id': 'bfd5f1d89e55b10a8b122a9d7ce31667ec1d086a', 'type': 'decoded', 'lvl': 2},
|
||||
#{'id': 'a92d459f70c4dea8a14688f585a5e2364be8b91fbf924290ead361d9b909dcf1', 'type': 'image', 'lvl': 3},
|
||||
{'id': 'archive/pastebin.com_pro/2020/01/27/iHjcWhkD.gz', 'type': 'item', 'lvl': 1},
|
||||
{'id': '0xA4BB02A75E6AF448', 'type': 'pgp', 'subtype': 'key', 'lvl': 1},
|
||||
{'id': '15efuhpw5V9B1opHAgNXKPBPqdYALXP4hc', 'type': 'cryptocurrency', 'subtype': 'bitcoin', 'lvl': 1}
|
||||
]
|
||||
create_list_of_objs_to_export(l_obj, mode='union')
|
||||
|
|
|
|||
|
|
@ -8,11 +8,11 @@ import redis
|
|||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages'))
|
||||
import Item
|
||||
import Cryptocurrency
|
||||
import Pgp
|
||||
import Decoded
|
||||
import Domain
|
||||
import Item
|
||||
import Screenshot
|
||||
import Correlate_object
|
||||
|
||||
|
|
@ -31,8 +31,18 @@ def get_global_id(obj_type, obj_id, obj_subtype=None):
|
|||
# sub type
|
||||
# obj type
|
||||
# obj value
|
||||
def get_global_id_from_misp_obj(misp_obj):
|
||||
pass
|
||||
def get_global_id_from_id(global_id):
|
||||
obj_meta = {}
|
||||
global_id = global_id.split(':', 3)
|
||||
if len(global_id) > 2:
|
||||
obj_meta['type'] = global_id[0]
|
||||
obj_meta['subtype'] = global_id[1]
|
||||
obj_meta['id'] = global_id[2]
|
||||
else:
|
||||
obj_meta['type'] = global_id[0]
|
||||
obj_meta['subtype'] = None
|
||||
obj_meta['id'] = global_id[1]
|
||||
return obj_meta
|
||||
|
||||
def get_misp_obj_tag(misp_obj):
|
||||
if misp_obj.attributes:
|
||||
|
|
@ -49,7 +59,7 @@ def get_object_metadata(misp_obj):
|
|||
if 'first_seen' in misp_obj.keys():
|
||||
obj_meta['first_seen'] = misp_obj.first_seen
|
||||
if 'last_seen' in misp_obj.keys():
|
||||
obj_meta['last_seen'] = misp_obj.first_seen
|
||||
obj_meta['last_seen'] = misp_obj.last_seen
|
||||
obj_meta['tags'] = get_misp_obj_tag(misp_obj)
|
||||
return obj_meta
|
||||
|
||||
|
|
@ -65,9 +75,9 @@ def unpack_item_obj(map_uuid_global_id, misp_obj):
|
|||
|
||||
if obj_id and io_content:
|
||||
res = Item.create_item(obj_id, obj_meta, io_content)
|
||||
print(res)
|
||||
#print(res)
|
||||
|
||||
map_uuid_global_id[misp_obj.uuid] = get_global_id('item', obj_id)
|
||||
map_uuid_global_id[misp_obj.uuid] = get_global_id('item', obj_id)
|
||||
|
||||
|
||||
|
||||
|
|
@ -85,12 +95,12 @@ def unpack_obj_pgp(map_uuid_global_id, misp_obj):
|
|||
|
||||
if obj_id and obj_subtype:
|
||||
obj_meta = get_object_metadata(misp_obj)
|
||||
print(obj_id)
|
||||
print(obj_meta)
|
||||
res = Pgp.pgp.create_correlation(obj_subtype, obj_id, obj_meta)
|
||||
print(res)
|
||||
|
||||
map_uuid_global_id[misp_obj.uuid] = get_global_id('pgp', obj_id, obj_subtype=obj_subtype)
|
||||
|
||||
#get_obj_relationship(misp_obj)
|
||||
|
||||
def unpack_obj_cryptocurrency(map_uuid_global_id, misp_obj):
|
||||
obj_id = None
|
||||
|
|
@ -103,19 +113,11 @@ def unpack_obj_cryptocurrency(map_uuid_global_id, misp_obj):
|
|||
|
||||
# valid cryptocurrency type
|
||||
if obj_subtype and obj_id:
|
||||
print('crypto')
|
||||
print(obj_id)
|
||||
print(obj_subtype)
|
||||
|
||||
obj_meta = get_object_metadata(misp_obj)
|
||||
print(obj_meta)
|
||||
res = Cryptocurrency.cryptocurrency.create_correlation(obj_subtype, obj_id, obj_meta)
|
||||
print(res)
|
||||
|
||||
map_uuid_global_id[misp_obj.uuid] = get_global_id('pgp', obj_id, obj_subtype=obj_subtype)
|
||||
|
||||
#get_obj_relationship(misp_obj)
|
||||
|
||||
def get_obj_type_from_relationship(misp_obj):
|
||||
obj_uuid = misp_obj.uuid
|
||||
obj_type = None
|
||||
|
|
@ -128,10 +130,6 @@ def get_obj_type_from_relationship(misp_obj):
|
|||
obj_type = 'decoded'
|
||||
return obj_type
|
||||
|
||||
def get_obj_relationship(misp_obj):
|
||||
for item in misp_obj.ObjectReference:
|
||||
print(item.to_json())
|
||||
|
||||
|
||||
# # TODO: covert md5 and sha1 to expected
|
||||
def unpack_file(map_uuid_global_id, misp_obj):
|
||||
|
|
@ -149,7 +147,8 @@ def unpack_file(map_uuid_global_id, misp_obj):
|
|||
|
||||
# # TODO: use/verify specified mimetype
|
||||
elif attribute.object_relation == 'mimetype':
|
||||
print(attribute.value)
|
||||
#print(attribute.value)
|
||||
pass
|
||||
|
||||
# # TODO: support more
|
||||
elif attribute.object_relation == 'sha1' and obj_type == 'decoded':
|
||||
|
|
@ -158,7 +157,6 @@ def unpack_file(map_uuid_global_id, misp_obj):
|
|||
obj_id = attribute.value
|
||||
|
||||
if obj_id and io_content:
|
||||
print(obj_type)
|
||||
obj_meta = get_object_metadata(misp_obj)
|
||||
if obj_type == 'screenshot':
|
||||
#Screenshot.create_screenshot(obj_id, obj_meta, io_content)
|
||||
|
|
@ -166,34 +164,35 @@ def unpack_file(map_uuid_global_id, misp_obj):
|
|||
else: #decoded
|
||||
Decoded.create_decoded(obj_id, obj_meta, io_content)
|
||||
|
||||
map_uuid_global_id[misp_obj.uuid] = get_global_id('item', obj_id)
|
||||
|
||||
def get_misp_import_fct(map_uuid_global_id, misp_obj):
|
||||
#print(misp_obj.ObjectReference)
|
||||
#for item in misp_obj.ObjectReference:
|
||||
# print(item.to_json())
|
||||
#obj_meta = get_object_metadata(misp_obj)
|
||||
|
||||
#print(misp_obj.name)
|
||||
|
||||
if misp_obj.name == 'ail-leak':
|
||||
#unpack_item_obj(map_uuid_global_id, misp_obj)
|
||||
#print(misp_obj.to_json())
|
||||
unpack_item_obj(map_uuid_global_id, misp_obj)
|
||||
pass
|
||||
elif misp_obj.name == 'domain-ip':
|
||||
pass
|
||||
elif misp_obj.name == 'pgp-meta':
|
||||
#unpack_obj_pgp(map_uuid_global_id, misp_obj)
|
||||
unpack_obj_pgp(map_uuid_global_id, misp_obj)
|
||||
pass
|
||||
elif misp_obj.name == 'coin-address':
|
||||
#unpack_obj_cryptocurrency(map_uuid_global_id, misp_obj)
|
||||
unpack_obj_cryptocurrency(map_uuid_global_id, misp_obj)
|
||||
pass
|
||||
elif misp_obj.name == 'file':
|
||||
unpack_file(map_uuid_global_id, misp_obj)
|
||||
print()
|
||||
print('---')
|
||||
print()
|
||||
#unpack_item_obj(map_uuid_global_id, misp_obj)
|
||||
pass
|
||||
|
||||
# import relationship between objects
|
||||
def create_obj_relationships(map_uuid_global_id, misp_obj):
|
||||
if misp_obj.uuid in map_uuid_global_id:
|
||||
for relationship in misp_obj.ObjectReference:
|
||||
if relationship.referenced_uuid in map_uuid_global_id:
|
||||
obj_meta_src = get_global_id_from_id(map_uuid_global_id[relationship.object_uuid])
|
||||
obj_meta_target = get_global_id_from_id(map_uuid_global_id[relationship.referenced_uuid])
|
||||
Correlate_object.create_obj_relationship(obj_meta_src['type'], obj_meta_src['id'], obj_meta_target['type'], obj_meta_target['id'],
|
||||
obj1_subtype=obj_meta_src['subtype'], obj2_subtype=obj_meta_target['subtype'])
|
||||
|
||||
|
||||
def import_objs_from_file(filepath):
|
||||
event_to_import = MISPEvent()
|
||||
event_to_import.load_file(filepath)
|
||||
|
|
@ -203,12 +202,17 @@ def import_objs_from_file(filepath):
|
|||
for misp_obj in event_to_import.objects:
|
||||
get_misp_import_fct(map_uuid_global_id, misp_obj)
|
||||
|
||||
print(map_uuid_global_id)
|
||||
for misp_obj in event_to_import.objects:
|
||||
create_obj_relationships(map_uuid_global_id, misp_obj)
|
||||
|
||||
#print(map_uuid_global_id)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
# misp = PyMISP('https://127.0.0.1:8443/', 'uXgcN42b7xuL88XqK5hubwD8Q8596VrrBvkHQzB0', False)
|
||||
|
||||
#import_objs_from_file('test_import_item.json')
|
||||
import_objs_from_file('test_import_item.json')
|
||||
|
||||
#Decoded.delete_correlation('23a44cc266880d26386a0a77318afbe09696f935')
|
||||
#Pgp.pgp.delete_correlation('key', '0xA4BB02A75E6AF448')
|
||||
|
|
|
|||
|
|
@ -52,9 +52,9 @@ def exist_object(object_type, correlation_id, type_id=None): # => work on object
|
|||
elif object_type == 'decoded':
|
||||
return Decoded.exist_decoded(correlation_id)
|
||||
elif object_type == 'pgp':
|
||||
return Pgp.pgp._exist_corelation_field(type_id, correlation_id)
|
||||
return Pgp.pgp.exist_correlation(type_id, correlation_id)
|
||||
elif object_type == 'cryptocurrency':
|
||||
return Cryptocurrency.cryptocurrency._exist_corelation_field(type_id, correlation_id)
|
||||
return Cryptocurrency.cryptocurrency.exist_correlation(type_id, correlation_id)
|
||||
elif object_type == 'screenshot' or object_type == 'image':
|
||||
return Screenshot.exist_screenshot(correlation_id)
|
||||
else:
|
||||
|
|
@ -203,6 +203,20 @@ def get_obj_tag_table_keys(object_type):
|
|||
if object_type=="domain":
|
||||
return ['id', 'first_seen', 'last_check', 'status'] # # TODO: add root screenshot
|
||||
|
||||
def create_obj_relationship(obj1_type, obj1_id, obj2_type, obj2_id, obj1_subtype=None, obj2_subtype=None):
|
||||
if obj1_type == 'domain':
|
||||
pass
|
||||
elif obj1_type == 'item':
|
||||
pass
|
||||
elif obj1_type == 'pgp':
|
||||
Pgp.pgp.save_obj_relationship(obj1_subtype, obj1_id, obj2_type, obj2_id)
|
||||
elif obj1_type == 'cryptocurrency':
|
||||
Cryptocurrency.cryptocurrency.save_obj_relationship(obj1_subtype, obj1_type, obj2_type, obj2_id)
|
||||
elif obj1_type == 'decoded':
|
||||
pass
|
||||
elif obj1_type == 'image':
|
||||
pass
|
||||
|
||||
|
||||
def create_graph_links(links_set):
|
||||
graph_links_list = []
|
||||
|
|
|
|||
|
|
@ -170,10 +170,26 @@ def get_decoded_correlated_object(sha1_string, correlation_objects=[]):
|
|||
decoded_correlation[correlation_object] = res
|
||||
return decoded_correlation
|
||||
|
||||
# # # TODO: check if item and decoded exist
|
||||
def save_decoded_item_correlation(sha1_string, item_id, decoder_type):
|
||||
item_date = Item.get_item_date(item_id)
|
||||
|
||||
|
||||
# domain
|
||||
if Item.is_crawled(item_id):
|
||||
domain = Item.get_item_domain(item_id)
|
||||
save_domain_decoded(domain, sha1_string)
|
||||
pass
|
||||
|
||||
def save_domain_decoded(domain, sha1_string):
|
||||
r_serv_metadata.sadd('hash_domain:{}'.format(domain), sha1_string) # domain - hash map
|
||||
r_serv_metadata.sadd('domain_hash:{}'.format(sha1_string), domain) # hash - domain ma
|
||||
|
||||
def save_decoded_correlation(sha1_string, referenced_obj_type, referenced_obj_id):
|
||||
if referenced_obj_type=='domain':
|
||||
save_domain_decoded(referenced_obj_type, sha1_string)
|
||||
elif referenced_obj_type=='item':
|
||||
pass
|
||||
|
||||
def get_decoded_file_content(sha1_string, mimetype=None):
|
||||
filepath = get_decoded_filepath(sha1_string, mimetype=mimetype)
|
||||
|
|
@ -189,7 +205,7 @@ def save_decoded_file_content(sha1_string, io_content, date_range, mimetype=None
|
|||
else:
|
||||
mimetype = get_file_mimetype(io_content.getvalue())
|
||||
|
||||
|
||||
|
||||
|
||||
filepath = get_decoded_filepath(sha1_string, mimetype=mimetype)
|
||||
if os.path.isfile(filepath):
|
||||
|
|
@ -205,6 +221,7 @@ def save_decoded_file_content(sha1_string, io_content, date_range, mimetype=None
|
|||
f.write(io_content.getvalue())
|
||||
|
||||
# create hash metadata
|
||||
# # TODO: save estimated type
|
||||
r_serv_metadata.hset('metadata_hash:{}'.format(sha1_string), 'size', os.path.getsize(filepath))
|
||||
|
||||
r_serv_metadata.hset('metadata_hash:{}'.format(sha1_string), 'first_seen', date_range['date_from'])
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ import ConfigLoader
|
|||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
|
||||
import Date
|
||||
import Item
|
||||
#import Tag
|
||||
|
||||
config_loader = ConfigLoader.ConfigLoader()
|
||||
|
|
@ -36,7 +37,7 @@ class Correlation(object):
|
|||
|
||||
def exist_correlation(self, subtype, obj_id):
|
||||
res = r_serv_metadata.zscore('{}_all:{}'.format(self.correlation_name, subtype), obj_id)
|
||||
if res:
|
||||
if res is not None:
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
|
|
@ -283,7 +284,7 @@ class Correlation(object):
|
|||
|
||||
|
||||
def get_correlation_all_object(self, correlation_type, correlation_value, correlation_objects=[]):
|
||||
if correlation_objects is None:
|
||||
if not correlation_objects:
|
||||
correlation_objects = get_all_correlation_objects()
|
||||
correlation_obj = {}
|
||||
for correlation_object in correlation_objects:
|
||||
|
|
@ -311,8 +312,8 @@ class Correlation(object):
|
|||
if date > last_seen:
|
||||
r_serv_metadata.hset('{}_metadata_{}:{}'.format(self.correlation_name, subtype, obj_id), 'last_seen', date)
|
||||
|
||||
def save_item_correlation(self, subtype, date, obj_id, item_id, item_date):
|
||||
update_correlation_daterange(subtype, obj_id, item_date)
|
||||
def save_item_correlation(self, subtype, obj_id, item_id, item_date):
|
||||
self.update_correlation_daterange(subtype, obj_id, item_date)
|
||||
|
||||
# global set
|
||||
r_serv_metadata.sadd('set_{}_{}:{}'.format(self.correlation_name, subtype, obj_id), item_id)
|
||||
|
|
@ -344,6 +345,12 @@ class Correlation(object):
|
|||
self.update_correlation_daterange(subtype, obj_id, date_range['date_to'])
|
||||
return True
|
||||
|
||||
def save_obj_relationship(self, subtype, obj_id, obj2_type, obj2_id):
|
||||
if obj2_type == 'domain':
|
||||
self.save_domain_correlation(obj2_id, subtype, obj_id)
|
||||
elif obj2_type == 'item':
|
||||
self.save_item_correlation(subtype, obj_id, obj2_id, Item.get_item_date(obj2_id))
|
||||
|
||||
def create_correlation(self, subtype, obj_id, obj_meta):
|
||||
res = self.sanythise_correlation_types([subtype], r_boolean=True)
|
||||
if not res:
|
||||
|
|
@ -359,8 +366,40 @@ class Correlation(object):
|
|||
#Tag.api_add_obj_tags(tags=obj_meta['tags'], object_id=obj_id, object_type=self.get_correlation_obj_type())
|
||||
return True
|
||||
|
||||
# # TODO: handle tags
|
||||
def delete_correlation(self, subtype, obj_id):
|
||||
pass
|
||||
res = self.sanythise_correlation_types([subtype], r_boolean=True)
|
||||
if not res:
|
||||
print('invalid subtype')
|
||||
return False
|
||||
if not self.exist_correlation(subtype, obj_id):
|
||||
return False
|
||||
|
||||
obj_correlations = self.get_correlation_all_object(subtype, obj_id)
|
||||
if 'domain' in obj_correlations:
|
||||
for domain in obj_correlations['domain']:
|
||||
r_serv_metadata.srem('domain_{}_{}:{}'.format(self.correlation_name, subtype, domain), obj_id)
|
||||
r_serv_metadata.delete('set_domain_{}_{}:{}'.format(self.correlation_name, subtype, obj_id))
|
||||
|
||||
|
||||
if 'paste' in obj_correlations: # TODO: handle item
|
||||
for item_id in obj_correlations['paste']:
|
||||
|
||||
r_serv_metadata.srem('item_{}_{}:{}'.format(self.correlation_name, subtype, item_id), obj_id)
|
||||
r_serv_metadata.delete('set_{}_{}:{}'.format(self.correlation_name, subtype, obj_id))
|
||||
|
||||
# delete daily correlation
|
||||
first_seen = self.get_correlation_first_seen(subtype, obj_id)
|
||||
last_seen = self.get_correlation_last_seen(subtype, obj_id)
|
||||
meta_date = Date.sanitise_date_range(first_seen, last_seen)
|
||||
date_range = Date.substract_date(meta_date['date_from'], meta_date['date_to'])
|
||||
for date_day in date_range:
|
||||
r_serv_metadata.hdel('{}:{}:{}'.format(self.correlation_name, subtype, date_day), obj_id)
|
||||
|
||||
r_serv_metadata.delete('{}_metadata_{}:{}'.format(self.correlation_name, subtype, obj_id))
|
||||
r_serv_metadata.zrem('{}_all:{}'.format(self.correlation_name, subtype), obj_id)
|
||||
|
||||
return True
|
||||
|
||||
######## API EXPOSED ########
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue