CIRCL
/
AIL-framework
mirror of https://github.com/CIRCL/AIL-framework
2
1
Fork 0
Code Issues Releases Wiki Activity

chg: [MISP export] add pgp object + add relationship: item + pgp + cryptocurrency

pull/486/head
Terrtia 2020-02-04 13:04:09 +01:00
parent 4a732ea9f3
commit b31fffa728
No known key found for this signature in database
GPG Key ID: 1E1B1F50D84613D0
7 changed files with 89 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

105
bin/export/MispExport.py
View File

@ -8,7 +8,8 @@ import redis
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages'))
#from Cryptocurrency import cryptocurrency
import Cryptocurrency
import Pgp
import Decoded
import Domain
import Item
@ -45,6 +46,7 @@ def export_ail_item(item_id):
def export_domain(domain):
domain_obj = Domain.Domain(domain)
dict_metadata = domain_obj.get_domain_metadata(tags=True)
dict_metadata['ports'] = ['80', '223', '443']
# create domain-ip obj
obj = MISPObject('domain-ip', standalone=True)
@ -55,6 +57,8 @@ def export_domain(domain):
l_obj_attr.append( obj.add_attribute('first-seen', value=dict_metadata['first_seen']) )
l_obj_attr.append( obj.add_attribute('last-seen', value=dict_metadata['last_check']) )
l_obj_attr.append( obj.add_attribute('domain', value=domain) )
for port in dict_metadata['ports']:
l_obj_attr.append( obj.add_attribute('port', value=port) )
# add tags
if dict_metadata['tags']:
@ -100,7 +104,7 @@ def export_screenshot(sha256_string):
# TODO: add tags
def export_cryptocurrency(crypto_type, crypto_address):
dict_metadata = cryptocurrency.get_metadata(crypto_address, crypto_type)
dict_metadata = Cryptocurrency.cryptocurrency.get_metadata(crypto_type, crypto_address)
obj = MISPObject('coin-address')
obj.first_seen = dict_metadata['first_seen']
@ -114,6 +118,26 @@ def export_cryptocurrency(crypto_type, crypto_address):
return obj
# TODO: add tags
def export_pgp(pgp_type, pgp_value):
dict_metadata = Pgp.pgp.get_metadata(pgp_type, pgp_value)
obj = MISPObject('pgp-meta', misp_objects_path_custom='../../../misp-objects/objects')
obj.first_seen = dict_metadata['first_seen']
obj.last_seen = dict_metadata['last_seen']
l_obj_attr = []
if pgp_type=='key':
l_obj_attr.append( obj.add_attribute('key-id', value=pgp_value) )
elif pgp_type=='name':
#l_obj_attr.append( obj.add_attribute('key-id', value='debug') )
l_obj_attr.append( obj.add_attribute('user-id-name', value=pgp_value) )
else: # mail
#l_obj_attr.append( obj.add_attribute('key-id', value='debug') )
l_obj_attr.append( obj.add_attribute('user-id-email', value=pgp_value) )
return obj
# filter objects to export, export only object who correlect which each other
def filter_obj_linked(l_obj):
for obj in l_obj:
@ -149,7 +173,7 @@ def add_obj_to_create_by_lvl(all_obj_to_export, set_relationship, dict_obj, lvl)
lvl = lvl - 1
# # TODO: filter by correlation types
obj_correlations = Correlate_object.get_object_correlation(dict_obj['type'], dict_obj['id'], dict_obj.get('subtype', None))
obj_correlations = Correlate_object.get_object_correlation(dict_obj['type'], dict_obj['id'], requested_correl_type=dict_obj.get('subtype', None))
for obj_type in obj_correlations:
dict_new_obj = {'type': obj_type}
if obj_type=='pgp' or obj_type=='cryptocurrency':
@ -195,10 +219,10 @@ def create_list_of_objs_to_export(l_obj, mode='union'):
# add object to event
event.add_object(dict_misp_obj[obj_global_id])
#print(event.to_json())
print(event.to_json())
misp = PyMISP('https://127.0.0.1:8443/', 'uXgcN42b7xuL88XqK5hubwD8Q8596VrrBvkHQzB0', False)
misp.add_event(event, pythonify=True)
#misp = PyMISP('https://127.0.0.1:8443/', 'uXgcN42b7xuL88XqK5hubwD8Q8596VrrBvkHQzB0', False)
#misp.add_event(event, pythonify=True)
def create_all_misp_obj(all_obj_to_export, set_relationship):
@ -221,7 +245,7 @@ def create_misp_obj(obj_type, obj_id):
return export_cryptocurrency(obj_subtype, obj_id)
elif obj_type == 'pgp':
obj_subtype, obj_id = obj_id.split(':', 1)
pass
return export_pgp(obj_subtype, obj_id)
elif obj_type == 'domain':
return export_domain(obj_id)
@ -246,12 +270,41 @@ def get_relationship_between_global_obj(obj_global_id_1, obj_global_id_2):
src = obj_global_id_2
dest = obj_global_id_1
return {'relation': 'included-in', 'src': src, 'dest': dest}
elif 'pgp':
return None
elif 'pgp' in type_tuple:
if obj_type_1 == 'pgp':
src = obj_global_id_1
dest = obj_global_id_2
else:
src = obj_global_id_2
dest = obj_global_id_1
return {'relation': 'extracted-from', 'src': src, 'dest': dest}
elif 'cryptocurrency':
return None
elif 'domain':
return None
if obj_type_1 == 'cryptocurrency':
src = obj_global_id_1
dest = obj_global_id_2
else:
src = obj_global_id_2
dest = obj_global_id_1
return {'relation': 'extracted-from', 'src': src, 'dest': dest}
elif 'domain' in type_tuple:
if 'item' in type_tuple:
if obj_type_1 == 'item':
src = obj_global_id_1
dest = obj_global_id_2
else:
src = obj_global_id_2
dest = obj_global_id_1
return {'relation': 'extracted-from', 'src': src, 'dest': dest} # replave by crawled-from
elif 'item' in type_tuple:
if 'domain' in type_tuple:
if obj_type_1 == 'item':
src = obj_global_id_1
dest = obj_global_id_2
else:
src = obj_global_id_2
dest = obj_global_id_1
return {'relation': 'extracted-from', 'src': src, 'dest': dest} # replave by crawled-from
return None
######
#
@ -263,31 +316,11 @@ def get_relationship_between_global_obj(obj_global_id_1, obj_global_id_2):
if __name__ == '__main__':
l_obj = [{'id': 'crawled/2019/11/08/6d3zimnpbwbzdgnp.onionf58258c8-c990-4707-b236-762a2b881183', 'type': 'item', 'lvl': 3},
{'id': '6d3zimnpbwbzdgnp.onion', 'type': 'domain', 'lvl': 0},
#{'id': '0xA4BB02A75E6AF448', 'type': 'pgp', 'subtype': 'key', 'lvl': 0},
{'id': 'a92d459f70c4dea8a14688f585a5e2364be8b91fbf924290ead361d9b909dcf1', 'type': 'image', 'lvl': 3}]
l_obj = [#{'id': 'crawled/2019/11/08/6d3zimnpbwbzdgnp.onionf58258c8-c990-4707-b236-762a2b881183', 'type': 'item', 'lvl': 3},
#{'id': '6d3zimnpbwbzdgnp.onion', 'type': 'domain', 'lvl': 0},
#{'id': 'a92d459f70c4dea8a14688f585a5e2364be8b91fbf924290ead361d9b909dcf1', 'type': 'image', 'lvl': 3},
{'id': '15efuhpw5V9B1opHAgNXKPBPqdYALXP4hc', 'type': 'cryptocurrency', 'subtype': 'bitcoin', 'lvl': 1}]
create_list_of_objs_to_export(l_obj, mode='union')
#event = MISPEvent()
#event.info = 'AIL framework export'
#
# obj_item = export_ail_item('crawled/2019/11/08/6d3zimnpbwbzdgnp.onionf58258c8-c990-4707-b236-762a2b881183')
# event.add_object(obj_item)
#
# obj_domain = export_domain('2222222222xkrmay.onion')
# event.add_object(obj_domain)
#
# obj_decoded = export_decoded('fc351baadefce6f702155fb908a9e84dd5dd0fa7')
# obj_decoded.add_reference(obj_domain.uuid, 'injected-into', 'add a comment')
# event.add_object(obj_decoded)
# obj_screenshot = export_screenshot('5fcc292ea8a699aa7a9ce93a704b78b8f493620ccdb2a5cebacb1069a4327211')
# obj_screenshot.add_reference(obj_domain.uuid, 'screenshot-of')
# event.add_object(obj_screenshot)
#print(event.to_json())

5
bin/lib/Decoded.py
View File

@ -12,6 +12,9 @@ import Item
import Date
import Tag
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
import correlation
import ConfigLoader
@ -144,7 +147,7 @@ def get_decoded_correlated_object(sha1_string, correlation_objects=[]):
:rtype: dict
'''
if correlation_objects is None:
correlation_objects = Correlation.get_all_correlation_objects()
correlation_objects = correlation.get_all_correlation_objects()
decoded_correlation = {}
for correlation_object in correlation_objects:
if correlation_object == 'paste':

8
bin/lib/Domain.py
View File

@ -15,14 +15,12 @@ import random
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
import Cryptocurrency
from Pgp import pgp
import Pgp
import Date
import Decoded
import Item
import Tag
cryptocurrency = Cryptocurrency.cryptocurrency
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
import ConfigLoader
import Correlate_object
@ -523,7 +521,7 @@ def get_domain_cryptocurrency(domain, currencies_type=None, get_nb=False):
:param currencies_type: list of cryptocurrencies type
:type currencies_type: list, optional
'''
return cryptocurrency.get_domain_correlation_dict(domain, correlation_type=currencies_type, get_nb=get_nb)
return Cryptocurrency.cryptocurrency.get_domain_correlation_dict(domain, correlation_type=currencies_type, get_nb=get_nb)
def get_domain_pgp(domain, currencies_type=None, get_nb=False):
'''
@ -533,7 +531,7 @@ def get_domain_pgp(domain, currencies_type=None, get_nb=False):
:param currencies_type: list of pgp type
:type currencies_type: list, optional
'''
return pgp.get_domain_correlation_dict(domain, correlation_type=currencies_type, get_nb=get_nb)
return Pgp.pgp.get_domain_correlation_dict(domain, correlation_type=currencies_type, get_nb=get_nb)
def get_domain_decoded(domain):
'''

9
bin/packages/Cryptocurrency.py
View File

@ -8,7 +8,7 @@ import redis
from hashlib import sha256
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages'))
from Correlation import Correlation
import correlation
import Item
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
@ -20,12 +20,7 @@ config_loader = None
digits58 = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz'
class Cryptocurrency(Correlation):
def __init__(self):
super().__init__('cryptocurrency', ['bitcoin', 'ethereum', 'bitcoin-cash', 'litecoin', 'monero', 'zcash', 'dash'])
cryptocurrency = Cryptocurrency()
cryptocurrency = correlation.Correlation('cryptocurrency', ['bitcoin', 'ethereum', 'bitcoin-cash', 'litecoin', 'monero', 'zcash', 'dash'])
# http://rosettacode.org/wiki/Bitcoin/address_validation#Python
def decode_base58(bc, length):

5
bin/packages/Item.py
View File

@ -11,9 +11,8 @@ from io import BytesIO
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
import Date
import Tag
import Correlation
import Cryptocurrency
from Pgp import pgp
import Pgp
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
import ConfigLoader
@ -166,7 +165,7 @@ def get_item_pgp(item_id, currencies_type=None, get_nb=False):
:param currencies_type: list of cryptocurrencies type
:type currencies_type: list, optional
'''
return pgp.get_item_correlation_dict(item_id, correlation_type=currencies_type, get_nb=get_nb)
return Pgp.pgp.get_item_correlation_dict(item_id, correlation_type=currencies_type, get_nb=get_nb)
def get_item_decoded(item_id):
'''

9
bin/packages/Pgp.py
View File

@ -6,7 +6,7 @@ import sys
import redis
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages'))
from Correlation import Correlation
import correlation
import Item
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
@ -16,12 +16,7 @@ config_loader = ConfigLoader.ConfigLoader()
serv_metadata = config_loader.get_redis_conn("ARDB_Metadata")
config_loader = None
class Pgp(Correlation):
def __init__(self):
super().__init__('pgpdump', ['key', 'mail', 'name'])
pgp = Pgp()
pgp = correlation.Correlation('pgpdump', ['key', 'mail', 'name'])
def get_pgp(request_dict, pgp_type):
# basic verification

9
bin/packages/Correlation.py → bin/packages/correlation.py
View File

@ -7,7 +7,6 @@ import redis
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
import ConfigLoader
import Correlate_object
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
import Date
@ -16,6 +15,12 @@ config_loader = ConfigLoader.ConfigLoader()
r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata")
config_loader = None
def get_all_correlation_objects():
'''
Return a list of all correllated objects
'''
return ['domain', 'paste']
class Correlation(object):
def __init__(self, correlation_name, all_correlation_types):
@ -236,7 +241,7 @@ class Correlation(object):
def get_correlation_all_object(self, correlation_type, correlation_value, correlation_objects=[]):
if correlation_objects is None:
correlation_objects = Correlate_object.get_all_correlation_objects()
correlation_objects = get_all_correlation_objects()
correlation_obj = {}
for correlation_object in correlation_objects:
if correlation_object == 'paste':