mirror of https://github.com/CIRCL/AIL-framework
fix: [MISP auto export] fix MISP_TheHive_feeder
Terrtia
parent
b50cc92573
commit
762b517150
|
|
@ -20,8 +20,12 @@ from Helper import Process
|
|||
from packages import Paste
|
||||
import ailleakObject
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages'))
|
||||
import Tag
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
|
||||
import ConfigLoader
|
||||
import item_basic
|
||||
|
||||
from pymisp import PyMISP
|
||||
|
||||
|
|
@ -94,26 +98,22 @@ def create_the_hive_alert(source, path, tag):
|
|||
def feeder(message, count=0):
|
||||
|
||||
if flag_the_hive or flag_misp:
|
||||
tag, path = message.split(';')
|
||||
tag, item_id = message.split(';')
|
||||
|
||||
## FIXME: remove it
|
||||
if PASTES_FOLDER not in path:
|
||||
path = os.path.join(PASTES_FOLDER, path)
|
||||
try:
|
||||
paste = Paste.Paste(path)
|
||||
except FileNotFoundError:
|
||||
if not item_basic.exist_item(item_id):
|
||||
if count < 10:
|
||||
r_serv_db.zincrby('mess_not_saved_export', message, 1)
|
||||
return 0
|
||||
else:
|
||||
r_serv_db.zrem('mess_not_saved_export', message)
|
||||
print('Error: {} do not exist, tag= {}'.format(path, tag))
|
||||
print('Error: {} do not exist, tag= {}'.format(item_id, tag))
|
||||
return 0
|
||||
|
||||
source = '/'.join(paste.p_path.split('/')[-6:])
|
||||
source = item_basic.get_source(item_id)
|
||||
|
||||
if HiveApi != False:
|
||||
if int(r_serv_db.get('hive:auto-alerts')) == 1:
|
||||
whitelist_hive = r_serv_db.scard('whitelist_hive')
|
||||
if r_serv_db.sismember('whitelist_hive', tag):
|
||||
create_the_hive_alert(source, path, tag)
|
||||
else:
|
||||
|
|
@ -121,7 +121,7 @@ def feeder(message, count=0):
|
|||
if flag_misp:
|
||||
if int(r_serv_db.get('misp:auto-events')) == 1:
|
||||
if r_serv_db.sismember('whitelist_misp', tag):
|
||||
misp_wrapper.pushToMISP(uuid_ail, path, tag)
|
||||
misp_wrapper.pushToMISP(uuid_ail, item_id, tag)
|
||||
else:
|
||||
print('misp, auto events creation disable')
|
||||
|
||||
|
|
@ -161,15 +161,15 @@ if __name__ == "__main__":
|
|||
print('Not connected to MISP')
|
||||
|
||||
if flag_misp:
|
||||
try:
|
||||
misp_wrapper = ailleakObject.ObjectWrapper(pymisp)
|
||||
r_serv_db.set('ail:misp', True)
|
||||
print('Connected to MISP:', misp_url)
|
||||
except Exception as e:
|
||||
flag_misp = False
|
||||
r_serv_db.set('ail:misp', False)
|
||||
print(e)
|
||||
print('Not connected to MISP')
|
||||
#try:
|
||||
misp_wrapper = ailleakObject.ObjectWrapper(pymisp)
|
||||
r_serv_db.set('ail:misp', True)
|
||||
print('Connected to MISP:', misp_url)
|
||||
#except Exception as e:
|
||||
# flag_misp = False
|
||||
# r_serv_db.set('ail:misp', False)
|
||||
# print(e)
|
||||
# print('Not connected to MISP')
|
||||
|
||||
# create The HIVE connection
|
||||
if flag_the_hive:
|
||||
|
|
|
|||
|
|
@ -45,4 +45,5 @@ if __name__ == '__main__':
|
|||
tag, item_id = message.split(';')
|
||||
|
||||
Tag.add_tag("item", tag, item_id)
|
||||
|
||||
p.populate_set_out(message, 'MISP_The_Hive_feeder')
|
||||
|
|
|
|||
|
|
@ -4,7 +4,10 @@
|
|||
import os
|
||||
import sys
|
||||
|
||||
from pymisp import MISPEvent, MISPObject
|
||||
from pymisp.tools.abstractgenerator import AbstractMISPObjectGenerator
|
||||
MISPEvent
|
||||
|
||||
from packages import Paste
|
||||
import datetime
|
||||
import json
|
||||
|
|
@ -12,28 +15,10 @@ from io import BytesIO
|
|||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
||||
import ConfigLoader
|
||||
import item_basic
|
||||
|
||||
class AilLeakObject(AbstractMISPObjectGenerator):
|
||||
def __init__(self, uuid_ail, p_source, p_date, p_content, p_duplicate, p_duplicate_number):
|
||||
super(AbstractMISPObjectGenerator, self).__init__('ail-leak')
|
||||
self._uuid = uuid_ail
|
||||
self._p_source = p_source
|
||||
self._p_date = p_date
|
||||
self._p_content = p_content
|
||||
self._p_duplicate = p_duplicate
|
||||
self._p_duplicate_number = p_duplicate_number
|
||||
self.generate_attributes()
|
||||
|
||||
def generate_attributes(self):
|
||||
self.add_attribute('origin', value=self._p_source, type='text')
|
||||
self.add_attribute('last-seen', value=self._p_date, type='datetime')
|
||||
if self._p_duplicate_number > 0:
|
||||
self.add_attribute('duplicate', value=self._p_duplicate, type='text')
|
||||
self.add_attribute('duplicate_number', value=self._p_duplicate_number, type='counter')
|
||||
self._pseudofile = BytesIO(self._p_content.encode())
|
||||
res = self.add_attribute('raw-data', value=self._p_source, data=self._pseudofile, type="attachment")# , ShadowAttribute=self.p_tag)
|
||||
#res.add_shadow_attributes(tag)
|
||||
self.add_attribute('sensor', value=self._uuid, type="text")
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'export'))
|
||||
import MispExport
|
||||
|
||||
class ObjectWrapper:
|
||||
def __init__(self, pymisp):
|
||||
|
|
@ -45,53 +30,48 @@ class ObjectWrapper:
|
|||
config_loader = None
|
||||
self.attribute_to_tag = None
|
||||
|
||||
def add_new_object(self, uuid_ail, path, p_source, tag):
|
||||
def add_new_object(self, uuid_ail, item_id, tag):
|
||||
self.uuid_ail = uuid_ail
|
||||
self.path = path
|
||||
self.p_source = p_source
|
||||
self.paste = Paste.Paste(path)
|
||||
self.p_date = self.date_to_str(self.paste.p_date)
|
||||
self.p_content = self.paste.get_p_content()
|
||||
self.p_tag = tag
|
||||
|
||||
temp = self.paste._get_p_duplicate()
|
||||
# self.paste = Paste.Paste(path)
|
||||
# temp = self.paste._get_p_duplicate()
|
||||
#
|
||||
# #beautifier
|
||||
# if not temp:
|
||||
# temp = ''
|
||||
#
|
||||
# p_duplicate_number = len(temp) if len(temp) >= 0 else 0
|
||||
#
|
||||
# to_ret = ""
|
||||
# for dup in temp[:10]:
|
||||
# dup = dup.replace('\'','\"').replace('(','[').replace(')',']')
|
||||
# dup = json.loads(dup)
|
||||
# algo = dup[0]
|
||||
# path = dup[1].split('/')[-6:]
|
||||
# path = '/'.join(path)[:-3] # -3 removes .gz
|
||||
# if algo == 'tlsh':
|
||||
# perc = 100 - int(dup[2])
|
||||
# else:
|
||||
# perc = dup[2]
|
||||
# to_ret += "{}: {} [{}%]\n".format(path, algo, perc)
|
||||
# p_duplicate = to_ret
|
||||
|
||||
#beautifier
|
||||
if not temp:
|
||||
temp = ''
|
||||
|
||||
p_duplicate_number = len(temp) if len(temp) >= 0 else 0
|
||||
|
||||
to_ret = ""
|
||||
for dup in temp[:10]:
|
||||
dup = dup.replace('\'','\"').replace('(','[').replace(')',']')
|
||||
dup = json.loads(dup)
|
||||
algo = dup[0]
|
||||
path = dup[1].split('/')[-6:]
|
||||
path = '/'.join(path)[:-3] # -3 removes .gz
|
||||
if algo == 'tlsh':
|
||||
perc = 100 - int(dup[2])
|
||||
else:
|
||||
perc = dup[2]
|
||||
to_ret += "{}: {} [{}%]\n".format(path, algo, perc)
|
||||
p_duplicate = to_ret
|
||||
|
||||
self.mispObject = AilLeakObject(self.uuid_ail, self.p_source, self.p_date, self.p_content, p_duplicate, p_duplicate_number)
|
||||
return MispExport.export_ail_item(item_id, [tag])
|
||||
|
||||
def date_to_str(self, date):
|
||||
return "{0}-{1}-{2}".format(date.year, date.month, date.day)
|
||||
|
||||
def get_all_related_events(self):
|
||||
to_search = "Daily AIL-leaks"
|
||||
result = self.pymisp.search_all(to_search)
|
||||
def get_all_related_events(self, to_search):
|
||||
result = self.pymisp.search(controller='events', eventinfo=to_search, metadata=False)
|
||||
events = []
|
||||
for e in result['response']:
|
||||
events.append({'id': e['Event']['id'], 'org_id': e['Event']['org_id'], 'info': e['Event']['info']})
|
||||
if result:
|
||||
for e in result:
|
||||
events.append({'id': e['Event']['id'], 'org_id': e['Event']['org_id'], 'info': e['Event']['info']})
|
||||
return events
|
||||
|
||||
def get_daily_event_id(self):
|
||||
to_match = "Daily AIL-leaks {}".format(datetime.date.today())
|
||||
events = self.get_all_related_events()
|
||||
events = self.get_all_related_events(to_match)
|
||||
for dic in events:
|
||||
info = dic['info']
|
||||
e_id = dic['id']
|
||||
|
|
@ -99,8 +79,8 @@ class ObjectWrapper:
|
|||
print('Found: ', info, '->', e_id)
|
||||
self.currentID_date = datetime.date.today()
|
||||
return e_id
|
||||
created_event = self.create_daily_event()['Event']
|
||||
new_id = created_event['id']
|
||||
created_event = self.create_daily_event()
|
||||
new_id = created_event['Event']['id']
|
||||
print('New event created:', new_id)
|
||||
self.currentID_date = datetime.date.today()
|
||||
return new_id
|
||||
|
|
@ -120,17 +100,20 @@ class ObjectWrapper:
|
|||
orgc_id = None
|
||||
sharing_group_id = None
|
||||
date = None
|
||||
event = self.pymisp.new_event(distribution, threat,
|
||||
analysis, info, date,
|
||||
published, orgc_id, org_id, sharing_group_id)
|
||||
eventUuid = event['Event']['uuid']
|
||||
self.pymisp.tag(eventUuid, 'infoleak:output-format="ail-daily"')
|
||||
return event
|
||||
|
||||
event = MISPEvent()
|
||||
event.distribution = distribution
|
||||
event.info = info
|
||||
event.analysis = analysis
|
||||
event.threat = threat
|
||||
event.published = published
|
||||
|
||||
event.add_tag('infoleak:output-format="ail-daily"')
|
||||
existing_event = self.pymisp.add_event(event)
|
||||
return existing_event
|
||||
|
||||
# Publish object to MISP
|
||||
def pushToMISP(self, uuid_ail, path, tag):
|
||||
self._p_source = path.split('/')[-5:]
|
||||
self._p_source = '/'.join(self._p_source)[:-3]
|
||||
def pushToMISP(self, uuid_ail, item_id, tag):
|
||||
|
||||
if self.currentID_date != datetime.date.today(): #refresh id
|
||||
self.eventID_to_push = self.get_daily_event_id()
|
||||
|
|
@ -138,42 +121,37 @@ class ObjectWrapper:
|
|||
mispTYPE = 'ail-leak'
|
||||
|
||||
# paste object already exist
|
||||
if self.paste_object_exist(self.eventID_to_push, self._p_source):
|
||||
if self.paste_object_exist(self.eventID_to_push, item_id):
|
||||
# add new tag
|
||||
self.tag(self.attribute_to_tag, tag)
|
||||
print(self._p_source + ' tagged: ' + tag)
|
||||
print(item_id + ' tagged: ' + tag)
|
||||
#create object
|
||||
else:
|
||||
self.add_new_object(uuid_ail, path, self._p_source, tag)
|
||||
misp_obj = self.add_new_object(uuid_ail, item_id, tag)
|
||||
|
||||
# deprecated
|
||||
# try:
|
||||
# templateID = [x['ObjectTemplate']['id'] for x in self.pymisp.get_object_templates_list() if x['ObjectTemplate']['name'] == mispTYPE][0]
|
||||
# except IndexError:
|
||||
# valid_types = ", ".join([x['ObjectTemplate']['name'] for x in self.pymisp.get_object_templates_list()])
|
||||
# print ("Template for type %s not found! Valid types are: %s" % (mispTYPE, valid_types))
|
||||
|
||||
|
||||
try:
|
||||
templateID = [x['ObjectTemplate']['id'] for x in self.pymisp.get_object_templates_list() if x['ObjectTemplate']['name'] == mispTYPE][0]
|
||||
except IndexError:
|
||||
valid_types = ", ".join([x['ObjectTemplate']['name'] for x in self.pymisp.get_object_templates_list()])
|
||||
print ("Template for type %s not found! Valid types are: %s" % (mispTYPE, valid_types))
|
||||
r = self.pymisp.add_object(self.eventID_to_push, templateID, self.mispObject)
|
||||
r = self.pymisp.add_object(self.eventID_to_push, misp_obj, pythonify=True)
|
||||
if 'errors' in r:
|
||||
print(r)
|
||||
else:
|
||||
# tag new object
|
||||
self.set_attribute_to_tag_uuid(self.eventID_to_push, self._p_source)
|
||||
self.tag(self.attribute_to_tag, tag)
|
||||
print('Pushed:', tag, '->', self._p_source)
|
||||
print('Pushed:', tag, '->', item_id)
|
||||
|
||||
def paste_object_exist(self, eventId, source):
|
||||
res = self.pymisp.search(controller='attributes', eventid=eventId, values=source)
|
||||
def paste_object_exist(self, eventId, item_id):
|
||||
res = self.pymisp.search(controller='attributes', eventid=eventId, value=item_id)
|
||||
# object already exist
|
||||
if res['response']:
|
||||
self.attribute_to_tag = res['response']['Attribute'][0]['uuid']
|
||||
if res.get('Attribute', []):
|
||||
self.attribute_to_tag = res['Attribute'][0]['uuid']
|
||||
return True
|
||||
# new object
|
||||
else:
|
||||
return False
|
||||
|
||||
def set_attribute_to_tag_uuid(self, eventId, source):
|
||||
res = self.pymisp.search(controller='attributes', eventid=eventId, values=source)
|
||||
self.attribute_to_tag = res['response']['Attribute'][0]['uuid']
|
||||
|
||||
def tag(self, uuid, tag):
|
||||
self.pymisp.tag(uuid, tag)
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ import Screenshot
|
|||
import Correlate_object
|
||||
|
||||
import AILObjects
|
||||
import Export
|
||||
|
||||
# # TODO: # FIXME: REFRACTOR ME => use UI/Global config
|
||||
sys.path.append('../../configs/keys')
|
||||
|
|
@ -59,8 +60,12 @@ def tag_misp_object_attributes(l_ref_obj_attr, tags):
|
|||
for tag in tags:
|
||||
obj_attr.add_tag(tag)
|
||||
|
||||
def export_ail_item(item_id):
|
||||
def export_ail_item(item_id, tags=[]):
|
||||
dict_metadata = Item.get_item({'id': item_id, 'date':True, 'tags':True, 'raw_content':True})[0]
|
||||
# force tags
|
||||
for tag in tags:
|
||||
if tag not in dict_metadata['tags']:
|
||||
dict_metadata['tags'].append(tag)
|
||||
|
||||
#obj = MISPObject('ail-item', standalone=True)
|
||||
obj = MISPObject('ail-leak', standalone=True)
|
||||
|
|
@ -69,6 +74,7 @@ def export_ail_item(item_id):
|
|||
l_obj_attr = []
|
||||
l_obj_attr.append( obj.add_attribute('first-seen', value=dict_metadata['date']) )
|
||||
l_obj_attr.append( obj.add_attribute('raw-data', value=item_id, data=dict_metadata['raw_content']) )
|
||||
l_obj_attr.append( obj.add_attribute('sensor', value=Export.get_ail_uuid()) )
|
||||
|
||||
# add tags
|
||||
if dict_metadata['tags']:
|
||||
|
|
|
|||
|
|
@ -33,6 +33,9 @@ def get_item_date(item_id, add_separator=False):
|
|||
else:
|
||||
return '{}{}{}'.format(l_directory[-4], l_directory[-3], l_directory[-2])
|
||||
|
||||
def get_source(item_id):
|
||||
return item_id.split('/')[-5]
|
||||
|
||||
# # TODO: add an option to check the tag
|
||||
def is_crawled(item_id):
|
||||
return item_id.startswith('crawled')
|
||||
|
|
|
|||
|
|
@ -48,7 +48,7 @@ def get_item_date(item_id, add_separator=False):
|
|||
return item_basic.get_item_date(item_id, add_separator=add_separator)
|
||||
|
||||
def get_source(item_id):
|
||||
return item_id.split('/')[-5]
|
||||
return item_basic.get_source(item_id)
|
||||
|
||||
def get_item_basename(item_id):
|
||||
return os.path.basename(item_id)
|
||||
|
|
|
|||
|
|
@ -8,10 +8,10 @@ import datetime
|
|||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
|
||||
import Date
|
||||
import Item
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
||||
import ConfigLoader
|
||||
import item_basic
|
||||
|
||||
from pytaxonomies import Taxonomies
|
||||
from pymispgalaxies import Galaxies, Clusters
|
||||
|
|
@ -383,8 +383,8 @@ def add_obj_tag(object_type, object_id, tag, obj_date=None):
|
|||
r_serv_tags.sadd('{}:{}'.format(tag, obj_date), object_id)
|
||||
|
||||
# add domain tag
|
||||
if Item.is_crawled(object_id) and tag!='infoleak:submission="crawler"' and tag != 'infoleak:submission="manual"':
|
||||
domain = Item.get_item_domain(object_id)
|
||||
if item_basic.is_crawled(object_id) and tag!='infoleak:submission="crawler"' and tag != 'infoleak:submission="manual"':
|
||||
domain = item_basic.get_item_domain(object_id)
|
||||
add_tag("domain", tag, domain)
|
||||
else:
|
||||
r_serv_metadata.sadd('tag:{}'.format(object_id), tag)
|
||||
|
|
@ -432,7 +432,7 @@ def delete_tag(object_type, tag, object_id, obj_date=None):
|
|||
# # TODO: move me
|
||||
def get_obj_date(object_type, object_id):
|
||||
if object_type == "item":
|
||||
return int(Item.get_item_date(object_id))
|
||||
return int(item_basic.get_item_date(object_id))
|
||||
else:
|
||||
return None
|
||||
|
||||
|
|
@ -573,3 +573,19 @@ def get_obj_by_tags(object_type, l_tags, date_from=None, date_to=None, nb_obj=50
|
|||
l_tagged_obj = list(l_tagged_obj)
|
||||
|
||||
return {"tagged_obj":l_tagged_obj, "page":page, "nb_pages":nb_pages, "nb_first_elem":start+1, "nb_last_elem":stop, "nb_all_elem":nb_all_elem}
|
||||
|
||||
|
||||
#### TAGS EXPORT ####
|
||||
# # TODO:
|
||||
def is_updated_tags_to_export(): # by type
|
||||
return False
|
||||
|
||||
def get_list_of_solo_tags_to_export_by_type(export_type): # by type
|
||||
if export_type in ['misp', 'thehive']:
|
||||
return r_serv_db.smembers('whitelist_{}'.format(export_type))
|
||||
else:
|
||||
return None
|
||||
#r_serv_db.smembers('whitelist_hive')
|
||||
|
||||
|
||||
#### -- ####
|
||||
|
|
|
|||
Loading…
Reference in New Issue