mirror of https://github.com/CIRCL/AIL-framework
misp event creation, auto and manual + the hive alert auto creation, manual case creation
parent
8522d50308
commit
31fc33ab03
|
@ -160,6 +160,8 @@ function launching_scripts {
|
|||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "alertHandler" bash -c './alertHandler.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "MISPtheHIVEfeeder" bash -c './MISP_The_Hive_feeder.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Tags" bash -c './Tags.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "SentimentAnalysis" bash -c './SentimentAnalysis.py; read x'
|
||||
|
|
|
@ -0,0 +1,165 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
"""
|
||||
module
|
||||
====================
|
||||
|
||||
This module send tagged pastes to MISP or THE HIVE Project
|
||||
|
||||
"""
|
||||
|
||||
import redis
|
||||
import sys
|
||||
import os
|
||||
import time
|
||||
import json
|
||||
import configparser
|
||||
|
||||
from pubsublogger import publisher
|
||||
from Helper import Process
|
||||
from packages import Paste
|
||||
import ailleakObject
|
||||
|
||||
import uuid
|
||||
|
||||
from pymisp import PyMISP
|
||||
|
||||
sys.path.append('../configs/keys')
|
||||
|
||||
# import MISP KEYS
|
||||
try:
|
||||
from mispKEYS import misp_url, misp_key, misp_verifycert
|
||||
flag_misp = True
|
||||
except:
|
||||
print('Misp keys not present')
|
||||
flag_misp = False
|
||||
|
||||
# import The Hive Keys
|
||||
try:
|
||||
from theHiveKEYS import the_hive_url, the_hive_key
|
||||
flag_the_hive = True
|
||||
except:
|
||||
print('The HIVE keys not present')
|
||||
flag_the_hive = False
|
||||
|
||||
from thehive4py.api import TheHiveApi
|
||||
from thehive4py.models import Alert, AlertArtifact
|
||||
from thehive4py.models import Case, CaseTask, CustomFieldHelper
|
||||
|
||||
|
||||
|
||||
def create_the_hive_alert(source, path, content, tag):
|
||||
tags = list(r_serv_metadata.smembers('tag:'+path))
|
||||
|
||||
artifacts = [
|
||||
AlertArtifact( dataType='uuid-ail', data=r_serv_db.get('ail:uuid') ),
|
||||
AlertArtifact( dataType='file', data=path, tags=tags )
|
||||
]
|
||||
|
||||
l_tags = tag.split(',')
|
||||
print(tag)
|
||||
|
||||
# Prepare the sample Alert
|
||||
sourceRef = str(uuid.uuid4())[0:6]
|
||||
alert = Alert(title='AIL Leak',
|
||||
tlp=3,
|
||||
tags=l_tags,
|
||||
description='infoleak',
|
||||
type='ail',
|
||||
source=source,
|
||||
sourceRef=sourceRef,
|
||||
artifacts=artifacts)
|
||||
|
||||
# Create the Alert
|
||||
id = None
|
||||
response = HiveApi.create_alert(alert)
|
||||
if response.status_code == 201:
|
||||
#print(json.dumps(response.json(), indent=4, sort_keys=True))
|
||||
print('Alert Created')
|
||||
print('')
|
||||
id = response.json()['id']
|
||||
else:
|
||||
print('ko: {}/{}'.format(response.status_code, response.text))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
publisher.port = 6380
|
||||
publisher.channel = "Script"
|
||||
|
||||
config_section = 'misp_the_hive_feeder'
|
||||
|
||||
configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg')
|
||||
if not os.path.exists(configfile):
|
||||
raise Exception('Unable to find the configuration file. \
|
||||
Did you set environment variables? \
|
||||
Or activate the virtualenv.')
|
||||
|
||||
cfg = configparser.ConfigParser()
|
||||
cfg.read(configfile)
|
||||
|
||||
r_serv_db = redis.StrictRedis(
|
||||
host=cfg.get("ARDB_DB", "host"),
|
||||
port=cfg.getint("ARDB_DB", "port"),
|
||||
db=cfg.getint("ARDB_DB", "db"),
|
||||
decode_responses=True)
|
||||
|
||||
r_serv_metadata = redis.StrictRedis(
|
||||
host=cfg.get("ARDB_Metadata", "host"),
|
||||
port=cfg.getint("ARDB_Metadata", "port"),
|
||||
db=cfg.getint("ARDB_Metadata", "db"),
|
||||
decode_responses=True)
|
||||
|
||||
uuid_ail = r_serv_db.get('ail:uuid')
|
||||
if uuid_ail is None:
|
||||
uuid_ail = r_serv_db.set('ail:uuid', uuid.uuid4() )
|
||||
|
||||
config_section = 'misp_the_hive_feeder'
|
||||
|
||||
p = Process(config_section)
|
||||
# create MISP connection
|
||||
if flag_misp:
|
||||
#try:
|
||||
pymisp = PyMISP(misp_url, misp_key, misp_verifycert)
|
||||
misp_wrapper = ailleakObject.ObjectWrapper(pymisp)
|
||||
r_serv_db.set('ail:misp', True)
|
||||
print('Connected to MISP:', misp_url)
|
||||
#except:
|
||||
#flag_misp = False
|
||||
#print('Not connected to MISP')
|
||||
|
||||
# create The HIVE connection
|
||||
if flag_the_hive:
|
||||
try:
|
||||
HiveApi = TheHiveApi(the_hive_url, the_hive_key)
|
||||
r_serv_db.set('ail:thehive', True)
|
||||
print('Connected to The HIVE:', the_hive_url)
|
||||
except:
|
||||
HiveApi = False
|
||||
print('Not connected to The HIVE')
|
||||
|
||||
while True:
|
||||
|
||||
# Get one message from the input queue
|
||||
message = p.get_from_set()
|
||||
if message is None:
|
||||
publisher.debug("{} queue is empty, waiting 1s".format(config_section))
|
||||
time.sleep(1)
|
||||
continue
|
||||
else:
|
||||
|
||||
if HiveApi or flag_misp:
|
||||
tag, path = message.split(';')
|
||||
paste = Paste.Paste(path)
|
||||
source = '/'.join(paste.p_path.split('/')[-6:])
|
||||
|
||||
full_path = os.path.join(os.environ['AIL_HOME'],
|
||||
p.config.get("Directories", "pastes"), path)
|
||||
|
||||
if HiveApi != False:
|
||||
create_the_hive_alert(source, path, full_path, tag)
|
||||
|
||||
if flag_misp:
|
||||
misp_wrapper.pushToMISP(uuid_ail, path, tag)
|
|
@ -66,3 +66,5 @@ if __name__ == '__main__':
|
|||
print("new paste: {}".format(path))
|
||||
print(" tagged: {}".format(tag))
|
||||
server_metadata.sadd('tag:'+path, tag)
|
||||
|
||||
p.populate_set_out(message, 'MISP_The_Hive_feeder')
|
||||
|
|
|
@ -8,12 +8,11 @@ import datetime
|
|||
import json
|
||||
from io import BytesIO
|
||||
|
||||
class AilleakObject(AbstractMISPObjectGenerator):
|
||||
def __init__(self, moduleName, p_source, p_date, p_content, p_duplicate, p_duplicate_number):
|
||||
class AilLeakObject(AbstractMISPObjectGenerator):
|
||||
def __init__(self, uuid_ail, p_source, p_date, p_content, p_duplicate, p_duplicate_number):
|
||||
super(AbstractMISPObjectGenerator, self).__init__('ail-leak')
|
||||
self._moduleName = moduleName
|
||||
self._p_source = p_source.split('/')[-5:]
|
||||
self._p_source = '/'.join(self._p_source)[:-3] # -3 removes .gz
|
||||
self._uuid = uuid_ail
|
||||
self._p_source = p_source
|
||||
self._p_date = p_date
|
||||
self._p_content = p_content
|
||||
self._p_duplicate = p_duplicate
|
||||
|
@ -21,14 +20,17 @@ class AilleakObject(AbstractMISPObjectGenerator):
|
|||
self.generate_attributes()
|
||||
|
||||
def generate_attributes(self):
|
||||
self.add_attribute('type', value=self._moduleName)
|
||||
self.add_attribute('origin', value=self._p_source, type='text')
|
||||
self.add_attribute('last-seen', value=self._p_date)
|
||||
self.add_attribute('last-seen', value=self._p_date, type='datetime')
|
||||
if self._p_duplicate_number > 0:
|
||||
self.add_attribute('duplicate', value=self._p_duplicate, type='text')
|
||||
self.add_attribute('duplicate_number', value=self._p_duplicate_number, type='counter')
|
||||
self._pseudofile = BytesIO(self._p_content)
|
||||
self.add_attribute('raw-data', value=self._p_source, data=self._pseudofile, type="attachment")
|
||||
self._pseudofile = BytesIO(self._p_content.encode())
|
||||
res = self.add_attribute('raw-data', value=self._p_source, data=self._pseudofile, type="attachment")# , ShadowAttribute=self.p_tag)
|
||||
#res.add_shadow_attributes(tag)
|
||||
self.add_attribute('sensor', value=self._uuid, type="text")
|
||||
# FIXME TODO: delete this
|
||||
self.add_attribute('type', value='Onion', type='text')
|
||||
|
||||
class ObjectWrapper:
|
||||
def __init__(self, pymisp):
|
||||
|
@ -38,30 +40,40 @@ class ObjectWrapper:
|
|||
cfg = configparser.ConfigParser()
|
||||
cfg.read('./packages/config.cfg')
|
||||
self.maxDuplicateToPushToMISP = cfg.getint("ailleakObject", "maxDuplicateToPushToMISP")
|
||||
self.attribute_to_tag = None
|
||||
|
||||
def add_new_object(self, moduleName, path):
|
||||
self.moduleName = moduleName
|
||||
def add_new_object(self, uuid_ail, path, p_source, tag):
|
||||
self.uuid_ail = uuid_ail
|
||||
self.path = path
|
||||
self.p_source = p_source
|
||||
self.paste = Paste.Paste(path)
|
||||
self.p_date = self.date_to_str(self.paste.p_date)
|
||||
self.p_source = self.paste.p_path
|
||||
self.p_content = self.paste.get_p_content()
|
||||
self.p_tag = tag
|
||||
|
||||
'''print(path)
|
||||
temp = self.paste._get_p_duplicate()
|
||||
|
||||
#beautifier
|
||||
temp = json.loads(temp)
|
||||
if not temp:
|
||||
temp = ''
|
||||
|
||||
temp = json.dumps(temp)
|
||||
print(temp)
|
||||
self.p_duplicate_number = len(temp) if len(temp) >= 0 else 0
|
||||
to_ret = ""
|
||||
for dup in temp[:self.maxDuplicateToPushToMISP]:
|
||||
print(dup)
|
||||
algo = dup[0]
|
||||
path = dup[1].split('/')[-5:]
|
||||
path = dup[1].split('/')[-6:]
|
||||
path = '/'.join(path)[:-3] # -3 removes .gz
|
||||
perc = dup[2]
|
||||
to_ret += "{}: {} [{}%]\n".format(path, algo, perc)
|
||||
self.p_duplicate = to_ret
|
||||
self.p_duplicate = to_ret'''
|
||||
self.p_duplicate = ""
|
||||
self.p_duplicate_number = 0
|
||||
|
||||
self.mispObject = AilleakObject(self.moduleName, self.p_source, self.p_date, self.p_content, self.p_duplicate, self.p_duplicate_number)
|
||||
self.mispObject = AilLeakObject(self.uuid_ail, self.p_source, self.p_date, self.p_content, self.p_duplicate, self.p_duplicate_number)
|
||||
|
||||
def date_to_str(self, date):
|
||||
return "{0}-{1}-{2}".format(date.year, date.month, date.day)
|
||||
|
@ -108,21 +120,57 @@ class ObjectWrapper:
|
|||
event = self.pymisp.new_event(distribution, threat,
|
||||
analysis, info, date,
|
||||
published, orgc_id, org_id, sharing_group_id)
|
||||
eventUuid = event['Event']['uuid']
|
||||
self.pymisp.tag(eventUuid, 'infoleak:source="unknown"')
|
||||
return event
|
||||
|
||||
# Publish object to MISP
|
||||
def pushToMISP(self):
|
||||
def pushToMISP(self, uuid_ail, path, tag):
|
||||
self._p_source = path.split('/')[-5:]
|
||||
self._p_source = '/'.join(self._p_source)[:-3]
|
||||
|
||||
if self.currentID_date != datetime.date.today(): #refresh id
|
||||
self.eventID_to_push = self.get_daily_event_id()
|
||||
|
||||
mispTYPE = 'ail-leak'
|
||||
try:
|
||||
templateID = [x['ObjectTemplate']['id'] for x in self.pymisp.get_object_templates_list() if x['ObjectTemplate']['name'] == mispTYPE][0]
|
||||
except IndexError:
|
||||
valid_types = ", ".join([x['ObjectTemplate']['name'] for x in self.pymisp.get_object_templates_list()])
|
||||
print ("Template for type %s not found! Valid types are: %s" % (mispTYPE, valid_types))
|
||||
r = self.pymisp.add_object(self.eventID_to_push, templateID, self.mispObject)
|
||||
if 'errors' in r:
|
||||
print(r)
|
||||
|
||||
# paste object already exist
|
||||
if self.paste_object_exist(self.eventID_to_push, self._p_source):
|
||||
# add new tag
|
||||
self.tag(self.attribute_to_tag, tag)
|
||||
print(self._p_source + ' tagged: ' + tag)
|
||||
#create object
|
||||
else:
|
||||
print('Pushed:', self.moduleName, '->', self.p_source)
|
||||
self.add_new_object(uuid_ail, path, self._p_source, tag)
|
||||
|
||||
|
||||
try:
|
||||
templateID = [x['ObjectTemplate']['id'] for x in self.pymisp.get_object_templates_list() if x['ObjectTemplate']['name'] == mispTYPE][0]
|
||||
except IndexError:
|
||||
valid_types = ", ".join([x['ObjectTemplate']['name'] for x in self.pymisp.get_object_templates_list()])
|
||||
print ("Template for type %s not found! Valid types are: %s" % (mispTYPE, valid_types))
|
||||
r = self.pymisp.add_object(self.eventID_to_push, templateID, self.mispObject)
|
||||
if 'errors' in r:
|
||||
print(r)
|
||||
else:
|
||||
# tag new object
|
||||
self.set_attribute_to_tag_uuid(self.eventID_to_push, self._p_source)
|
||||
self.tag(self.attribute_to_tag, tag)
|
||||
print('Pushed:', tag, '->', self._p_source)
|
||||
|
||||
def paste_object_exist(self, eventId, source):
|
||||
res = self.pymisp.search(controller='attributes', eventid=eventId, values=source)
|
||||
# object already exist
|
||||
if res['response']:
|
||||
self.attribute_to_tag = res['response']['Attribute'][0]['uuid']
|
||||
return True
|
||||
# new object
|
||||
else:
|
||||
return False
|
||||
|
||||
def set_attribute_to_tag_uuid(self, eventId, source):
|
||||
res = self.pymisp.search(controller='attributes', eventid=eventId, values=source)
|
||||
self.attribute_to_tag = res['response']['Attribute'][0]['uuid']
|
||||
|
||||
def tag(self, uuid, tag):
|
||||
self.pymisp.tag(uuid, tag)
|
||||
|
|
|
@ -20,16 +20,10 @@ from packages import Paste
|
|||
from pubsublogger import publisher
|
||||
from Helper import Process
|
||||
|
||||
from pymisp import PyMISP
|
||||
import ailleakObject
|
||||
import sys
|
||||
sys.path.append('../')
|
||||
try:
|
||||
from mispKEYS import misp_url, misp_key, misp_verifycert
|
||||
flag_misp = True
|
||||
except:
|
||||
print('Misp keys not present')
|
||||
flag_misp = False
|
||||
|
||||
flag_misp = False
|
||||
|
||||
if __name__ == "__main__":
|
||||
publisher.port = 6380
|
||||
|
@ -38,16 +32,6 @@ if __name__ == "__main__":
|
|||
config_section = 'alertHandler'
|
||||
|
||||
p = Process(config_section)
|
||||
if flag_misp:
|
||||
try:
|
||||
pymisp = PyMISP(misp_url, misp_key, misp_verifycert)
|
||||
print('Connected to MISP:', misp_url)
|
||||
except:
|
||||
flag_misp = False
|
||||
print('Not connected to MISP')
|
||||
|
||||
if flag_misp:
|
||||
wrapper = ailleakObject.ObjectWrapper(pymisp)
|
||||
|
||||
# port generated automatically depending on the date
|
||||
curYear = datetime.now().year
|
||||
|
@ -77,12 +61,3 @@ if __name__ == "__main__":
|
|||
server.sadd(key, p_path)
|
||||
|
||||
publisher.info('Saved warning paste {}'.format(p_path))
|
||||
|
||||
# Create MISP AIL-leak object and push it
|
||||
if flag_misp:
|
||||
allowed_modules = ['credential', 'phone', 'creditcards']
|
||||
if module_name in allowed_modules:
|
||||
wrapper.add_new_object(module_name, p_path)
|
||||
wrapper.pushToMISP()
|
||||
else:
|
||||
print('not pushing to MISP:', module_name, p_path)
|
||||
|
|
|
@ -76,7 +76,7 @@ class Paste(object):
|
|||
port=cfg.getint("Redis_Data_Merging", "port"),
|
||||
db=cfg.getint("Redis_Data_Merging", "db"),
|
||||
decode_responses=True)
|
||||
self.store_duplicate = redis.StrictRedis(
|
||||
self.store_metadata = redis.StrictRedis(
|
||||
host=cfg.get("ARDB_Metadata", "host"),
|
||||
port=cfg.getint("ARDB_Metadata", "port"),
|
||||
db=cfg.getint("ARDB_Metadata", "db"),
|
||||
|
@ -105,6 +105,7 @@ class Paste(object):
|
|||
self.p_max_length_line = None
|
||||
self.array_line_above_threshold = None
|
||||
self.p_duplicate = None
|
||||
self.p_tags = None
|
||||
|
||||
def get_p_content(self):
|
||||
"""
|
||||
|
@ -277,12 +278,19 @@ class Paste(object):
|
|||
return False, var
|
||||
|
||||
def _get_p_duplicate(self):
|
||||
self.p_duplicate = self.store_duplicate.smembers('dup:'+self.p_path)
|
||||
self.p_duplicate = self.store_metadata.smembers('dup:'+self.p_path)
|
||||
if self.p_duplicate is not None:
|
||||
return list(self.p_duplicate)
|
||||
else:
|
||||
return '[]'
|
||||
|
||||
def _get_p_tags(self):
|
||||
self.p_tags = self.store_metadata.smembers('tag:'+path, tag)
|
||||
if self.self.p_tags is not None:
|
||||
return list(self.p_tags)
|
||||
else:
|
||||
return '[]'
|
||||
|
||||
def save_all_attributes_redis(self, key=None):
|
||||
"""
|
||||
Saving all the attributes in a "Redis-like" Database (Redis, LevelDB)
|
||||
|
@ -333,7 +341,7 @@ class Paste(object):
|
|||
Save an attribute as a field
|
||||
"""
|
||||
for tuple in value:
|
||||
self.store_duplicate.sadd('dup:'+self.p_path, tuple)
|
||||
self.store_metadata.sadd('dup:'+self.p_path, tuple)
|
||||
|
||||
def save_others_pastes_attribute_duplicate(self, list_value):
|
||||
"""
|
||||
|
@ -341,7 +349,7 @@ class Paste(object):
|
|||
"""
|
||||
for hash_type, path, percent, date in list_value:
|
||||
to_add = (hash_type, self.p_path, percent, date)
|
||||
self.store_duplicate.sadd('dup:'+path,to_add)
|
||||
self.store_metadata.sadd('dup:'+path,to_add)
|
||||
|
||||
def _get_from_redis(self, r_serv):
|
||||
ans = {}
|
||||
|
|
|
@ -82,6 +82,10 @@ subscribe = Redis_alertHandler
|
|||
|
||||
[Tags]
|
||||
subscribe = Redis_Tags
|
||||
publish = Redis_Tags_feed
|
||||
|
||||
[misp_the_hive_feeder]
|
||||
subscribe = Redis_Tags_feed
|
||||
|
||||
#[send_to_queue]
|
||||
#subscribe = Redis_Cve
|
||||
|
|
|
@ -0,0 +1,6 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
the_hive_url = ''
|
||||
the_hive_key = '' # The Hive auth key can be found on the The Hive web interface under the User Management
|
||||
the_hive_verifycert = True
|
|
@ -1,5 +1,7 @@
|
|||
pymisp
|
||||
|
||||
thehive4py
|
||||
|
||||
redis
|
||||
#filemagic conflict with magic
|
||||
crcmod
|
||||
|
|
|
@ -7,6 +7,7 @@
|
|||
import configparser
|
||||
import redis
|
||||
import os
|
||||
import sys
|
||||
|
||||
# FLASK #
|
||||
app = None
|
||||
|
@ -22,7 +23,6 @@ if not os.path.exists(configfile):
|
|||
cfg = configparser.ConfigParser()
|
||||
cfg.read(configfile)
|
||||
|
||||
|
||||
# REDIS #
|
||||
r_serv = redis.StrictRedis(
|
||||
host=cfg.get("Redis_Queues", "host"),
|
||||
|
@ -90,6 +90,32 @@ r_serv_db = redis.StrictRedis(
|
|||
db=cfg.getint("ARDB_DB", "db"),
|
||||
decode_responses=True)
|
||||
|
||||
|
||||
sys.path.append('../../configs/keys')
|
||||
# MISP #
|
||||
from pymisp import PyMISP
|
||||
try:
|
||||
from mispKEYS import misp_url, misp_key, misp_verifycert
|
||||
pymisp = PyMISP(misp_url, misp_key, misp_verifycert)
|
||||
misp_event_url = misp_url + '/events/view/'
|
||||
print('Misp connected')
|
||||
except:
|
||||
print('Misp not connected')
|
||||
pymisp = None
|
||||
misp_event_url = '#'
|
||||
# The Hive #
|
||||
from thehive4py.api import TheHiveApi
|
||||
import thehive4py.exceptions
|
||||
try:
|
||||
from theHiveKEYS import the_hive_url, the_hive_key
|
||||
HiveApi = TheHiveApi(the_hive_url, the_hive_key)
|
||||
hive_case_url = the_hive_url+'/index.html#/case/id_here/details'
|
||||
print('The Hive connected')
|
||||
except:
|
||||
print('The HIVE not connected')
|
||||
HiveApi = None
|
||||
hive_case_url = '#'
|
||||
|
||||
# VARIABLES #
|
||||
max_preview_char = int(cfg.get("Flask", "max_preview_char")) # Maximum number of character to display in the tooltip
|
||||
max_preview_modal = int(cfg.get("Flask", "max_preview_modal")) # Maximum number of character to display in the modal
|
||||
|
|
|
@ -14,10 +14,17 @@ import os
|
|||
import sys
|
||||
import datetime
|
||||
import uuid
|
||||
from io import BytesIO
|
||||
from Date import Date
|
||||
|
||||
import Paste
|
||||
|
||||
from pytaxonomies import Taxonomies
|
||||
from pymispgalaxies import Galaxies, Clusters
|
||||
|
||||
from pymisp.mispevent import MISPObject
|
||||
from thehive4py.models import Case, CaseTask, CustomFieldHelper, CaseObservable
|
||||
|
||||
# ============ VARIABLES ============
|
||||
import Flask_config
|
||||
|
||||
|
@ -28,6 +35,10 @@ r_serv_metadata = Flask_config.r_serv_metadata
|
|||
r_serv_db = Flask_config.r_serv_db
|
||||
r_serv_log_submit = Flask_config.r_serv_log_submit
|
||||
|
||||
pymisp = Flask_config.pymisp
|
||||
|
||||
HiveApi = Flask_config.HiveApi
|
||||
|
||||
PasteSubmit = Blueprint('PasteSubmit', __name__, template_folder='templates')
|
||||
|
||||
valid_filename_chars = "-_ %s%s" % (string.ascii_letters, string.digits)
|
||||
|
@ -35,6 +46,9 @@ valid_filename_chars = "-_ %s%s" % (string.ascii_letters, string.digits)
|
|||
ALLOWED_EXTENSIONS = set(['txt', 'zip', 'gz', 'tar.gz'])
|
||||
UPLOAD_FOLDER = Flask_config.UPLOAD_FOLDER
|
||||
|
||||
misp_event_url = Flask_config.misp_event_url
|
||||
hive_case_url = Flask_config.hive_case_url
|
||||
|
||||
# ============ FUNCTIONS ============
|
||||
def one():
|
||||
return 1
|
||||
|
@ -58,8 +72,6 @@ def clean_filename(filename, whitelist=valid_filename_chars, replace=' '):
|
|||
|
||||
def launch_submit(ltags, ltagsgalaxies, paste_content, UUID, password, isfile = False):
|
||||
|
||||
print(UUID)
|
||||
|
||||
# save temp value on disk
|
||||
r_serv_db.set(UUID + ':ltags', ltags)
|
||||
r_serv_db.set(UUID + ':ltagsgalaxies', ltagsgalaxies)
|
||||
|
@ -116,6 +128,105 @@ def addTagsVerification(tags, tagsgalaxies):
|
|||
else:
|
||||
return False
|
||||
return True
|
||||
|
||||
def date_to_str(date):
|
||||
return "{0}-{1}-{2}".format(date.year, date.month, date.day)
|
||||
|
||||
def misp_create_event(distribution, threat_level_id, analysis, info, l_tags, path):
|
||||
|
||||
paste = Paste.Paste(path)
|
||||
source = path.split('/')[-6:]
|
||||
source = '/'.join(source)[:-3]
|
||||
ail_uuid = r_serv_db.get('ail:uuid')
|
||||
pseudofile = BytesIO(paste.get_p_content().encode())
|
||||
|
||||
today = datetime.date.today()
|
||||
# [0-3]
|
||||
published = False
|
||||
org_id = None
|
||||
orgc_id = None
|
||||
sharing_group_id = None
|
||||
date = today
|
||||
event = pymisp.new_event(distribution, threat_level_id,
|
||||
analysis, info, date,
|
||||
published, orgc_id, org_id, sharing_group_id)
|
||||
eventUuid = event['Event']['uuid']
|
||||
eventid = event['Event']['id']
|
||||
|
||||
# add tags
|
||||
for tag in l_tags:
|
||||
pymisp.tag(eventUuid, tag)
|
||||
|
||||
# create attributes
|
||||
obj_name = 'ail-leak'
|
||||
leak_obj = MISPObject(obj_name)
|
||||
leak_obj.add_attribute('sensor', value=ail_uuid, type="text")
|
||||
leak_obj.add_attribute('origin', value=source, type='text')
|
||||
leak_obj.add_attribute('last-seen', value=date_to_str(paste.p_date), type='datetime')
|
||||
leak_obj.add_attribute('raw-data', value=source, data=pseudofile, type="attachment")
|
||||
# FIXME TODO: delete this
|
||||
leak_obj.add_attribute('type', value='Onion', type='text')
|
||||
|
||||
try:
|
||||
templateID = [x['ObjectTemplate']['id'] for x in pymisp.get_object_templates_list() if x['ObjectTemplate']['name'] == obj_name][0]
|
||||
except IndexError:
|
||||
valid_types = ", ".join([x['ObjectTemplate']['name'] for x in pymisp.get_object_templates_list()])
|
||||
print ("Template for type {} not found! Valid types are: {%s}".format(obj_name, valid_types))
|
||||
r = pymisp.add_object(eventid, templateID, leak_obj)
|
||||
if 'errors' in r:
|
||||
return False
|
||||
else:
|
||||
#if self._p_duplicate_number > 0:
|
||||
#event.add_attribute('duplicate', value=self._p_duplicate, type='text')
|
||||
#event.add_attribute('duplicate_number', value=self._p_duplicate_number, type='counter')
|
||||
event_url = misp_event_url + eventid
|
||||
return eventid
|
||||
|
||||
def hive_create_case(hive_tlp, threat_level, hive_description, hive_case_title, l_tags, path):
|
||||
|
||||
ail_uuid = r_serv_db.get('ail:uuid')
|
||||
source = path.split('/')[-6:]
|
||||
source = '/'.join(source)[:-3]
|
||||
# get paste date
|
||||
var = path.split('/')
|
||||
last_seen = "{0}-{1}-{2}".format(var[-4], var[-3], var[-2])
|
||||
|
||||
case = Case(title=hive_case_title,
|
||||
tlp=hive_tlp,
|
||||
severity=threat_level,
|
||||
flag=False,
|
||||
tags=l_tags,
|
||||
description='hive_description')
|
||||
|
||||
# Create the case
|
||||
id = None
|
||||
response = HiveApi.create_case(case)
|
||||
if response.status_code == 201:
|
||||
id = response.json()['id']
|
||||
|
||||
observ_sensor = CaseObservable(dataType="other", data=[ail_uuid], message="sensor")
|
||||
observ_file = CaseObservable(dataType="file", data=[path], tags=l_tags)
|
||||
observ_source = CaseObservable(dataType="other", data=[source], message="source")
|
||||
observ_last_seen = CaseObservable(dataType="other", data=[last_seen], message="last-seen")
|
||||
|
||||
res = HiveApi.create_case_observable(id,observ_sensor)
|
||||
if res.status_code != 201:
|
||||
print('ko: {}/{}'.format(res.status_code, res.text))
|
||||
res = HiveApi.create_case_observable(id, observ_source)
|
||||
if res.status_code != 201:
|
||||
print('ko: {}/{}'.format(res.status_code, res.text))
|
||||
res = HiveApi.create_case_observable(id, observ_file)
|
||||
if res.status_code != 201:
|
||||
print('ko: {}/{}'.format(res.status_code, res.text))
|
||||
res = HiveApi.create_case_observable(id, observ_last_seen)
|
||||
if res.status_code != 201:
|
||||
print('ko: {}/{}'.format(res.status_code, res.text))
|
||||
|
||||
return hive_case_url.replace('id_here', id)
|
||||
else:
|
||||
print('ko: {}/{}'.format(response.status_code, response.text))
|
||||
return False
|
||||
|
||||
# ============= ROUTES ==============
|
||||
|
||||
@PasteSubmit.route("/PasteSubmit/", methods=['GET'])
|
||||
|
@ -150,10 +261,7 @@ def submit():
|
|||
else:
|
||||
ltags ='submitted'
|
||||
|
||||
print(request.files)
|
||||
|
||||
if 'file' in request.files:
|
||||
print(request.files)
|
||||
|
||||
file = request.files['file']
|
||||
if file:
|
||||
|
@ -196,7 +304,6 @@ def submit():
|
|||
|
||||
# get id
|
||||
UUID = str(uuid.uuid4())
|
||||
print(UUID)
|
||||
|
||||
#if paste_name:
|
||||
# clean file name
|
||||
|
@ -280,5 +387,42 @@ def submit_status():
|
|||
else:
|
||||
return 'INVALID UUID'
|
||||
|
||||
|
||||
@PasteSubmit.route("/PasteSubmit/create_misp_event", methods=['POST'])
|
||||
def create_misp_event():
|
||||
|
||||
distribution = int(request.form['misp_data[Event][distribution]'])
|
||||
threat_level_id = int(request.form['misp_data[Event][threat_level_id]'])
|
||||
analysis = int(request.form['misp_data[Event][analysis]'])
|
||||
info = request.form['misp_data[Event][info]']
|
||||
path = request.form['paste']
|
||||
|
||||
#verify input
|
||||
if (0 <= distribution <= 3) and (1 <= threat_level_id <= 4) and (0 <= analysis <= 2):
|
||||
|
||||
l_tags = list(r_serv_metadata.smembers('tag:'+path))
|
||||
event = misp_create_event(distribution, threat_level_id, analysis, info, l_tags, path)
|
||||
|
||||
|
||||
return event
|
||||
|
||||
@PasteSubmit.route("/PasteSubmit/create_hive_case", methods=['POST'])
|
||||
def create_hive_case():
|
||||
|
||||
hive_tlp = int(request.form['hive_tlp'])
|
||||
threat_level = int(request.form['threat_level_hive'])
|
||||
hive_description = request.form['hive_description']
|
||||
hive_case_title = request.form['hive_case_title']
|
||||
path = request.form['paste']
|
||||
|
||||
#verify input
|
||||
if (0 <= hive_tlp <= 3) and (1 <= threat_level <= 4):
|
||||
|
||||
l_tags = list(r_serv_metadata.smembers('tag:'+path))
|
||||
case = hive_create_case(hive_tlp, threat_level, hive_description, hive_case_title, l_tags, path)
|
||||
|
||||
|
||||
return case
|
||||
|
||||
# ========= REGISTRATION =========
|
||||
app.register_blueprint(PasteSubmit)
|
||||
|
|
|
@ -170,8 +170,136 @@
|
|||
</tbody>
|
||||
</table>
|
||||
|
||||
<div>
|
||||
<button class="btn btn-default btn-lg" data-toggle="modal" data-target="#myModal_MISP">
|
||||
Create
|
||||
<img id="misp-logo" src="{{url_for('static', filename='image/misp-logo.png') }}" height="32">
|
||||
Event
|
||||
</button>
|
||||
<!-- Modal MISP-->
|
||||
<div class="modal fade" id="myModal_MISP" role="dialog">
|
||||
<div class="modal-dialog">
|
||||
|
||||
<div class="modal-content">
|
||||
<div class="modal-header text-center">
|
||||
<img id="misp-logo" src="{{url_for('static', filename='image/misp-logo.png') }}" >
|
||||
</div>
|
||||
<div class="modal-body">
|
||||
|
||||
<form method="post" action="/PasteSubmit/create_misp_event"target="_blank">
|
||||
|
||||
<div class="input select required">
|
||||
<label for="EventDistribution">Distribution</label>
|
||||
<select name="misp_data[Event][distribution]" id="EventDistribution" required="required">
|
||||
<option value="0" selected="selected">Your organisation only</option>
|
||||
<option value="1">This community only</option>
|
||||
<option value="2">Connected communities</option>
|
||||
<option value="3">All communities</option>
|
||||
</select>
|
||||
</div>
|
||||
<div class="input clear required">
|
||||
<label for="EventThreatLevelId">Threat Level</label>
|
||||
<select name="misp_data[Event][threat_level_id]" id="EventThreatLevelId" required="required">
|
||||
<option value="1" selected="selected">High</option>
|
||||
<option value="2">Medium</option>
|
||||
<option value="3">Low</option>
|
||||
<option value="4">Undefined</option>
|
||||
</select>
|
||||
</div>
|
||||
<div class="input select required">
|
||||
<label for="EventAnalysis">Analysis</label>
|
||||
<select name="misp_data[Event][analysis]" id="EventAnalysis" required="required">
|
||||
<option value="0">Initial</option>
|
||||
<option value="1">Ongoing</option>
|
||||
<option value="2">Completed</option>
|
||||
</select>
|
||||
</div>
|
||||
<div class="clear required">
|
||||
<label for="EventInfo">Event Info</label>
|
||||
<input name="misp_data[Event][info]" class="form-control span6" placeholder="Quick Event Description or Tracking Info" type="text" id="EventInfo" required="required"/>
|
||||
</div>
|
||||
<input type="hidden" id="paste" name="paste" value="{{ request.args.get('paste') }}">
|
||||
|
||||
</div>
|
||||
<div class="modal-footer center">
|
||||
<button class="btn btn-primary">
|
||||
<span class="glyphicon glyphicon-ok "></span> Create Event
|
||||
</button>
|
||||
|
||||
</form>
|
||||
<button type="button" class="btn btn-default" data-dismiss="modal" >Close</button>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<button class="btn btn-primary btn-lg" data-toggle="modal" data-target="#myModal_hive">
|
||||
Create
|
||||
<img id="thehive-icon" src="{{url_for('static', filename='image/thehive_icon.png') }}">
|
||||
Case
|
||||
</button>
|
||||
<!-- Modal HIVE-->
|
||||
<div class="modal fade" id="myModal_hive" role="dialog">
|
||||
<div class="modal-dialog">
|
||||
|
||||
<div class="modal-content">
|
||||
<div class="modal-header text-center">
|
||||
<img id="misp-logo" src="{{url_for('static', filename='image/thehive-logo.png') }}" width="500" >
|
||||
</div>
|
||||
|
||||
<div class="modal-body">
|
||||
|
||||
<form method="post" action="/PasteSubmit/create_hive_case" target="_blank">
|
||||
|
||||
<div class="input clear required">
|
||||
<label for="EventThreatLevelId">Threat Level</label>
|
||||
<select name="threat_level_hive" id="EventThreatLevelId" required="required">
|
||||
<option value="1">High</option>
|
||||
<option value="2" selected="selected">Medium</option>
|
||||
<option value="3">Low</option>
|
||||
</select>
|
||||
</div>
|
||||
<div class="input select required">
|
||||
<label for="TLP">TLP</label>
|
||||
<select name="hive_tlp" id="hive_tlp" required="required" class="selectpicker">
|
||||
<option value="0">White</option>
|
||||
<option value="1">Green</option>
|
||||
<option value="2" selected="selected">Amber</option>
|
||||
<option value="3">Red</option>
|
||||
</select>
|
||||
</div>
|
||||
<div class="clear required">
|
||||
<label for="hive_case_title">Title</label>
|
||||
<input name="hive_case_title" class="form-control span6" placeholder="Title" type="text" id="hive_case_title" required="required"/>
|
||||
</div>
|
||||
<div class="clear required">
|
||||
<label for="hive_description">Description</label>
|
||||
<input name="hive_description" class="form-control span6" placeholder="Quick Case Description" type="text" id="hive_description" required="required"/>
|
||||
</div>
|
||||
<input type="hidden" id="paste" name="paste" value="{{ request.args.get('paste') }}">
|
||||
|
||||
</div>
|
||||
<div class="modal-footer center">
|
||||
<button href="" class="btn btn-primary">
|
||||
<span class="glyphicon glyphicon-ok "></span> Create Case
|
||||
</button>
|
||||
|
||||
</form>
|
||||
<button type="button" class="btn btn-default" data-dismiss="modal" >Close</button>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
<div class="panel-body" id="panel-body">
|
||||
|
||||
{% if duplicate_list|length == 0 %}
|
||||
<h3> No Duplicate </h3>
|
||||
{% else %}
|
||||
|
@ -203,8 +331,8 @@
|
|||
<h3> Content: </h3>
|
||||
<a href="{{ url_for('showsavedpastes.showsavedrawpaste') }}?paste={{ request.args.get('paste') }}" id='raw_paste' > [Raw content] </a>
|
||||
<p data-initsize="{{ initsize }}"> <pre id="paste-holder">{{ content }}</pre></p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</body>
|
||||
|
||||
|
|
Loading…
Reference in New Issue