mirror of https://github.com/CIRCL/AIL-framework
chg: [DB] remove ARDB + temp disable MISP import and auto export
Terrtia
parent
54a0bcb022
commit
80efc9cdbb
|
|
@ -215,6 +215,8 @@ function launching_scripts {
|
|||
# sleep 0.1
|
||||
echo -e $GREEN"\t* Launching scripts"$DEFAULT
|
||||
|
||||
screen -S "Script_AIL" -X screen -t "Mixer" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Mixer.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Global" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Global.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Categ" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Categ.py; read x"
|
||||
|
|
@ -310,10 +312,8 @@ function launching_scripts {
|
|||
##################################
|
||||
screen -S "Script_AIL" -X screen -t "ModuleInformation" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./ModulesInformationV2.py -k 0 -c 1; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Mixer" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Mixer.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "MISPtheHIVEfeeder" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./MISP_The_Hive_feeder.py; read x"
|
||||
sleep 0.1
|
||||
# screen -S "Script_AIL" -X screen -t "MISPtheHIVEfeeder" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./MISP_The_Hive_feeder.py; read x"
|
||||
# sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "IPAddress" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./IPAddress.py; read x"
|
||||
|
||||
|
||||
|
|
@ -610,7 +610,6 @@ function launch_all {
|
|||
checking_configuration;
|
||||
update;
|
||||
launch_redis;
|
||||
launch_ardb;
|
||||
launch_kvrocks;
|
||||
launch_logs;
|
||||
launch_queues;
|
||||
|
|
|
|||
|
|
@ -1,24 +1,24 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
import os
|
||||
import sys
|
||||
import uuid
|
||||
|
||||
sys.path.append(os.environ['AIL_BIN'])
|
||||
##################################
|
||||
# Import Project packages
|
||||
##################################
|
||||
from lib.ConfigLoader import ConfigLoader
|
||||
# import os
|
||||
# import sys
|
||||
# import uuid
|
||||
#
|
||||
# sys.path.append(os.environ['AIL_BIN'])
|
||||
# ##################################
|
||||
# # Import Project packages
|
||||
# ##################################
|
||||
# from lib.ConfigLoader import ConfigLoader
|
||||
|
||||
## LOAD CONFIG ##
|
||||
config_loader = ConfigLoader()
|
||||
|
||||
r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata") ######################################
|
||||
config_loader = None
|
||||
# config_loader = ConfigLoader()
|
||||
#
|
||||
# r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata") ######################################
|
||||
# config_loader = None
|
||||
## -- ##
|
||||
|
||||
sys.path.append('../../configs/keys')
|
||||
# sys.path.append('../../configs/keys')
|
||||
##################################
|
||||
# Import Keys
|
||||
##################################
|
||||
|
|
|
|||
|
|
@ -1,248 +1,248 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
import os
|
||||
import sys
|
||||
import uuid
|
||||
|
||||
from hashlib import sha1, sha256
|
||||
|
||||
sys.path.append(os.environ['AIL_BIN'])
|
||||
from lib.objects import ail_objects
|
||||
|
||||
|
||||
from lib.objects import Items
|
||||
|
||||
# MISP
|
||||
from pymisp import MISPEvent, MISPObject, PyMISP
|
||||
|
||||
# # TODO: deplace me in another fil
|
||||
def get_global_id(obj_type, obj_id, obj_subtype=None):
|
||||
if obj_subtype:
|
||||
return '{}:{}:{}'.format(obj_type, obj_subtype, obj_id)
|
||||
else:
|
||||
return '{}:{}'.format(obj_type, obj_id)
|
||||
|
||||
# sub type
|
||||
# obj type
|
||||
# obj value
|
||||
def get_global_id_from_id(global_id):
|
||||
obj_meta = {}
|
||||
global_id = global_id.split(':', 3)
|
||||
if len(global_id) > 2:
|
||||
obj_meta['type'] = global_id[0]
|
||||
obj_meta['subtype'] = global_id[1]
|
||||
obj_meta['id'] = global_id[2]
|
||||
else:
|
||||
obj_meta['type'] = global_id[0]
|
||||
obj_meta['subtype'] = ''
|
||||
obj_meta['id'] = global_id[1]
|
||||
return obj_meta
|
||||
|
||||
def get_import_dir():
|
||||
return os.path.join(os.environ['AIL_HOME'], 'temp/import')
|
||||
|
||||
def sanitize_import_file_path(filename):
|
||||
IMPORT_FOLDER = get_import_dir()
|
||||
filename = os.path.join(IMPORT_FOLDER, filename)
|
||||
filename = os.path.realpath(filename)
|
||||
# path traversal
|
||||
if not os.path.commonprefix([filename, IMPORT_FOLDER]) == IMPORT_FOLDER:
|
||||
return os.path.join(IMPORT_FOLDER, str(uuid.uuid4()) + '.json')
|
||||
# check if file already exist
|
||||
if os.path.isfile(filename):
|
||||
return os.path.join(IMPORT_FOLDER, str(uuid.uuid4()) + '.json')
|
||||
return filename
|
||||
|
||||
def get_misp_obj_tag(misp_obj):
|
||||
if misp_obj.attributes:
|
||||
misp_tags = misp_obj.attributes[0].tags
|
||||
tags = []
|
||||
for misp_tag in misp_tags:
|
||||
tags.append(misp_tag.name)
|
||||
return tags
|
||||
else:
|
||||
return []
|
||||
|
||||
def get_object_metadata(misp_obj):
|
||||
obj_meta = {}
|
||||
if 'first_seen' in misp_obj.keys():
|
||||
obj_meta['first_seen'] = misp_obj.first_seen
|
||||
if 'last_seen' in misp_obj.keys():
|
||||
obj_meta['last_seen'] = misp_obj.last_seen
|
||||
obj_meta['tags'] = get_misp_obj_tag(misp_obj)
|
||||
return obj_meta
|
||||
|
||||
def unpack_item_obj(map_uuid_global_id, misp_obj):
|
||||
obj_meta = get_object_metadata(misp_obj)
|
||||
obj_id = None
|
||||
io_content = None
|
||||
|
||||
for attribute in misp_obj.attributes:
|
||||
if attribute.object_relation == 'raw-data':
|
||||
obj_id = attribute.value # # TODO: sanitize
|
||||
io_content = attribute.data # # TODO: check if type == io
|
||||
|
||||
if obj_id and io_content:
|
||||
res = Items.create_item(obj_id, obj_meta, io_content)
|
||||
|
||||
map_uuid_global_id[misp_obj.uuid] = get_global_id('item', obj_id)
|
||||
|
||||
|
||||
|
||||
## TODO: handle multiple pgp in the same object
|
||||
def unpack_obj_pgp(map_uuid_global_id, misp_obj):
|
||||
# TODO ail_objects import_misp_object(misp_obj)
|
||||
pass
|
||||
# # get obj sub type
|
||||
# obj_attr = misp_obj.attributes[0]
|
||||
# obj_id = obj_attr.value
|
||||
# if obj_attr.object_relation == 'key-id':
|
||||
# obj_subtype = 'key'
|
||||
# elif obj_attr.object_relation == 'user-id-name':
|
||||
# obj_subtype = 'name'
|
||||
# elif obj_attr.object_relation == 'user-id-email':
|
||||
# obj_subtype = 'mail'
|
||||
#
|
||||
# if obj_id and obj_subtype:
|
||||
# obj_meta = get_object_metadata(misp_obj)
|
||||
# # res = Pgp.pgp.create_correlation(obj_subtype, obj_id, obj_meta)
|
||||
# # TODO ail_objects import_misp_object(misp_obj)
|
||||
#
|
||||
# map_uuid_global_id[misp_obj.uuid] = get_global_id('pgp', obj_id, obj_subtype=obj_subtype)
|
||||
|
||||
|
||||
def unpack_obj_cryptocurrency(map_uuid_global_id, misp_obj):
|
||||
# TODO ail_objects import_misp_object(misp_obj)
|
||||
pass
|
||||
#
|
||||
# obj_id = None
|
||||
# obj_subtype = None
|
||||
# for attribute in misp_obj.attributes:
|
||||
# if attribute.object_relation == 'address': # # TODO: handle xmr address field
|
||||
# obj_id = attribute.value
|
||||
# elif attribute.object_relation == 'symbol':
|
||||
# obj_subtype = Cryptocurrency.get_cryptocurrency_type(attribute.value)
|
||||
#
|
||||
# # valid cryptocurrency type
|
||||
# if obj_subtype and obj_id:
|
||||
# obj_meta = get_object_metadata(misp_obj)
|
||||
# # res = Cryptocurrency.cryptocurrency.create_correlation(obj_subtype, obj_id, obj_meta)
|
||||
#
|
||||
# map_uuid_global_id[misp_obj.uuid] = get_global_id('cryptocurrency', obj_id, obj_subtype=obj_subtype)
|
||||
|
||||
def get_obj_type_from_relationship(misp_obj):
|
||||
obj_uuid = misp_obj.uuid
|
||||
obj_type = None
|
||||
|
||||
for relation in misp_obj.ObjectReference:
|
||||
if relation.object_uuid == obj_uuid:
|
||||
if relation.relationship_type == "screenshot-of":
|
||||
return 'screenshot'
|
||||
if relation.relationship_type == "included-in":
|
||||
obj_type = 'decoded'
|
||||
return obj_type
|
||||
|
||||
|
||||
# # TODO: covert md5 and sha1 to expected
|
||||
def unpack_file(map_uuid_global_id, misp_obj):
|
||||
|
||||
obj_type = get_obj_type_from_relationship(misp_obj)
|
||||
if obj_type:
|
||||
obj_id = None
|
||||
io_content = None
|
||||
for attribute in misp_obj.attributes:
|
||||
# get file content
|
||||
if attribute.object_relation == 'attachment':
|
||||
io_content = attribute.data
|
||||
elif attribute.object_relation == 'malware-sample':
|
||||
io_content = attribute.data
|
||||
|
||||
# # TODO: use/verify specified mimetype
|
||||
elif attribute.object_relation == 'mimetype':
|
||||
#print(attribute.value)
|
||||
pass
|
||||
|
||||
# # TODO: support more
|
||||
elif attribute.object_relation == 'sha1' and obj_type == 'decoded':
|
||||
obj_id = attribute.value
|
||||
elif attribute.object_relation == 'sha256' and obj_type == 'screenshot':
|
||||
obj_id = attribute.value
|
||||
|
||||
# get SHA1/sha256
|
||||
if io_content and not obj_id:
|
||||
if obj_type=='screenshot':
|
||||
obj_id = sha256(io_content.getvalue()).hexdigest()
|
||||
else: # decoded file
|
||||
obj_id = sha1(io_content.getvalue()).hexdigest()
|
||||
|
||||
if obj_id and io_content:
|
||||
obj_meta = get_object_metadata(misp_obj)
|
||||
if obj_type == 'screenshot':
|
||||
# TODO MIGRATE + REFACTOR ME
|
||||
# Screenshot.create_screenshot(obj_id, obj_meta, io_content)
|
||||
map_uuid_global_id[misp_obj.uuid] = get_global_id('image', obj_id)
|
||||
else: #decoded
|
||||
# TODO MIGRATE + REFACTOR ME
|
||||
# Decoded.create_decoded(obj_id, obj_meta, io_content)
|
||||
map_uuid_global_id[misp_obj.uuid] = get_global_id('decoded', obj_id)
|
||||
|
||||
|
||||
def get_misp_import_fct(map_uuid_global_id, misp_obj):
|
||||
if misp_obj.name == 'ail-leak':
|
||||
unpack_item_obj(map_uuid_global_id, misp_obj)
|
||||
elif misp_obj.name == 'domain-crawled':
|
||||
pass
|
||||
elif misp_obj.name == 'pgp-meta':
|
||||
unpack_obj_pgp(map_uuid_global_id, misp_obj)
|
||||
elif misp_obj.name == 'coin-address':
|
||||
unpack_obj_cryptocurrency(map_uuid_global_id, misp_obj)
|
||||
elif misp_obj.name == 'file':
|
||||
unpack_file(map_uuid_global_id, misp_obj)
|
||||
|
||||
# import relationship between objects
|
||||
def create_obj_relationships(map_uuid_global_id, misp_obj):
|
||||
if misp_obj.uuid in map_uuid_global_id:
|
||||
for relationship in misp_obj.ObjectReference:
|
||||
if relationship.referenced_uuid in map_uuid_global_id:
|
||||
obj_meta_src = get_global_id_from_id(map_uuid_global_id[relationship.object_uuid])
|
||||
obj_meta_target = get_global_id_from_id(map_uuid_global_id[relationship.referenced_uuid])
|
||||
|
||||
if obj_meta_src == 'decoded' or obj_meta_src == 'item':
|
||||
print('000000')
|
||||
print(obj_meta_src)
|
||||
print(obj_meta_target)
|
||||
print('111111')
|
||||
|
||||
# TODO CREATE OBJ RELATIONSHIP
|
||||
|
||||
def import_objs_from_file(filepath):
|
||||
map_uuid_global_id = {}
|
||||
|
||||
event_to_import = MISPEvent()
|
||||
try:
|
||||
event_to_import.load_file(filepath)
|
||||
except:
|
||||
return map_uuid_global_id
|
||||
|
||||
for misp_obj in event_to_import.objects:
|
||||
get_misp_import_fct(map_uuid_global_id, misp_obj)
|
||||
|
||||
for misp_obj in event_to_import.objects:
|
||||
create_obj_relationships(map_uuid_global_id, misp_obj)
|
||||
|
||||
return map_uuid_global_id
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
# misp = PyMISP('https://127.0.0.1:8443/', 'uXgcN42b7xuL88XqK5hubwD8Q8596VrrBvkHQzB0', False)
|
||||
|
||||
import_objs_from_file('ail_export_c777a4d1-5f63-4fa2-86c0-07da677bdac2.json')
|
||||
|
||||
#Screenshot.delete_screenshot('a92d459f70c4dea8a14688f585a5e2364be8b91fbf924290ead361d9b909dcf1')
|
||||
#Decoded.delete_decoded('d59a110ab233fe87cefaa0cf5603b047b432ee07')
|
||||
#Pgp.pgp.delete_correlation('key', '0xA4BB02A75E6AF448')
|
||||
|
||||
#Item.delete_item('submitted/2020/02/10/b2485894-4325-469b-bc8f-6ad1c2dbb202.gz')
|
||||
#Item.delete_item('archive/pastebin.com_pro/2020/02/10/K2cerjP4.gz')
|
||||
# import os
|
||||
# import sys
|
||||
# import uuid
|
||||
#
|
||||
# from hashlib import sha1, sha256
|
||||
#
|
||||
# sys.path.append(os.environ['AIL_BIN'])
|
||||
# from lib.objects import ail_objects
|
||||
#
|
||||
#
|
||||
# from lib.objects import Items
|
||||
#
|
||||
# # MISP
|
||||
# from pymisp import MISPEvent, MISPObject, PyMISP
|
||||
#
|
||||
# # # TODO: deplace me in another fil
|
||||
# def get_global_id(obj_type, obj_id, obj_subtype=None):
|
||||
# if obj_subtype:
|
||||
# return '{}:{}:{}'.format(obj_type, obj_subtype, obj_id)
|
||||
# else:
|
||||
# return '{}:{}'.format(obj_type, obj_id)
|
||||
#
|
||||
# # sub type
|
||||
# # obj type
|
||||
# # obj value
|
||||
# def get_global_id_from_id(global_id):
|
||||
# obj_meta = {}
|
||||
# global_id = global_id.split(':', 3)
|
||||
# if len(global_id) > 2:
|
||||
# obj_meta['type'] = global_id[0]
|
||||
# obj_meta['subtype'] = global_id[1]
|
||||
# obj_meta['id'] = global_id[2]
|
||||
# else:
|
||||
# obj_meta['type'] = global_id[0]
|
||||
# obj_meta['subtype'] = ''
|
||||
# obj_meta['id'] = global_id[1]
|
||||
# return obj_meta
|
||||
#
|
||||
# def get_import_dir():
|
||||
# return os.path.join(os.environ['AIL_HOME'], 'temp/import')
|
||||
#
|
||||
# def sanitize_import_file_path(filename):
|
||||
# IMPORT_FOLDER = get_import_dir()
|
||||
# filename = os.path.join(IMPORT_FOLDER, filename)
|
||||
# filename = os.path.realpath(filename)
|
||||
# # path traversal
|
||||
# if not os.path.commonprefix([filename, IMPORT_FOLDER]) == IMPORT_FOLDER:
|
||||
# return os.path.join(IMPORT_FOLDER, str(uuid.uuid4()) + '.json')
|
||||
# # check if file already exist
|
||||
# if os.path.isfile(filename):
|
||||
# return os.path.join(IMPORT_FOLDER, str(uuid.uuid4()) + '.json')
|
||||
# return filename
|
||||
#
|
||||
# def get_misp_obj_tag(misp_obj):
|
||||
# if misp_obj.attributes:
|
||||
# misp_tags = misp_obj.attributes[0].tags
|
||||
# tags = []
|
||||
# for misp_tag in misp_tags:
|
||||
# tags.append(misp_tag.name)
|
||||
# return tags
|
||||
# else:
|
||||
# return []
|
||||
#
|
||||
# def get_object_metadata(misp_obj):
|
||||
# obj_meta = {}
|
||||
# if 'first_seen' in misp_obj.keys():
|
||||
# obj_meta['first_seen'] = misp_obj.first_seen
|
||||
# if 'last_seen' in misp_obj.keys():
|
||||
# obj_meta['last_seen'] = misp_obj.last_seen
|
||||
# obj_meta['tags'] = get_misp_obj_tag(misp_obj)
|
||||
# return obj_meta
|
||||
#
|
||||
# def unpack_item_obj(map_uuid_global_id, misp_obj):
|
||||
# obj_meta = get_object_metadata(misp_obj)
|
||||
# obj_id = None
|
||||
# io_content = None
|
||||
#
|
||||
# for attribute in misp_obj.attributes:
|
||||
# if attribute.object_relation == 'raw-data':
|
||||
# obj_id = attribute.value # # TODO: sanitize
|
||||
# io_content = attribute.data # # TODO: check if type == io
|
||||
#
|
||||
# if obj_id and io_content:
|
||||
# res = Items.create_item(obj_id, obj_meta, io_content)
|
||||
#
|
||||
# map_uuid_global_id[misp_obj.uuid] = get_global_id('item', obj_id)
|
||||
#
|
||||
#
|
||||
#
|
||||
# ## TODO: handle multiple pgp in the same object
|
||||
# def unpack_obj_pgp(map_uuid_global_id, misp_obj):
|
||||
# # TODO ail_objects import_misp_object(misp_obj)
|
||||
# pass
|
||||
# # # get obj sub type
|
||||
# # obj_attr = misp_obj.attributes[0]
|
||||
# # obj_id = obj_attr.value
|
||||
# # if obj_attr.object_relation == 'key-id':
|
||||
# # obj_subtype = 'key'
|
||||
# # elif obj_attr.object_relation == 'user-id-name':
|
||||
# # obj_subtype = 'name'
|
||||
# # elif obj_attr.object_relation == 'user-id-email':
|
||||
# # obj_subtype = 'mail'
|
||||
# #
|
||||
# # if obj_id and obj_subtype:
|
||||
# # obj_meta = get_object_metadata(misp_obj)
|
||||
# # # res = Pgp.pgp.create_correlation(obj_subtype, obj_id, obj_meta)
|
||||
# # # TODO ail_objects import_misp_object(misp_obj)
|
||||
# #
|
||||
# # map_uuid_global_id[misp_obj.uuid] = get_global_id('pgp', obj_id, obj_subtype=obj_subtype)
|
||||
#
|
||||
#
|
||||
# def unpack_obj_cryptocurrency(map_uuid_global_id, misp_obj):
|
||||
# # TODO ail_objects import_misp_object(misp_obj)
|
||||
# pass
|
||||
# #
|
||||
# # obj_id = None
|
||||
# # obj_subtype = None
|
||||
# # for attribute in misp_obj.attributes:
|
||||
# # if attribute.object_relation == 'address': # # TODO: handle xmr address field
|
||||
# # obj_id = attribute.value
|
||||
# # elif attribute.object_relation == 'symbol':
|
||||
# # obj_subtype = Cryptocurrency.get_cryptocurrency_type(attribute.value)
|
||||
# #
|
||||
# # # valid cryptocurrency type
|
||||
# # if obj_subtype and obj_id:
|
||||
# # obj_meta = get_object_metadata(misp_obj)
|
||||
# # # res = Cryptocurrency.cryptocurrency.create_correlation(obj_subtype, obj_id, obj_meta)
|
||||
# #
|
||||
# # map_uuid_global_id[misp_obj.uuid] = get_global_id('cryptocurrency', obj_id, obj_subtype=obj_subtype)
|
||||
#
|
||||
# def get_obj_type_from_relationship(misp_obj):
|
||||
# obj_uuid = misp_obj.uuid
|
||||
# obj_type = None
|
||||
#
|
||||
# for relation in misp_obj.ObjectReference:
|
||||
# if relation.object_uuid == obj_uuid:
|
||||
# if relation.relationship_type == "screenshot-of":
|
||||
# return 'screenshot'
|
||||
# if relation.relationship_type == "included-in":
|
||||
# obj_type = 'decoded'
|
||||
# return obj_type
|
||||
#
|
||||
#
|
||||
# # # TODO: covert md5 and sha1 to expected
|
||||
# def unpack_file(map_uuid_global_id, misp_obj):
|
||||
#
|
||||
# obj_type = get_obj_type_from_relationship(misp_obj)
|
||||
# if obj_type:
|
||||
# obj_id = None
|
||||
# io_content = None
|
||||
# for attribute in misp_obj.attributes:
|
||||
# # get file content
|
||||
# if attribute.object_relation == 'attachment':
|
||||
# io_content = attribute.data
|
||||
# elif attribute.object_relation == 'malware-sample':
|
||||
# io_content = attribute.data
|
||||
#
|
||||
# # # TODO: use/verify specified mimetype
|
||||
# elif attribute.object_relation == 'mimetype':
|
||||
# #print(attribute.value)
|
||||
# pass
|
||||
#
|
||||
# # # TODO: support more
|
||||
# elif attribute.object_relation == 'sha1' and obj_type == 'decoded':
|
||||
# obj_id = attribute.value
|
||||
# elif attribute.object_relation == 'sha256' and obj_type == 'screenshot':
|
||||
# obj_id = attribute.value
|
||||
#
|
||||
# # get SHA1/sha256
|
||||
# if io_content and not obj_id:
|
||||
# if obj_type=='screenshot':
|
||||
# obj_id = sha256(io_content.getvalue()).hexdigest()
|
||||
# else: # decoded file
|
||||
# obj_id = sha1(io_content.getvalue()).hexdigest()
|
||||
#
|
||||
# if obj_id and io_content:
|
||||
# obj_meta = get_object_metadata(misp_obj)
|
||||
# if obj_type == 'screenshot':
|
||||
# # TODO MIGRATE + REFACTOR ME
|
||||
# # Screenshot.create_screenshot(obj_id, obj_meta, io_content)
|
||||
# map_uuid_global_id[misp_obj.uuid] = get_global_id('image', obj_id)
|
||||
# else: #decoded
|
||||
# # TODO MIGRATE + REFACTOR ME
|
||||
# # Decoded.create_decoded(obj_id, obj_meta, io_content)
|
||||
# map_uuid_global_id[misp_obj.uuid] = get_global_id('decoded', obj_id)
|
||||
#
|
||||
#
|
||||
# def get_misp_import_fct(map_uuid_global_id, misp_obj):
|
||||
# if misp_obj.name == 'ail-leak':
|
||||
# unpack_item_obj(map_uuid_global_id, misp_obj)
|
||||
# elif misp_obj.name == 'domain-crawled':
|
||||
# pass
|
||||
# elif misp_obj.name == 'pgp-meta':
|
||||
# unpack_obj_pgp(map_uuid_global_id, misp_obj)
|
||||
# elif misp_obj.name == 'coin-address':
|
||||
# unpack_obj_cryptocurrency(map_uuid_global_id, misp_obj)
|
||||
# elif misp_obj.name == 'file':
|
||||
# unpack_file(map_uuid_global_id, misp_obj)
|
||||
#
|
||||
# # import relationship between objects
|
||||
# def create_obj_relationships(map_uuid_global_id, misp_obj):
|
||||
# if misp_obj.uuid in map_uuid_global_id:
|
||||
# for relationship in misp_obj.ObjectReference:
|
||||
# if relationship.referenced_uuid in map_uuid_global_id:
|
||||
# obj_meta_src = get_global_id_from_id(map_uuid_global_id[relationship.object_uuid])
|
||||
# obj_meta_target = get_global_id_from_id(map_uuid_global_id[relationship.referenced_uuid])
|
||||
#
|
||||
# if obj_meta_src == 'decoded' or obj_meta_src == 'item':
|
||||
# print('000000')
|
||||
# print(obj_meta_src)
|
||||
# print(obj_meta_target)
|
||||
# print('111111')
|
||||
#
|
||||
# # TODO CREATE OBJ RELATIONSHIP
|
||||
#
|
||||
# def import_objs_from_file(filepath):
|
||||
# map_uuid_global_id = {}
|
||||
#
|
||||
# event_to_import = MISPEvent()
|
||||
# try:
|
||||
# event_to_import.load_file(filepath)
|
||||
# except:
|
||||
# return map_uuid_global_id
|
||||
#
|
||||
# for misp_obj in event_to_import.objects:
|
||||
# get_misp_import_fct(map_uuid_global_id, misp_obj)
|
||||
#
|
||||
# for misp_obj in event_to_import.objects:
|
||||
# create_obj_relationships(map_uuid_global_id, misp_obj)
|
||||
#
|
||||
# return map_uuid_global_id
|
||||
#
|
||||
#
|
||||
# if __name__ == '__main__':
|
||||
#
|
||||
# # misp = PyMISP('https://127.0.0.1:8443/', 'uXgcN42b7xuL88XqK5hubwD8Q8596VrrBvkHQzB0', False)
|
||||
#
|
||||
# import_objs_from_file('ail_export_c777a4d1-5f63-4fa2-86c0-07da677bdac2.json')
|
||||
#
|
||||
# #Screenshot.delete_screenshot('a92d459f70c4dea8a14688f585a5e2364be8b91fbf924290ead361d9b909dcf1')
|
||||
# #Decoded.delete_decoded('d59a110ab233fe87cefaa0cf5603b047b432ee07')
|
||||
# #Pgp.pgp.delete_correlation('key', '0xA4BB02A75E6AF448')
|
||||
#
|
||||
# #Item.delete_item('submitted/2020/02/10/b2485894-4325-469b-bc8f-6ad1c2dbb202.gz')
|
||||
# #Item.delete_item('archive/pastebin.com_pro/2020/02/10/K2cerjP4.gz')
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
|
||||
import os
|
||||
import sys
|
||||
import redis
|
||||
from uuid import uuid4
|
||||
|
||||
sys.path.append(os.environ['AIL_BIN'])
|
||||
##################################
|
||||
|
|
@ -20,6 +20,9 @@ AIL_OBJECTS = sorted({'cve', 'cryptocurrency', 'decoded', 'domain', 'item', 'pgp
|
|||
def get_ail_uuid():
|
||||
return r_serv_db.get('ail:uuid')
|
||||
|
||||
def generate_uuid():
|
||||
return str(uuid4())
|
||||
|
||||
#### AIL OBJECTS ####
|
||||
|
||||
def get_all_objects():
|
||||
|
|
|
|||
|
|
@ -17,7 +17,6 @@ from lib import Tag
|
|||
|
||||
config_loader = ConfigLoader.ConfigLoader()
|
||||
r_cache = config_loader.get_redis_conn("Redis_Cache")
|
||||
r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata")
|
||||
r_object = config_loader.get_db_conn("Kvrocks_Objects")
|
||||
config_loader = None
|
||||
|
||||
|
|
@ -180,21 +179,21 @@ def get_all_domain_node_by_item_id(item_id, l_nodes=[]):
|
|||
# FIXME:
|
||||
#### UNKNOW SECTION ####
|
||||
|
||||
def get_obj_id_item_id(parent_type, parent_id):
|
||||
all_parents_type = ['twitter_id', 'jabber_id', 'telegram_id']
|
||||
if parent_type in all_parents_type:
|
||||
return r_serv_metadata.hget('map:{}:item_id'.format(parent_type), parent_id)
|
||||
else:
|
||||
return None
|
||||
# def get_obj_id_item_id(parent_type, parent_id):
|
||||
# all_parents_type = ['twitter_id', 'jabber_id', 'telegram_id']
|
||||
# if parent_type in all_parents_type:
|
||||
# return r_serv_metadata.hget('map:{}:item_id'.format(parent_type), parent_id)
|
||||
# else:
|
||||
# return None
|
||||
|
||||
# # TODO: # FIXME: TO MIGRATE ??????
|
||||
def add_map_obj_id_item_id(obj_id, item_id, obj_type):
|
||||
if obj_type == 'twitter_id':
|
||||
r_serv_metadata.hset('map:twitter_id:item_id', obj_id, item_id)
|
||||
if obj_type == 'jabber_id':
|
||||
r_serv_metadata.hset('map:jabber_id:item_id', obj_id, item_id)
|
||||
if obj_type == 'telegram_id':
|
||||
r_serv_metadata.hset('map:telegram_id:item_id', obj_id, item_id)
|
||||
# # # TODO: # FIXME: TO MIGRATE ??????
|
||||
# def add_map_obj_id_item_id(obj_id, item_id, obj_type):
|
||||
# if obj_type == 'twitter_id':
|
||||
# r_serv_metadata.hset('map:twitter_id:item_id', obj_id, item_id)
|
||||
# if obj_type == 'jabber_id':
|
||||
# r_serv_metadata.hset('map:jabber_id:item_id', obj_id, item_id)
|
||||
# if obj_type == 'telegram_id':
|
||||
# r_serv_metadata.hset('map:telegram_id:item_id', obj_id, item_id)
|
||||
|
||||
# delete twitter id
|
||||
|
||||
|
|
|
|||
|
|
@ -5,8 +5,6 @@
|
|||
The Submit paste module
|
||||
================
|
||||
|
||||
This module is taking paste in redis queue ARDB_DB and submit to global
|
||||
|
||||
"""
|
||||
|
||||
##################################
|
||||
|
|
@ -15,7 +13,6 @@ This module is taking paste in redis queue ARDB_DB and submit to global
|
|||
import os
|
||||
import sys
|
||||
import gzip
|
||||
import io
|
||||
import base64
|
||||
import datetime
|
||||
import time
|
||||
|
|
@ -51,11 +48,8 @@ class SubmitPaste(AbstractModule):
|
|||
super(SubmitPaste, self).__init__(queue_name='submit_paste')
|
||||
|
||||
# TODO KVROCKS
|
||||
self.r_serv_db = ConfigLoader.ConfigLoader().get_redis_conn("ARDB_DB")
|
||||
self.r_serv_db = ConfigLoader.ConfigLoader().get_redis_conn("Kvrocks_DB")
|
||||
self.r_serv_log_submit = ConfigLoader.ConfigLoader().get_redis_conn("Redis_Log_submit")
|
||||
self.r_serv_tags = ConfigLoader.ConfigLoader().get_redis_conn("ARDB_Tags")
|
||||
self.r_serv_metadata = ConfigLoader.ConfigLoader().get_redis_conn("ARDB_Metadata")
|
||||
self.serv_statistics = ConfigLoader.ConfigLoader().get_redis_conn("ARDB_Statistics")
|
||||
|
||||
self.pending_seconds = 3
|
||||
|
||||
|
|
@ -305,7 +299,6 @@ class SubmitPaste(AbstractModule):
|
|||
self.r_serv_log_submit.sadd(f'{uuid}:paste_submit_link', rel_item_path)
|
||||
|
||||
curr_date = datetime.date.today()
|
||||
self.serv_statistics.hincrby(curr_date.strftime("%Y%m%d"),'submit_paste', 1)
|
||||
self.redis_logger.debug("paste submitted")
|
||||
else:
|
||||
self.addError(uuid, f'File: {save_path} already exist in submitted pastes')
|
||||
|
|
@ -335,7 +328,6 @@ class SubmitPaste(AbstractModule):
|
|||
self.addError(uuid, errorMessage)
|
||||
self.r_serv_log_submit.set(f'{uuid}:end', 1)
|
||||
curr_date = datetime.date.today()
|
||||
self.serv_statistics.hincrby(curr_date.strftime("%Y%m%d"), 'submit_abord', 1)
|
||||
self.remove_submit_uuid(uuid)
|
||||
|
||||
# # TODO: use Item function
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ from lib import ConfigLoader
|
|||
|
||||
|
||||
config_loader = ConfigLoader.ConfigLoader()
|
||||
r_serv_db = config_loader.get_redis_conn("ARDB_DB")
|
||||
r_serv_db = config_loader.get_redis_conn("Kvrocks_DB")
|
||||
r_serv_log_submit = config_loader.get_redis_conn("Redis_Log_submit")
|
||||
config_loader = None
|
||||
|
||||
|
|
@ -62,11 +62,11 @@ def create_import_queue(tags, galaxy, paste_content, UUID, password=None, isfile
|
|||
|
||||
def check_import_status(UUID):
|
||||
if not is_valid_uuid_v4(UUID):
|
||||
return ({'status': 'error', 'reason': 'Invalid uuid'}, 400)
|
||||
return {'status': 'error', 'reason': 'Invalid uuid'}, 400
|
||||
|
||||
processing = r_serv_log_submit.get(UUID + ':processing')
|
||||
if not processing:
|
||||
return ({'status': 'error', 'reason': 'Unknown uuid'}, 404)
|
||||
return {'status': 'error', 'reason': 'Unknown uuid'}, 404
|
||||
|
||||
# nb_total = r_serv_log_submit.get(UUID + ':nb_total')
|
||||
# nb_sucess = r_serv_log_submit.get(UUID + ':nb_sucess')
|
||||
|
|
@ -90,4 +90,4 @@ def check_import_status(UUID):
|
|||
status = 'imported'
|
||||
dict_import_status['status'] = status
|
||||
|
||||
return (dict_import_status, 200)
|
||||
return dict_import_status, 200
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ if [ -z "$VIRTUAL_ENV" ]; then
|
|||
echo export AIL_BIN=$(pwd)/bin/ >> ./AILENV/bin/activate
|
||||
echo export AIL_FLASK=$(pwd)/var/www/ >> ./AILENV/bin/activate
|
||||
echo export AIL_REDIS=$(pwd)/redis/src/ >> ./AILENV/bin/activate
|
||||
echo export AIL_ARDB=$(pwd)/ardb/src/ >> ./AILENV/bin/activate
|
||||
echo export AIL_KVROCKS=$(pwd)/kvrocks/src/ >> ./AILENV/bin/activate
|
||||
|
||||
fi
|
||||
|
||||
|
|
|
|||
|
|
@ -84,10 +84,10 @@ sudo make install
|
|||
popd
|
||||
|
||||
# ARDB #
|
||||
test ! -d ardb/ && git clone https://github.com/ail-project/ardb.git
|
||||
pushd ardb/
|
||||
make
|
||||
popd
|
||||
#test ! -d ardb/ && git clone https://github.com/ail-project/ardb.git
|
||||
#pushd ardb/
|
||||
#make
|
||||
#popd
|
||||
|
||||
DEFAULT_HOME=$(pwd)
|
||||
|
||||
|
|
@ -148,8 +148,8 @@ echo "AIL current version:"
|
|||
git describe --abbrev=0 --tags
|
||||
popd
|
||||
|
||||
# LAUNCH ARDB
|
||||
bash ${AIL_BIN}/LAUNCH.sh -lav &
|
||||
# LAUNCH Kvrocks
|
||||
bash ${AIL_BIN}/LAUNCH.sh -lkv &
|
||||
wait
|
||||
echo ""
|
||||
|
||||
|
|
|
|||
|
|
@ -25,9 +25,7 @@ from flask_login import login_required
|
|||
##################################
|
||||
# Import Project packages
|
||||
##################################
|
||||
from export import Export
|
||||
from lib import Tag
|
||||
from lib.objects.Items import Item
|
||||
|
||||
from packages import Import_helper
|
||||
|
||||
|
|
@ -75,7 +73,7 @@ def limit_content_length():
|
|||
# ============ FUNCTIONS ============
|
||||
|
||||
def allowed_file(filename):
|
||||
if not '.' in filename:
|
||||
if '.' not in filename:
|
||||
return True
|
||||
else:
|
||||
file_ext = filename.rsplit('.', 1)[1].lower()
|
||||
|
|
@ -86,7 +84,7 @@ def allowed_file(filename):
|
|||
def clean_filename(filename, whitelist=valid_filename_chars, replace=' '):
|
||||
# replace characters
|
||||
for r in replace:
|
||||
filename = filename.replace(r,'_')
|
||||
filename = filename.replace(r, '_')
|
||||
|
||||
# keep only valid ascii chars
|
||||
cleaned_filename = unicodedata.normalize('NFKD', filename).encode('ASCII', 'ignore').decode()
|
||||
|
|
@ -116,8 +114,6 @@ def PasteSubmit_page():
|
|||
@login_analyst
|
||||
@limit_content_length()
|
||||
def submit():
|
||||
|
||||
#paste_name = request.form['paste_name']
|
||||
logger.debug('submit')
|
||||
|
||||
password = request.form['archive_pass']
|
||||
|
|
@ -127,10 +123,10 @@ def submit():
|
|||
paste_source = request.form['paste_source']
|
||||
|
||||
if paste_source:
|
||||
# limit source length
|
||||
# limit source length
|
||||
paste_source = paste_source.replace('/', '')[:80]
|
||||
if paste_source in ['crawled', 'tests']:
|
||||
content = f'Invalid source'
|
||||
content = 'Invalid source'
|
||||
logger.info(paste_source)
|
||||
return content, 400
|
||||
|
||||
|
|
@ -150,9 +146,9 @@ def submit():
|
|||
|
||||
submitted_tag = 'infoleak:submission="manual"'
|
||||
|
||||
#active taxonomies
|
||||
# active taxonomies
|
||||
active_taxonomies = Tag.get_active_taxonomies()
|
||||
#active galaxies
|
||||
# active galaxies
|
||||
active_galaxies = Tag.get_active_galaxies()
|
||||
|
||||
if ltags or ltagsgalaxies:
|
||||
|
|
@ -179,16 +175,12 @@ def submit():
|
|||
# get UUID
|
||||
UUID = str(uuid.uuid4())
|
||||
|
||||
'''if paste_name:
|
||||
# clean file name
|
||||
UUID = clean_filename(paste_name)'''
|
||||
|
||||
# create submitted dir
|
||||
if not os.path.exists(UPLOAD_FOLDER):
|
||||
logger.debug('create folder')
|
||||
os.makedirs(UPLOAD_FOLDER)
|
||||
|
||||
if not '.' in file_import.filename:
|
||||
if '.' not in file_import.filename:
|
||||
logger.debug('add UUID to path')
|
||||
full_path = os.path.join(UPLOAD_FOLDER, UUID)
|
||||
else:
|
||||
|
|
@ -202,23 +194,22 @@ def submit():
|
|||
full_path = os.path.join(UPLOAD_FOLDER, name)
|
||||
logger.debug(f'full path {full_path}')
|
||||
|
||||
#Flask verify the file size
|
||||
# Flask verify the file size
|
||||
file_import.save(full_path)
|
||||
logger.debug('file saved')
|
||||
|
||||
Import_helper.create_import_queue(ltags, ltagsgalaxies, full_path, UUID, password, True)
|
||||
|
||||
return render_template("submit_items.html",
|
||||
active_taxonomies = active_taxonomies,
|
||||
active_galaxies = active_galaxies,
|
||||
UUID = UUID)
|
||||
active_taxonomies=active_taxonomies,
|
||||
active_galaxies=active_galaxies,
|
||||
UUID=UUID)
|
||||
|
||||
else:
|
||||
content = f'wrong file type, allowed_extensions: {allowed_extensions} or remove the extension'
|
||||
logger.info(content)
|
||||
return content, 400
|
||||
|
||||
|
||||
elif paste_content != '':
|
||||
logger.debug(f'entering text paste management')
|
||||
if sys.getsizeof(paste_content) < Flask_config.SUBMIT_PASTE_TEXT_MAX_SIZE:
|
||||
|
|
@ -242,7 +233,6 @@ def submit():
|
|||
logger.error(content)
|
||||
return content, 400
|
||||
|
||||
|
||||
return PasteSubmit_page()
|
||||
|
||||
@PasteSubmit.route("/PasteSubmit/submit_status", methods=['GET'])
|
||||
|
|
@ -278,10 +268,7 @@ def submit_status():
|
|||
else:
|
||||
prog = 0
|
||||
|
||||
if error:
|
||||
isError = True
|
||||
else:
|
||||
isError = False
|
||||
isError = bool(error)
|
||||
|
||||
if end == '0':
|
||||
end = False
|
||||
|
|
@ -327,6 +314,8 @@ def submit_status():
|
|||
@login_required
|
||||
@login_analyst
|
||||
def edit_tag_export():
|
||||
return abort(404)
|
||||
|
||||
misp_auto_events = r_serv_db.get('misp:auto-events')
|
||||
hive_auto_alerts = r_serv_db.get('hive:auto-alerts')
|
||||
|
||||
|
|
@ -393,6 +382,9 @@ def edit_tag_export():
|
|||
@login_required
|
||||
@login_analyst
|
||||
def tag_export_edited():
|
||||
return abort(404)
|
||||
|
||||
|
||||
tag_enabled_misp = request.form.getlist('tag_enabled_misp')
|
||||
tag_enabled_hive = request.form.getlist('tag_enabled_hive')
|
||||
|
||||
|
|
@ -419,6 +411,8 @@ def tag_export_edited():
|
|||
@login_required
|
||||
@login_analyst
|
||||
def enable_misp_auto_event():
|
||||
return abort(404)
|
||||
|
||||
r_serv_db.set('misp:auto-events', 1)
|
||||
return edit_tag_export()
|
||||
|
||||
|
|
@ -426,6 +420,8 @@ def enable_misp_auto_event():
|
|||
@login_required
|
||||
@login_analyst
|
||||
def disable_misp_auto_event():
|
||||
return abort(404)
|
||||
|
||||
r_serv_db.set('misp:auto-events', 0)
|
||||
return edit_tag_export()
|
||||
|
||||
|
|
@ -433,6 +429,8 @@ def disable_misp_auto_event():
|
|||
@login_required
|
||||
@login_analyst
|
||||
def enable_hive_auto_alert():
|
||||
return abort(404)
|
||||
|
||||
r_serv_db.set('hive:auto-alerts', 1)
|
||||
return edit_tag_export()
|
||||
|
||||
|
|
@ -440,6 +438,8 @@ def enable_hive_auto_alert():
|
|||
@login_required
|
||||
@login_analyst
|
||||
def disable_hive_auto_alert():
|
||||
return abort(404)
|
||||
|
||||
r_serv_db.set('hive:auto-alerts', 0)
|
||||
return edit_tag_export()
|
||||
|
||||
|
|
@ -447,6 +447,8 @@ def disable_hive_auto_alert():
|
|||
@login_required
|
||||
@login_analyst
|
||||
def add_push_tag():
|
||||
return abort(404)
|
||||
|
||||
tag = request.args.get('tag')
|
||||
if tag is not None:
|
||||
|
||||
|
|
@ -466,6 +468,9 @@ def add_push_tag():
|
|||
@login_required
|
||||
@login_analyst
|
||||
def delete_push_tag():
|
||||
return abort(404)
|
||||
|
||||
|
||||
tag = request.args.get('tag')
|
||||
|
||||
infoleak_tags = Taxonomies().get('infoleak').machinetags()
|
||||
|
|
|
|||
|
|
@ -69,11 +69,11 @@
|
|||
</span>
|
||||
</h5>
|
||||
<ul class="nav flex-md-column flex-row navbar-nav justify-content-between w-100">
|
||||
<li class="nav-item">
|
||||
<a class="nav-link" href="{{url_for('import_export.import_object')}}" id="nav_misp_import">
|
||||
<b>Import</b>
|
||||
</a>
|
||||
</li>
|
||||
{# <li class="nav-item">#}
|
||||
{# <a class="nav-link" href="{{url_for('import_export.import_object')}}" id="nav_misp_import">#}
|
||||
{# <b>Import</b>#}
|
||||
{# </a>#}
|
||||
{# </li>#}
|
||||
<li class="nav-item">
|
||||
<a class="nav-link" href="{{url_for('import_export.objects_misp_export')}}" id="nav_misp_export">
|
||||
<b>Export</b>
|
||||
|
|
|
|||
Loading…
Reference in New Issue